diff --git a/.docker/build-infrastructure-clang/Dockerfile b/.docker/build-infrastructure-clang/Dockerfile deleted file mode 100644 index 2d2726a..0000000 --- a/.docker/build-infrastructure-clang/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# -# YugabyteDB build. -# These steps are roughly based on: -# https://docs.yugabyte.com/latest/contribute/core-database/build-from-src-centos/ -# -FROM almalinux:8.5 - -ARG MVN_VERSION=3.8.4 - -COPY usr/local/bin/patch_postgres_h.sh /usr/local/bin/patch_postgres_h.sh -COPY usr/local/bin/yb-first-pass-build.sh /usr/local/bin/yb-first-pass-build.sh -COPY usr/local/bin/yb-rebuild-extensions.sh /usr/local/bin/yb-rebuild-extensions.sh -COPY usr/local/bin/yb-rebuild.sh /usr/local/bin/yb-rebuild.sh -COPY usr/local/bin/yb-release.sh /usr/local/bin/yb-release.sh -COPY usr/local/bin/yb-tests.sh /usr/local/bin/yb-tests.sh - -RUN chmod +x /usr/local/bin/yb-first-pass-build.sh \ - && chmod +x /usr/local/bin/yb-rebuild-extensions.sh \ - && chmod +x /usr/local/bin/yb-rebuild.sh \ - && chmod +x /usr/local/bin/yb-release.sh \ - && chmod +x /usr/local/bin/yb-tests.sh \ - && chmod +x /usr/local/bin/patch_postgres_h.sh \ - && echo 'YugabyteDB build tooling' \ - && dnf update -y \ - && dnf group install "Development Tools" -y \ - && dnf install bzip2 clang cmake curl java-1.8.0-openjdk java-1.8.0-openjdk-devel libatomic python3 python3-pip python3-devel rsync wget -y \ - && echo 'Enabling required locale for tests' \ - && dnf install langpacks-en glibc-all-langpacks -y \ - && wget https://dlcdn.apache.org/maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz -P /tmp \ - && tar xf /tmp/apache-maven-${MVN_VERSION}-bin.tar.gz -C /opt \ - && ln -s /opt/apache-maven-${MVN_VERSION}/bin/mvn /usr/bin/mvn - -VOLUME /root/.m2 -VOLUME /opt/yb-build -VOLUME /yb-source diff --git a/.docker/build-infrastructure-clang/usr/local/bin/yb-first-pass-build.sh b/.docker/build-infrastructure-clang/usr/local/bin/yb-first-pass-build.sh deleted file mode 100644 index 919481e..0000000 --- a/.docker/build-infrastructure-clang/usr/local/bin/yb-first-pass-build.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -eu -# working directory -cd /yb-source -# ensure the source code -if [ ! -d "./.git" ]; then - echo "Checking out '${YB_REPOSITORY}'..." - git clone "${YB_REPOSITORY}" . -else - echo "'${YB_REPOSITORY}' already checked out..." -fi -# checkout the version to work with -git checkout "${YB_SOURCE_VERSION}" -# optionally, install extensions for compilation -extra_extensions="" -count=$(find /extensions/ -maxdepth 1 -type d | grep -v '^/extensions/$' | wc -l) -if [ $count -ne 0 ]; then - for d in /extensions/*/ ; do - ext_name=$(basename "$d") - echo "Discovered an extension to add: '${ext_name}'" - extra_extensions="$extra_extensions $ext_name" - rm -rf "src/postgres/third-party-extensions/${ext_name}" - cp -v -r "$d" src/postgres/third-party-extensions/ - done -fi -if [ -z "${extra_extensions}" ]; then - echo "There were no extra extensions to compile with..." -else - echo "Appending '${extra_extensions}' to src/postgres/third-party-extensions/Makefile" - sed -i "1{s/$/${extra_extensions}/}" src/postgres/third-party-extensions/Makefile -fi -# patch postgres.h -/usr/local/bin/patch_postgres_h.sh -# first pass compile -./yb_build.sh release --clang12 -# done -echo "Your first pass build of YugabyteDB ${YB_SOURCE_VERSION} is complete" diff --git a/.docker/build-infrastructure-clang/usr/local/bin/yb-rebuild-extensions.sh b/.docker/build-infrastructure-clang/usr/local/bin/yb-rebuild-extensions.sh deleted file mode 100644 index 51c470d..0000000 --- a/.docker/build-infrastructure-clang/usr/local/bin/yb-rebuild-extensions.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -eu -# working directory -cd /yb-source -# reset the Makefile of the third-party extensions -git checkout -- src/postgres/third-party-extensions/Makefile -# checkout the version to work with -git checkout "${YB_SOURCE_VERSION}" -# optionally, install extensions for compilation -extra_extensions="" -count=$(find /extensions/ -maxdepth 1 -type d | grep -v '^/extensions/$' | wc -l) -if [ $count -ne 0 ]; then - for d in /extensions/*/ ; do - ext_name=$(basename "$d") - echo "Discovered an extension to add: '${ext_name}'" - extra_extensions="$extra_extensions $ext_name" - rm -rf "src/postgres/third-party-extensions/${ext_name}" - cp -v -r "$d" src/postgres/third-party-extensions/ - done -fi -if [ -z "${extra_extensions}" ]; then - echo "There were no extra extensions to compile with..." -else - echo "Appending '${extra_extensions}' to src/postgres/third-party-extensions/Makefile" - sed -i "1{s/$/${extra_extensions}/}" src/postgres/third-party-extensions/Makefile -fi -# patch postgres.h -/usr/local/bin/patch_postgres_h.sh -# rebuild extensions only: -./yb_build.sh release --clang12 --clean-thirdparty -# done -echo "Your rebuild of YugabyteDB third-party for ${YB_SOURCE_VERSION} is complete" \ No newline at end of file diff --git a/.docker/build-infrastructure-clang/usr/local/bin/yb-rebuild.sh b/.docker/build-infrastructure-clang/usr/local/bin/yb-rebuild.sh deleted file mode 100644 index 4c87a1e..0000000 --- a/.docker/build-infrastructure-clang/usr/local/bin/yb-rebuild.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -eu -# working directory -cd /yb-source -# reset the Makefile of the third-party extensions -git checkout -- src/postgres/third-party-extensions/Makefile -# checkout the version to work with -git checkout "${YB_SOURCE_VERSION}" -# optionally, install extensions for compilation -extra_extensions="" -count=$(find /extensions/ -maxdepth 1 -type d | grep -v '^/extensions/$' | wc -l) -if [ $count -ne 0 ]; then - for d in /extensions/*/ ; do - ext_name=$(basename "$d") - echo "Discovered an extension to add: '${ext_name}'" - extra_extensions="$extra_extensions $ext_name" - rm -rf "src/postgres/third-party-extensions/${ext_name}" - cp -v -r "$d" src/postgres/third-party-extensions/ - done -fi -if [ -z "${extra_extensions}" ]; then - echo "There were no extra extensions to compile with..." -else - echo "Appending '${extra_extensions}' to src/postgres/third-party-extensions/Makefile" - sed -i "1{s/$/${extra_extensions}/}" src/postgres/third-party-extensions/Makefile -fi -# patch postgres.h -/usr/local/bin/patch_postgres_h.sh -# recompile -./yb_build.sh release --clang12 -# done -echo "Your rebuild of YugabyteDB ${YB_SOURCE_VERSION} is complete" \ No newline at end of file diff --git a/.docker/build-infrastructure-clang/usr/local/bin/yb-release.sh b/.docker/build-infrastructure-clang/usr/local/bin/yb-release.sh deleted file mode 100644 index b45a720..0000000 --- a/.docker/build-infrastructure-clang/usr/local/bin/yb-release.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -eu -# working directory -cd /yb-source -# remove old build, if exists... -rm -rf /yb-source/build/yugabyte-*.tar.gz -# create the release archive -yes | ./yb_release -# move to a well known location -mv /yb-source/build/yugabyte-*.tar.gz /yb-source/build/yugabyte-${YB_RELEASE_VERSION}.tar.gz -# done -echo "Your build is available in /yb-source/build/yugabyte-${YB_RELEASE_VERSION}.tar.gz" \ No newline at end of file diff --git a/.docker/build-infrastructure-gcc/Dockerfile b/.docker/build-infrastructure-gcc/Dockerfile deleted file mode 100644 index 0d68493..0000000 --- a/.docker/build-infrastructure-gcc/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# -# YugabyteDB build steps from: -# https://docs.yugabyte.com/latest/contribute/core-database/build-from-src-centos/ -# -FROM centos:7.9.2009 - -ARG GCC_VERSION=9.4.0 -ARG GCC_MAKE_PARALLELISM=32 -ARG MVN_VERSION=3.8.4 - -COPY usr/local/bin/patch_postgres_h.sh /usr/local/bin/patch_postgres_h.sh -COPY usr/local/bin/yb-first-pass-build.sh /usr/local/bin/yb-first-pass-build.sh -COPY usr/local/bin/yb-rebuild-extensions.sh /usr/local/bin/yb-rebuild-extensions.sh -COPY usr/local/bin/yb-rebuild.sh /usr/local/bin/yb-rebuild.sh -COPY usr/local/bin/yb-release.sh /usr/local/bin/yb-release.sh -COPY usr/local/bin/yb-tests.sh /usr/local/bin/yb-tests.sh - -RUN chmod +x /usr/local/bin/yb-first-pass-build.sh \ - && chmod +x /usr/local/bin/yb-rebuild-extensions.sh \ - && chmod +x /usr/local/bin/yb-rebuild.sh \ - && chmod +x /usr/local/bin/yb-release.sh \ - && chmod +x /usr/local/bin/yb-tests.sh \ - && chmod +x /usr/local/bin/patch_postgres_h.sh \ - && echo 'YugabyteDB build tooling' \ - && yum -y update \ - && yum -y groupinstall 'Development Tools' \ - && yum install -y ruby perl-Digest epel-release ccache git python2-pip python-devel python3 python3-pip python3-devel which \ - && yum install -y cmake3 ctest3 ninja-build \ - && yum install -y git wget bzip2 \ - && yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel \ - && echo 'Download Maven newer than default 3.0.5:' \ - && wget https://dlcdn.apache.org/maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz -P /tmp \ - && tar xf /tmp/apache-maven-${MVN_VERSION}-bin.tar.gz -C /opt \ - && ln -s /opt/apache-maven-${MVN_VERSION}/bin/mvn /usr/bin/mvn \ - && ln -s /usr/bin/cmake3 /usr/local/bin/cmake \ - && ln -s /usr/bin/ctest3 /usr/local/bin/ctest \ - && echo 'Updated GCC for building extensions later on' \ - && cd /tmp \ - && wget http://ftp.mirrorservice.org/sites/sourceware.org/pub/gcc/releases/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.gz \ - && tar zxf gcc-${GCC_VERSION}.tar.gz \ - && cd gcc-${GCC_VERSION} \ - && ./contrib/download_prerequisites \ - && ./configure --disable-multilib --enable-languages=c,c++ \ - && make -j ${GCC_MAKE_PARALLELISM} \ - && make install \ - && echo 'Cleanup' \ - && yum clean all -y \ - && rm -rf /tmp/gcc-${GCC_VERSION} - -VOLUME /root/.m2 -VOLUME /opt/yb-build -VOLUME /yb-source \ No newline at end of file diff --git a/.docker/build-infrastructure-gcc/usr/local/bin/patch_postgres_h.sh b/.docker/build-infrastructure-gcc/usr/local/bin/patch_postgres_h.sh deleted file mode 100644 index 2efa69e..0000000 --- a/.docker/build-infrastructure-gcc/usr/local/bin/patch_postgres_h.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -eu -# working directory -pushd /yb-source -# reset the postgres.h file -git checkout -- src/postgres/src/include/postgres.h -# make a copy so patch status can be verified: -cp src/postgres/src/include/postgres.h src/postgres/src/include/postgres.h.bak - -# Patch postgres.h, define YUGABYTEDB so extensions can distinguish if they need to -# compile for YugabyteDB or regular Postgres. -echo "#ifndef YUGABYTEDB -#define YUGABYTEDB 1 -#endif" >> src/postgres/src/include/postgres.h - -# done -echo "Patch status: postgres.h. Changes:" -set +e -diff src/postgres/src/include/postgres.h src/postgres/src/include/postgres.h.bak -if (( $? == 0 || $? > 1)); then - echo "postgres.h: patch failed!" - popd - exit 1 -else - echo "postgres.h: patch succeeded" - popd -fi diff --git a/.docker/build-infrastructure-gcc/usr/local/bin/yb-first-pass-build.sh b/.docker/build-infrastructure-gcc/usr/local/bin/yb-first-pass-build.sh deleted file mode 100644 index c4de0a3..0000000 --- a/.docker/build-infrastructure-gcc/usr/local/bin/yb-first-pass-build.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -eu -# working directory -cd /yb-source -# ensure the source code -if [ ! -d "./.git" ]; then - echo "Checking out '${YB_REPOSITORY}'..." - git clone "${YB_REPOSITORY}" . -else - echo "'${YB_REPOSITORY}' already checked out..." -fi -# checkout the version to work with -git checkout "${YB_SOURCE_VERSION}" -# optionally, install extensions for compilation -extra_extensions="" -count=$(find /extensions/ -maxdepth 1 -type d | grep -v '^/extensions/$' | wc -l) -if [ $count -ne 0 ]; then - for d in /extensions/*/ ; do - ext_name=$(basename "$d") - echo "Discovered an extension to add: '${ext_name}'" - extra_extensions="$extra_extensions $ext_name" - rm -rf "src/postgres/third-party-extensions/${ext_name}" - cp -v -r "$d" src/postgres/third-party-extensions/ - done -fi -if [ -z "${extra_extensions}" ]; then - echo "There were no extra extensions to compile with..." -else - echo "Appending '${extra_extensions}' to src/postgres/third-party-extensions/Makefile" - sed -i "1{s/$/${extra_extensions}/}" src/postgres/third-party-extensions/Makefile -fi -# patch postgres.h -/usr/local/bin/patch_postgres_h.sh -# first pass compile -./yb_build.sh release --gcc -# done -echo "Your first pass build of YugabyteDB ${YB_SOURCE_VERSION} is complete" diff --git a/.docker/build-infrastructure-gcc/usr/local/bin/yb-rebuild-extensions.sh b/.docker/build-infrastructure-gcc/usr/local/bin/yb-rebuild-extensions.sh deleted file mode 100644 index b482324..0000000 --- a/.docker/build-infrastructure-gcc/usr/local/bin/yb-rebuild-extensions.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -eu -# working directory -cd /yb-source -# reset the Makefile of the third-party extensions -git checkout -- src/postgres/third-party-extensions/Makefile -# checkout the version to work with -git checkout "${YB_SOURCE_VERSION}" -# optionally, install extensions for compilation -extra_extensions="" -count=$(find /extensions/ -maxdepth 1 -type d | grep -v '^/extensions/$' | wc -l) -if [ $count -ne 0 ]; then - for d in /extensions/*/ ; do - ext_name=$(basename "$d") - echo "Discovered an extension to add: '${ext_name}'" - extra_extensions="$extra_extensions $ext_name" - rm -rf "src/postgres/third-party-extensions/${ext_name}" - cp -v -r "$d" src/postgres/third-party-extensions/ - done -fi -if [ -z "${extra_extensions}" ]; then - echo "There were no extra extensions to compile with..." -else - echo "Appending '${extra_extensions}' to src/postgres/third-party-extensions/Makefile" - sed -i "1{s/$/${extra_extensions}/}" src/postgres/third-party-extensions/Makefile -fi -# patch postgres.h -/usr/local/bin/patch_postgres_h.sh -# rebuild extensions only: -./yb_build.sh release --gcc --clean-thirdparty -# done -echo "Your rebuild of YugabyteDB third-party for ${YB_SOURCE_VERSION} is complete" \ No newline at end of file diff --git a/.docker/build-infrastructure-gcc/usr/local/bin/yb-release.sh b/.docker/build-infrastructure-gcc/usr/local/bin/yb-release.sh deleted file mode 100644 index b45a720..0000000 --- a/.docker/build-infrastructure-gcc/usr/local/bin/yb-release.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -eu -# working directory -cd /yb-source -# remove old build, if exists... -rm -rf /yb-source/build/yugabyte-*.tar.gz -# create the release archive -yes | ./yb_release -# move to a well known location -mv /yb-source/build/yugabyte-*.tar.gz /yb-source/build/yugabyte-${YB_RELEASE_VERSION}.tar.gz -# done -echo "Your build is available in /yb-source/build/yugabyte-${YB_RELEASE_VERSION}.tar.gz" \ No newline at end of file diff --git a/.docker/build-infrastructure-gcc/usr/local/bin/yb-tests.sh b/.docker/build-infrastructure-gcc/usr/local/bin/yb-tests.sh deleted file mode 100644 index 84f0874..0000000 --- a/.docker/build-infrastructure-gcc/usr/local/bin/yb-tests.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -usage() { - echo `basename $0`: ERROR: $* 1>&2 - echo usage: `basename $0` '[ cpp | cxx | java ]' 1>&2 - exit 1 -} - -usage-cxx() { - echo `basename $0`: ERROR: $* 1>&2 - echo usage: `basename $0` 'cxx test-name [sub-test]' 1>&2 - exit 1 -} - -usage-java() { - echo `basename $0`: ERROR: $* 1>&2 - echo usage: `basename $0` 'java test.Class[#selectedTest]' 1>&2 - exit 1 -} - -if [ -z "$(find $HOME/.m2/repository -name surefire-junit4-2.22.0.jar)" ]; then - echo "Downloading a required Java dependency (https://github.com/yugabyte/yugabyte-db/issues/9416)" - mvn dependency:get -Dartifact=org.apache.maven.surefire:surefire-junit4:2.22.0 -else - echo "Java dependencies OK" -fi -cd /yb-source -./yb_build.sh debug --gcc --sj # don't build java here, we might not need it - -case "${1}" in - cpp) - ./yb_build.sh debug --ctest --gcc - ;; - cxx) - if [ "$#" -eq 1 ]; then - usage-cxx - else - if [ "$#" -eq 3 ]; then - ./yb_build.sh debug --gcc --cxx-test "${2}" --gtest_filter "${3}" - else - ./yb_build.sh debug --gcc --cxx-test "${2}" - fi - fi - ;; - java) - if [ "$#" -eq 1 ]; then - shift - ./yb_build.sh debug "$@" --gcc --java-tests - else - testpath="${2}" - shift - shift - ./yb_build.sh debug "$@" --gcc --java-test "${testpath}" - fi - ;; - raw) - shift - ./yb_build.sh debug "$@" --gcc - ;; - *) - usage - ;; -esac \ No newline at end of file diff --git a/.docker/build-infrastructure/Dockerfile b/.docker/build-infrastructure/Dockerfile new file mode 100644 index 0000000..7218508 --- /dev/null +++ b/.docker/build-infrastructure/Dockerfile @@ -0,0 +1,78 @@ +# +# YugabyteDB build steps from: +# https://docs.yugabyte.com/preview/contribute/core-database/build-from-src-almalinux/ +# + +ARG BUILD_PLATFORM= +ARG ALMALINUX_VERSION= + +FROM --platform=${BUILD_PLATFORM} almalinux:${ALMALINUX_VERSION} + +# Maven 3.9.x apparently has breaking changes, +# end effect is, with 3.9.x, java build doesn't work - build hangs at random places. + +ARG GCC_VERSION=11 +ARG PYTHON_VERSION=39 + +COPY usr/local/bin/apply-extensions.sh /usr/local/bin/apply-extensions.sh +COPY usr/local/bin/apply-patches.sh /usr/local/bin/apply-patches.sh +COPY usr/local/bin/patch_postgres_h.sh /usr/local/bin/patch_postgres_h.sh +COPY usr/local/bin/yb-first-pass-build.sh /usr/local/bin/yb-first-pass-build.sh +COPY usr/local/bin/yb-rebuild-extensions.sh /usr/local/bin/yb-rebuild-extensions.sh +COPY usr/local/bin/yb-rebuild.sh /usr/local/bin/yb-rebuild.sh +COPY usr/local/bin/yb-release.sh /usr/local/bin/yb-release.sh +COPY usr/local/bin/yb-tests.sh /usr/local/bin/yb-tests.sh + +RUN chmod +x /usr/local/bin/apply-extensions.sh \ + && chmod +x /usr/local/bin/apply-patches.sh \ + && chmod +x /usr/local/bin/patch_postgres_h.sh \ + && chmod +x /usr/local/bin/yb-first-pass-build.sh \ + && chmod +x /usr/local/bin/yb-rebuild-extensions.sh \ + && chmod +x /usr/local/bin/yb-rebuild.sh \ + && chmod +x /usr/local/bin/yb-release.sh \ + && chmod +x /usr/local/bin/yb-tests.sh \ + && echo 'YugabyteDB build tooling' \ + && dnf -y update \ + && dnf -y groupinstall 'Development Tools' \ + && dnf install -y epel-release libatomic rsync \ + && dnf install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-devel \ + && dnf install -y cmake3 \ + && echo 'Install and configure Java 11' \ + && echo 'Remove any other Java version installed by Development Tools group' \ + && dnf list installed "java-*" | grep java- | awk '{print $1}' | sed 's/.'$(uname -m)'//' | xargs -0 dnf remove -y \ + && dnf install -y java-11-openjdk java-11-openjdk-devel \ + && echo 'Additional programs' \ + && dnf install -y bzip2 git jq wget which \ + && echo 'Latest Ninja' \ + && export latest_ninja_download=$(curl -Ls "https://api.github.com/repos/ninja-build/ninja/releases/latest" | jq '.assets[] | select(.name == "ninja-linux.zip") | .browser_download_url' -r) \ + && curl -Ls "$latest_ninja_download" | zcat | tee /usr/local/bin/ninja >/dev/null \ + && chmod +x /usr/local/bin/ninja \ + && echo 'Download Maven newer than default 3.0.5:' \ + && export latest_38_maven=$(curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/apache/maven/tags | jq '[ .[] | select(.name | startswith("maven-3.8")) ] | .[0].name' -r | sed 's/maven-//') \ + && wget https://dlcdn.apache.org/maven/maven-3/${latest_38_maven}/binaries/apache-maven-${latest_38_maven}-bin.tar.gz -P /tmp \ + && tar xf /tmp/apache-maven-${latest_38_maven}-bin.tar.gz -C /opt \ + && ln -s /opt/apache-maven-${latest_38_maven}/bin/mvn /usr/bin/mvn \ + && ln -s /usr/bin/cmake3 /usr/local/bin/cmake \ + && ln -s /usr/bin/ctest3 /usr/local/bin/ctest \ + && echo "Compiler: GCC and libatomic toolset" \ + && dnf install -y gcc-toolset-${GCC_VERSION} gcc-toolset-${GCC_VERSION}-libatomic-devel \ + && echo "Compiler: clang" \ + && dnf install -y clang clang-tools-extra \ + && echo "Enable locale required for tests" \ + && dnf install -y langpacks-en glibc-all-langpacks glibc-langpack-en glibc-locale-source \ + && localedef --no-archive -i en_US -f UTF-8 en_US.UTF-8 \ + && echo "Yugabyted-ui: release support" \ + && dnf install -y npm golang \ + && echo "Yugabyted-ui: install patchelf: https://github.com/yugabyte/yugabyte-db/issues/18258" \ + && mkdir -p /tmp/patchelf && cd /tmp/patchelf && git clone https://github.com/NixOS/patchelf.git . \ + && ./bootstrap.sh && ./configure && make && make install \ + && cd - && rm -rf /tmp/patchelf \ + && echo 'Cleanup' \ + && dnf clean all -y + +VOLUME /extensions +VOLUME /patches +VOLUME /root/.m2 +VOLUME /opt/yb-build +VOLUME /yb-source +VOLUME /yb-build-cache diff --git a/.docker/build-infrastructure/patches/python/yugabyte/library_packager.py.diff b/.docker/build-infrastructure/patches/python/yugabyte/library_packager.py.diff new file mode 100644 index 0000000..e8ef0f6 --- /dev/null +++ b/.docker/build-infrastructure/patches/python/yugabyte/library_packager.py.diff @@ -0,0 +1,51 @@ +diff --git a/python/yugabyte/library_packager.py b/python/yugabyte/library_packager.py +index 0bf9b386b7..377402bab1 100644 +--- a/python/yugabyte/library_packager.py ++++ b/python/yugabyte/library_packager.py +@@ -281,6 +281,7 @@ class LibraryPackager: + + if using_linuxbrew(): + # In Linuxbrew mode, we will set the rp ++ subprocess.check_call(['chmod', 'u+w', file_path]) + remove_rpath(file_path) + return + +@@ -288,6 +289,7 @@ class LibraryPackager: + new_rpath = ':'.join(LibraryPackager.get_relative_rpath_items( + dest_root_dir, os.path.dirname(file_abs_path))) + logging.info("Setting rpath on file %s to %s", file_path, new_rpath) ++ subprocess.check_call(['chmod', 'u+w', file_path]) + set_rpath(file_path, new_rpath) + + def install_dyn_linked_binary(self, src_path: str, dest_dir: str) -> str: +@@ -647,6 +649,20 @@ class LibraryPackager: + with open(post_install_path, 'w') as post_install_script_output: + post_install_script_output.write(new_post_install_script) + ++ @staticmethod ++ def is_elf(file_path: str) -> bool: ++ # This is an atomic option but on slow filesystms we do get 0777 and then permissions change. ++ # Check the file header and only progress if it's an ELF file... ++ n_bytes = 4 ++ fstat = os.stat(file_path) ++ if fstat.st_size < n_bytes: # Cannot be an ELF file because we need at least 4 bytes ++ return False ++ else: ++ fc = open(file_path, "rb") ++ header = fc.read(n_bytes) ++ fc.close() ++ return header == b'\x7fELF' ++ + def postprocess_distribution(self, build_target: str) -> None: + """ + build_target is different from self.dest_dir because this function is invoked after +@@ -663,6 +679,9 @@ class LibraryPackager: + if not should_manipulate_rpath_of(file_path): + continue + ++ if not LibraryPackager.is_elf(file_path): ++ continue ++ + if is_linuxbrew: + # For a Linuxbrew-based package, we will set rpath in post_install.sh, so + # we should remove it here. diff --git a/.docker/build-infrastructure/patches/yugabyted-ui/build.sh.diff b/.docker/build-infrastructure/patches/yugabyted-ui/build.sh.diff new file mode 100644 index 0000000..0c98fe9 --- /dev/null +++ b/.docker/build-infrastructure/patches/yugabyted-ui/build.sh.diff @@ -0,0 +1,16 @@ +diff --git a/yugabyted-ui/build.sh b/yugabyted-ui/build.sh +index 917e7286db..b3554a8c7f 100755 +--- a/yugabyted-ui/build.sh ++++ b/yugabyted-ui/build.sh +@@ -52,8 +52,9 @@ if is_mac; then + exit + fi + +-echo "Running ldd on ${OUT_FILE}" +-ldd "${OUT_FILE}" ++# echo "Running ldd on ${OUT_FILE}" ++# ldd "${OUT_FILE}" ++log "Skipping ldd on generated library because it doesn't work on M1/M2 mac and the command is irrelevant anyway" + + ld_interpreter=$( patchelf --print-interpreter "${OUT_FILE}" ) + if [[ ${ld_interpreter} == /lib*/ld-linux-* ]]; then diff --git a/.docker/build-infrastructure-gcc/usr/local/bin/yb-rebuild.sh b/.docker/build-infrastructure/usr/local/bin/apply-extensions.sh similarity index 74% rename from .docker/build-infrastructure-gcc/usr/local/bin/yb-rebuild.sh rename to .docker/build-infrastructure/usr/local/bin/apply-extensions.sh index e6000c8..1b5b6e1 100644 --- a/.docker/build-infrastructure-gcc/usr/local/bin/yb-rebuild.sh +++ b/.docker/build-infrastructure/usr/local/bin/apply-extensions.sh @@ -1,11 +1,10 @@ #!/usr/bin/env bash set -eu -# working directory + cd /yb-source -# reset the Makefile of the third-party extensions + git checkout -- src/postgres/third-party-extensions/Makefile -# checkout the version to work with -git checkout "${YB_SOURCE_VERSION}" + # optionally, install extensions for compilation extra_extensions="" count=$(find /extensions/ -maxdepth 1 -type d | grep -v '^/extensions/$' | wc -l) @@ -24,9 +23,4 @@ else echo "Appending '${extra_extensions}' to src/postgres/third-party-extensions/Makefile" sed -i "1{s/$/${extra_extensions}/}" src/postgres/third-party-extensions/Makefile fi -# patch postgres.h -/usr/local/bin/patch_postgres_h.sh -# recompile -./yb_build.sh release --gcc -# done -echo "Your rebuild of YugabyteDB ${YB_SOURCE_VERSION} is complete" \ No newline at end of file +# patch postgres.h \ No newline at end of file diff --git a/.docker/build-infrastructure/usr/local/bin/apply-patches.sh b/.docker/build-infrastructure/usr/local/bin/apply-patches.sh new file mode 100644 index 0000000..614b82b --- /dev/null +++ b/.docker/build-infrastructure/usr/local/bin/apply-patches.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -eu +# working directory +cd /yb-source + +patches_base=/patches/ +for patch_file in $(find ${patches_base} -type f); do + if [[ $patch_file == *.diff ]]; then + relative=$(echo $patch_file | sed 's!'${patches_base}'!!') + [[ $relative == /* ]] && relative=${relative:1} + relative=${relative/%?????/} + git checkout -- "${relative}" + git apply "${patch_file}" + if [ $? -gt 0 ]; then + echo "ERROR: failed applying patch: ${patch_file} to ${relative}" + exit 1 + else + echo "PATCH APPLIED: patch: ${patch_file} applied to ${relative}" + fi + fi +done + +# patch postgres.h +/usr/local/bin/patch_postgres_h.sh diff --git a/.docker/build-infrastructure-clang/usr/local/bin/patch_postgres_h.sh b/.docker/build-infrastructure/usr/local/bin/patch_postgres_h.sh similarity index 100% rename from .docker/build-infrastructure-clang/usr/local/bin/patch_postgres_h.sh rename to .docker/build-infrastructure/usr/local/bin/patch_postgres_h.sh diff --git a/.docker/build-infrastructure/usr/local/bin/yb-first-pass-build.sh b/.docker/build-infrastructure/usr/local/bin/yb-first-pass-build.sh new file mode 100644 index 0000000..fcda51a --- /dev/null +++ b/.docker/build-infrastructure/usr/local/bin/yb-first-pass-build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -eu +# working directory +cd /yb-source +# ensure the source code +if [ ! -d "./.git" ]; then + echo "Checking out '${YB_REPOSITORY}'..." + git clone "${YB_REPOSITORY}" . +else + echo "'${YB_REPOSITORY}' already checked out..." +fi + +/usr/local/bin/apply-patches.sh +/usr/local/bin/apply-extensions.sh + +# checkout the version to work with +git checkout "${YB_SOURCE_VERSION}" + +# first pass compile +[ -n "${YB_CONFIGURED_COMPILER_TYPE}" ] && export YB_COMPILER_TYPE=${YB_CONFIGURED_COMPILER_TYPE} +[ -n "${YB_CONFIGURED_COMPILER_ARCH}" ] && export YB_TARGET_ARCH=${YB_CONFIGURED_COMPILER_ARCH} +./yb_build.sh release +# done +echo "Your first pass build of YugabyteDB ${YB_SOURCE_VERSION} is complete" diff --git a/.docker/build-infrastructure/usr/local/bin/yb-rebuild-extensions.sh b/.docker/build-infrastructure/usr/local/bin/yb-rebuild-extensions.sh new file mode 100644 index 0000000..9a39a77 --- /dev/null +++ b/.docker/build-infrastructure/usr/local/bin/yb-rebuild-extensions.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -eu +# working directory +cd /yb-source + +git reset --hard +git checkout "${YB_SOURCE_VERSION}" + +/usr/local/bin/apply-patches.sh +/usr/local/bin/apply-extensions.sh + +# rebuild extensions only: +[ -n "${YB_CONFIGURED_COMPILER_TYPE}" ] && export YB_COMPILER_TYPE=${YB_CONFIGURED_COMPILER_TYPE} +[ -n "${YB_CONFIGURED_COMPILER_ARCH}" ] && export YB_TARGET_ARCH=${YB_CONFIGURED_COMPILER_ARCH} + +export CLEAN_THIRD_PARTY_FLAG= +if [ -f /yb-source/build/latest_build_root ]; then + echo "Previous build discovered, discovering third-party directory..." + latest_build_root=$(cat /yb-source/build/latest_build_root) + third_party_dir_file="${latest_build_root}/thirdparty_path.txt" + if [ ! -f "${third_party_dir_file}" ]; then + echo "Warning: no ${third_party_dir_file}, cannot discover third-party directory" + else + latest_thirdparty_dir=$(cat "${third_party_dir_file}") + if [ ! -d "${latest_thirdparty_dir}" ]; then + echo "Warning: no ${latest_thirdparty_dir} directory" + else + export YB_THIRDPARTY_DIR="${latest_thirdparty_dir}" + echo "Third-party directory at ${YB_THIRDPARTY_DIR} is okay, going to clean third-party" + export CLEAN_THIRD_PARTY_FLAG="--clean-thirdparty" + fi + fi +fi + +./yb_build.sh release --force ${CLEAN_THIRD_PARTY_FLAG} +# done +echo "Your rebuild of YugabyteDB third-party for ${YB_SOURCE_VERSION} is complete" diff --git a/.docker/build-infrastructure/usr/local/bin/yb-rebuild.sh b/.docker/build-infrastructure/usr/local/bin/yb-rebuild.sh new file mode 100644 index 0000000..b776189 --- /dev/null +++ b/.docker/build-infrastructure/usr/local/bin/yb-rebuild.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -eu + +# cleanup any potential previous runs: +rm -rfv /tmp/yb_test.tmp* + +# working directory +cd /yb-source + +git reset --hard +git checkout "${YB_SOURCE_VERSION}" + +/usr/local/bin/apply-patches.sh +/usr/local/bin/apply-extensions.sh + +# recompile +[ -n "${YB_CONFIGURED_COMPILER_TYPE}" ] && export YB_COMPILER_TYPE=${YB_CONFIGURED_COMPILER_TYPE} +[ -n "${YB_CONFIGURED_COMPILER_ARCH}" ] && export YB_TARGET_ARCH=${YB_CONFIGURED_COMPILER_ARCH} +./yb_build.sh release +# done +echo "Your rebuild of YugabyteDB ${YB_SOURCE_VERSION} is complete" diff --git a/.docker/build-infrastructure/usr/local/bin/yb-release.sh b/.docker/build-infrastructure/usr/local/bin/yb-release.sh new file mode 100644 index 0000000..4979cba --- /dev/null +++ b/.docker/build-infrastructure/usr/local/bin/yb-release.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eu +# working directory +cd /yb-source + +/usr/local/bin/apply-patches.sh + +[ -n "${YB_CONFIGURED_COMPILER_TYPE}" ] && export YB_COMPILER_TYPE=${YB_CONFIGURED_COMPILER_TYPE} +[ -n "${YB_CONFIGURED_COMPILER_ARCH}" ] && export YB_TARGET_ARCH=${YB_CONFIGURED_COMPILER_ARCH} + +# remove old build, if exists... +rm -rf /yb-source/build/yugabyte-${YB_RELEASE_VERSION}.tar.gz + +# create the release archive +./yb_release --build_archive --build=release --force --keep_tmp_dir \ + --save_release_path_to_file=/yb-source/build/latest-release + +mv $(cat /yb-source/build/latest-release) /yb-source/build/yugabyte-${YB_RELEASE_VERSION}.tar.gz +# done +echo "Your build is available in /yb-source/build/yugabyte-${YB_RELEASE_VERSION}.tar.gz" diff --git a/.docker/build-infrastructure-clang/usr/local/bin/yb-tests.sh b/.docker/build-infrastructure/usr/local/bin/yb-tests.sh similarity index 66% rename from .docker/build-infrastructure-clang/usr/local/bin/yb-tests.sh rename to .docker/build-infrastructure/usr/local/bin/yb-tests.sh index 516c1ff..80e4074 100644 --- a/.docker/build-infrastructure-clang/usr/local/bin/yb-tests.sh +++ b/.docker/build-infrastructure/usr/local/bin/yb-tests.sh @@ -24,40 +24,43 @@ if [ -z "$(find $HOME/.m2/repository -name surefire-junit4-2.22.0.jar)" ]; then else echo "Java dependencies OK" fi + cd /yb-source -./yb_build.sh debug --clang12 --sj # don't build java here, we might not need it +[ -n "${YB_CONFIGURED_COMPILER_TYPE}" ] && export YB_COMPILER_TYPE=${YB_CONFIGURED_COMPILER_TYPE} +[ -n "${YB_CONFIGURED_COMPILER_ARCH}" ] && export YB_TARGET_ARCH=${YB_CONFIGURED_COMPILER_ARCH} +./yb_build.sh debug --sj # don't build java here, we might not need it case "${1}" in cpp) - ./yb_build.sh debug --ctest --clang12 + ./yb_build.sh debug --ctest ;; cxx) if [ "$#" -eq 1 ]; then usage-cxx else if [ "$#" -eq 3 ]; then - ./yb_build.sh debug --clang12 --cxx-test "${2}" --gtest_filter "${3}" + ./yb_build.sh debug --cxx-test "${2}" --gtest_filter "${3}" else - ./yb_build.sh debug --clang12 --cxx-test "${2}" + ./yb_build.sh debug --cxx-test "${2}" fi fi ;; java) if [ "$#" -eq 1 ]; then shift - ./yb_build.sh debug "$@" --clang12 --java-tests + ./yb_build.sh debug "$@" --java-tests else testpath="${2}" shift shift - ./yb_build.sh debug "$@" --clang12 --java-test "${testpath}" + ./yb_build.sh debug "$@" --java-test "${testpath}" fi ;; raw) shift - ./yb_build.sh debug "$@" --clang12 + ./yb_build.sh debug "$@" ;; *) usage ;; -esac \ No newline at end of file +esac diff --git a/.docker/yugabyte-db/Dockerfile b/.docker/yugabyte-db/Dockerfile index 8d9370d..7a85230 100644 --- a/.docker/yugabyte-db/Dockerfile +++ b/.docker/yugabyte-db/Dockerfile @@ -1,28 +1,35 @@ -FROM centos:7.9.2009 as builder +ARG BUILD_PLATFORM= +ARG ALMALINUX_VERSION= +ARG ALPINE_VERSION= + +# Use AlmaLinux so that we can run post_install.sh, +# patchelf will be executed in the target location. +FROM --platform=${BUILD_PLATFORM} almalinux:${ALMALINUX_VERSION} as builder ARG GID=1060 ARG GROUPNAME=yugabyte ARG UID=1060 ARG USERNAME=yugabyte - -ARG YB_REPOSITORY=https://github.com/yugabyte/yugabyte-db.git -ARG YB_SOURCE_VERSION=v2.11.2 -ARG YB_RELEASE_VERSION=2.11.2.0 - +ARG YB_RELEASE_VERSION=2.19.0.0 COPY yugabyte-${YB_RELEASE_VERSION}.tar.gz /yugabyte-${YB_RELEASE_VERSION}.tar.gz - -LABEL YB_REPOSITORY=https://github.com/yugabyte/yugabyte-db.git \ - YB_SOURCE_VERSION=${YB_SOURCE_VERSION} \ - YB_RELEASE_VERSION=${YB_RELEASE_VERSION} - RUN mkdir -p /home/${USERNAME} \ && tar xvfz /yugabyte-${YB_RELEASE_VERSION}.tar.gz -C /home/${USERNAME} --strip 1 \ && /home/${USERNAME}/bin/post_install.sh -FROM alpine:3.15 +# Buld the final image. Use low footprint base. +FROM --platform=${BUILD_PLATFORM} alpine:${ALPINE_VERSION} ARG GID=1060 ARG GROUPNAME=yugabyte ARG UID=1060 ARG USERNAME=yugabyte +ARG YB_REPOSITORY=https://github.com/yugabyte/yugabyte-db.git +ARG YB_SOURCE_VERSION=v2.19.0.0 +ARG YB_RELEASE_VERSION=2.19.0.0 +ARG RELEASE_SPEC= + +LABEL YB_REPOSITORY=${YB_REPOSITORY} \ + YB_SOURCE_VERSION=${YB_SOURCE_VERSION} \ + YB_RELEASE_VERSION=${YB_RELEASE_VERSION} \ + RELEASE_SPEC=${RELEASE_SPEC} RUN apk add --no-cache bash gawk sed grep bc coreutils openssl ca-certificates \ && addgroup -g ${GID} ${GROUPNAME} \ @@ -42,7 +49,6 @@ RUN echo "export PS1='[\u@\h \W]\$ '" >> /home/${USERNAME}/.bashrc \ && ln -s /home/${USERNAME}/bin/ysqlsh /usr/local/bin/ysqlsh \ && ln -s /home/${USERNAME}/bin/yugabyted /usr/local/bin/yugabyted \ && echo 'Setting up dependencies required for yugabyted' \ - && apk --no-cache add python3 procps \ - && ln -s /usr/bin/python3 /usr/bin/python + && apk --no-cache add python3 procps USER ${USERNAME} diff --git a/Makefile b/Makefile index 3f6f639..db5454b 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,30 @@ -YB_BUILD_INFRASTRUCTURE_DOCKER_TAG?=local/yugabyte-db-build-infrastructure +YB_BUILD_INFRASTRUCTURE_DOCKER_TAG?=docker.io/radekg/yugabyte-db-build-infrastructure +YB_BUILD_INFRASTRUCTURE_DOCKER_VER=$(shell git rev-parse --short HEAD) -# clang build infrastructure -YB_BUILD_INFRASTRUCTURE_CLANG_TAG?=clang -YB_BUILD_INFRASTRUCTURE_CLANG_VERSION?=12 - -# GCC build infrastructure -YB_BUILD_INFRASTRUCTURE_GCC_TAG?=gcc -YB_BUILD_INFRASTRUCTURE_GCC_VERSION?=9.4.0 -YB_BUILD_INFRASTRUCTURE_GCC_MAKE_PARALLELISM?=32 +# GCC version, GCC needs to be installed explicitly +GCC_VERSION=11 +empty= -# By default, use clang-12 build -USE_BUILD_INFRASTRUCTURE?=clang12 +# Set known defaults +USE_COMPILER_TYPE=${empty} +USE_COMPILER_ARCH?=x86_64 +USE_DOCKER_VER=${YB_BUILD_INFRASTRUCTURE_DOCKER_VER} YB_REPOSITORY?=https://github.com/yugabyte/yugabyte-db.git -YB_SOURCE_VERSION?=v2.11.2 +YB_SOURCE_VERSION?=v2.19.0.0 +YB_RELEASE_VERSION?=2.19.0.0 -YB_RELEASE_VERSION?=2.11.2.0 +ifeq (${USE_COMPILER_ARCH},${empty}) +TEMP_ARCH=default +else +TEMP_ARCH=${USE_COMPILER_ARCH} +endif + +ifeq ($(USE_COMPILER_TYPE),${empty}) +TEMP_PREFIX=clang-default-${TEMP_ARCH}-${YB_SOURCE_VERSION} +else +TEMP_PREFIX=${USE_COMPILER_TYPE}-${TEMP_ARCH}-${YB_SOURCE_VERSION} +endif YB_RELEASE_DOCKER_TAG?=local/yugabyte-db YB_RELEASE_DOCKER_VERSION?=${YB_RELEASE_VERSION} @@ -26,97 +35,214 @@ YB_RELEASE_DOCKER_ARG_USER?=yb YB_POSTGRES_WITH_ICU?=true -empty= YB_COMPOSE_SHARED_PRELOAD_LIBRARIES?=${empty} CURRENT_DIR=$(dir $(realpath $(firstword $(MAKEFILE_LIST)))) PLATFORM=$(shell uname -s) -.PHONY: ybdb-build-infrastructure-clang -ybdb-build-infrastructure-clang: - cd ${CURRENT_DIR}/.docker/build-infrastructure-clang \ - && docker build --no-cache \ - -t ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${YB_BUILD_INFRASTRUCTURE_CLANG_TAG}${YB_BUILD_INFRASTRUCTURE_CLANG_VERSION} . \ - && docker tag ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${YB_BUILD_INFRASTRUCTURE_CLANG_TAG}${YB_BUILD_INFRASTRUCTURE_CLANG_VERSION} \ - ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${YB_BUILD_INFRASTRUCTURE_CLANG_TAG}-latest - -.PHONY: ybdb-build-infrastructure-gcc -ybdb-build-infrastructure-gcc: - cd ${CURRENT_DIR}/.docker/build-infrastructure-gcc \ - && docker build --no-cache \ - --build-arg GCC_VERSION=${YB_BUILD_INFRASTRUCTURE_GCC_VERSION} \ - --build-arg GCC_MAKE_PARALLELISM=${YB_BUILD_INFRASTRUCTURE_GCC_MAKE_PARALLELISM} \ - -t ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${YB_BUILD_INFRASTRUCTURE_GCC_TAG}${YB_BUILD_INFRASTRUCTURE_GCC_VERSION} . \ - && docker tag ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${YB_BUILD_INFRASTRUCTURE_GCC_TAG}${YB_BUILD_INFRASTRUCTURE_GCC_VERSION} \ - ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${YB_BUILD_INFRASTRUCTURE_GCC_TAG}-latest +.PHONY: ybdb-build-infrastructure +ybdb-build-infrastructure: + cd ${CURRENT_DIR}/.docker/build-infrastructure \ + && docker buildx build --no-cache \ + --build-arg BUILD_PLATFORM=linux/amd64 \ + --build-arg ALMALINUX_VERSION=8.8 \ + --build-arg GCC_VERSION=${GCC_VERSION} \ + --platform linux/amd64 \ + -t ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${YB_BUILD_INFRASTRUCTURE_DOCKER_VER} . \ + && docker tag ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${YB_BUILD_INFRASTRUCTURE_DOCKER_VER} \ + ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:latest + +# First pass build: +# ----------------- + +.PHONY: ybdb-build-first-pass-clang +ybdb-build-first-pass-clang: +# Use the default clang version available in the build infrastructure: + $(MAKE) ybdb-build-first-pass + +.PHONY: ybdb-build-first-pass-gcc +ybdb-build-first-pass-gcc: + $(MAKE) USE_COMPILER_TYPE=gcc${GCC_VERSION} ybdb-build-first-pass .PHONY: ybdb-build-first-pass ybdb-build-first-pass: ifeq ($(PLATFORM),Linux) - sudo rm -rf ${CURRENT_DIR}/.tmp/yb-build \ - && sudo rm -rf ${CURRENT_DIR}/.tmp/yb-maven \ - && sudo rm -rf ${CURRENT_DIR}/.tmp/yb-source + sudo rm -rf ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build \ + && sudo rm -rf ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source \ + && sudo rm -rf ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache else - rm -rf ${CURRENT_DIR}/.tmp/yb-build \ - && sudo rm -rf ${CURRENT_DIR}/.tmp/yb-maven \ - && sudo rm -rf ${CURRENT_DIR}/.tmp/yb-source + rm -rf ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build \ + && rm -rf ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source \ + && rm -rf ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache endif - mkdir -p ${CURRENT_DIR}/.tmp/yb-build \ + mkdir -p ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build \ && mkdir -p ${CURRENT_DIR}/.tmp/yb-maven \ - && mkdir -p ${CURRENT_DIR}/.tmp/yb-source \ - && mkdir -p ${CURRENT_DIR}/.tmp/extensions \ + && mkdir -p ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source \ + && mkdir -p ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/extensions \ + && mkdir -p ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache \ && docker run --rm -ti \ + --platform linux/amd64 \ -e YB_REPOSITORY=${YB_REPOSITORY} \ -e YB_SOURCE_VERSION=${YB_SOURCE_VERSION} \ -e YB_POSTGRES_WITH_ICU=${YB_POSTGRES_WITH_ICU} \ + -e YB_CONFIGURED_COMPILER_TYPE=${USE_COMPILER_TYPE} \ + -e YB_CONFIGURED_COMPILER_ARCH=${USE_COMPILER_ARCH} \ + -e YB_CCACHE_DIR=/yb-build-cache \ + -e LANG=en_US.UTF-8 \ -v ${CURRENT_DIR}/.tmp/yb-maven:/root/.m2 \ - -v ${CURRENT_DIR}/.tmp/yb-build:/opt/yb-build \ - -v ${CURRENT_DIR}/.tmp/yb-source:/yb-source \ - -v ${CURRENT_DIR}/.tmp/extensions:/extensions \ - ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_BUILD_INFRASTRUCTURE} yb-first-pass-build.sh + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build:/opt/yb-build \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source:/yb-source \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache:/yb-build-cache \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/extensions:/extensions \ + ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_DOCKER_VER} yb-first-pass-build.sh + +# Build infrastructure shell: +# --------------------------- + +.PHONY: ybdb-infrastructure-shell-clang +ybdb-infrastructure-shell-clang: + $(MAKE) ybdb-infrastructure-shell + +.PHONY: ybdb-infrastructure-shell-gcc +ybdb-infrastructure-shell-gcc: + $(MAKE) USE_COMPILER_TYPE=gcc${GCC_VERSION} ybdb-infrastructure-shell + +.PHONY: ybdb-infrastructure-shell +ybdb-infrastructure-shell: + docker run --rm -ti \ + --platform linux/amd64 \ + -e YB_REPOSITORY=${YB_REPOSITORY} \ + -e YB_SOURCE_VERSION=${YB_SOURCE_VERSION} \ + -e YB_POSTGRES_WITH_ICU=${YB_POSTGRES_WITH_ICU} \ + -e YB_CONFIGURED_COMPILER_TYPE=${USE_COMPILER_TYPE} \ + -e YB_CONFIGURED_COMPILER_ARCH=${USE_COMPILER_ARCH} \ + -e YB_CCACHE_DIR=/yb-build-cache \ + -e LANG=en_US.UTF-8 \ + -v ${CURRENT_DIR}/.docker/build-infrastructure/patches:/patches \ + -v ${CURRENT_DIR}/.tmp/yb-maven:/root/.m2 \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build:/opt/yb-build \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source:/yb-source \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache:/yb-build-cache \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/extensions:/extensions \ + ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_DOCKER_VER} /bin/bash + +# Rebuild extensions: +# ------------------- + +.PHONY: ybdb-rebuild-extensions-clang +ybdb-rebuild-extensions-clang: +# Use the default clang version available in the build infrastructure: + $(MAKE) ybdb-rebuild-extensions + +.PHONY: ybdb-rebuild-extensions-gcc +ybdb-rebuild-extensions-gcc: + $(MAKE) USE_COMPILER_TYPE=gcc${GCC_VERSION} ybdb-rebuild-extensions .PHONY: ybdb-rebuild-extensions ybdb-rebuild-extensions: docker run --rm -ti \ + --platform linux/amd64 \ -e YB_SOURCE_VERSION=${YB_SOURCE_VERSION} \ -e YB_POSTGRES_WITH_ICU=${YB_POSTGRES_WITH_ICU} \ + -e YB_CONFIGURED_COMPILER_TYPE=${USE_COMPILER_TYPE} \ + -e YB_CONFIGURED_COMPILER_ARCH=${USE_COMPILER_ARCH} \ + -e YB_CCACHE_DIR=/yb-build-cache \ + -e LANG=en_US.UTF-8 \ + -v ${CURRENT_DIR}/.docker/build-infrastructure/patches:/patches \ -v ${CURRENT_DIR}/.tmp/yb-maven:/root/.m2 \ - -v ${CURRENT_DIR}/.tmp/yb-build:/opt/yb-build \ - -v ${CURRENT_DIR}/.tmp/yb-source:/yb-source \ - -v ${CURRENT_DIR}/.tmp/extensions:/extensions \ - ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_BUILD_INFRASTRUCTURE} yb-rebuild-extensions.sh + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build:/opt/yb-build \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source:/yb-source \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache:/yb-build-cache \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/extensions:/extensions \ + ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_DOCKER_VER} yb-rebuild-extensions.sh + +# Rebuild: +# -------- + +.PHONY: ybdb-rebuild-clang +ybdb-rebuild-clang: +# Use the default clang version available in the build infrastructure: + $(MAKE) ybdb-rebuild + +.PHONY: ybdb-rebuild-gcc +ybdb-rebuild-gcc: + $(MAKE) USE_COMPILER_TYPE=gcc${GCC_VERSION} ybdb-rebuild .PHONY: ybdb-rebuild ybdb-rebuild: docker run --rm -ti \ + --platform linux/amd64 \ -e YB_SOURCE_VERSION=${YB_SOURCE_VERSION} \ -e YB_POSTGRES_WITH_ICU=${YB_POSTGRES_WITH_ICU} \ + -e YB_CONFIGURED_COMPILER_TYPE=${USE_COMPILER_TYPE} \ + -e YB_CONFIGURED_COMPILER_ARCH=${USE_COMPILER_ARCH} \ + -e YB_CCACHE_DIR=/yb-build-cache \ + -e LANG=en_US.UTF-8 \ + -v ${CURRENT_DIR}/.docker/build-infrastructure/patches:/patches \ -v ${CURRENT_DIR}/.tmp/yb-maven:/root/.m2 \ - -v ${CURRENT_DIR}/.tmp/yb-build:/opt/yb-build \ - -v ${CURRENT_DIR}/.tmp/yb-source:/yb-source \ - -v ${CURRENT_DIR}/.tmp/extensions:/extensions \ - ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_BUILD_INFRASTRUCTURE} yb-rebuild.sh + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build:/opt/yb-build \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source:/yb-source \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache:/yb-build-cache \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/extensions:/extensions \ + ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_DOCKER_VER} yb-rebuild.sh + +# Build a distribution: +# --------------------- + +.PHONY: ybdb-distribution-clang +ybdb-distribution-clang: +# Use the default clang version available in the build infrastructure: + $(MAKE) ybdb-distribution + +.PHONY: ybdb-distribution-gcc +ybdb-distribution-gcc: + $(MAKE) USE_COMPILER_TYPE=gcc${GCC_VERSION} ybdb-distribution .PHONY: ybdb-distribution ybdb-distribution: docker run --rm -ti \ + --platform linux/amd64 \ -e YB_RELEASE_VERSION=${YB_RELEASE_VERSION} \ + -e YB_CONFIGURED_COMPILER_TYPE=${USE_COMPILER_TYPE} \ + -e YB_CONFIGURED_COMPILER_ARCH=${USE_COMPILER_ARCH} \ + -e YB_CCACHE_DIR=/yb-build-cache \ + -e LANG=en_US.UTF-8 \ + -v ${CURRENT_DIR}/.docker/build-infrastructure/patches:/patches \ -v ${CURRENT_DIR}/.tmp/yb-maven:/root/.m2 \ - -v ${CURRENT_DIR}/.tmp/yb-build:/opt/yb-build \ - -v ${CURRENT_DIR}/.tmp/yb-source:/yb-source \ - -v ${CURRENT_DIR}/.tmp/extensions:/extensions \ - ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_BUILD_INFRASTRUCTURE} yb-release.sh + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build:/opt/yb-build \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source:/yb-source \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache:/yb-build-cache \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/extensions:/extensions \ + ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_DOCKER_VER} yb-release.sh + +# Build Docker image from a distribution: +# --------------------------------------- + +.PHONY: ybdb-build-docker-clang +ybdb-build-docker-clang: +# Use the default clang version available in the build infrastructure: + $(MAKE) ybdb-build-docker + +.PHONY: ybdb-build-docker-gcc +ybdb-build-docker-gcc: + $(MAKE) USE_COMPILER_TYPE=gcc${GCC_VERSION} ybdb-build-docker .PHONY: ybdb-build-docker ybdb-build-docker: ifeq ($(PLATFORM),Linux) - sudo chmod 0644 ${CURRENT_DIR}/.tmp/yb-source/build/yugabyte-*.tar.gz + sudo chmod 0644 ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source/build/yugabyte-*.tar.gz endif - mkdir -p ${CURRENT_DIR}/.tmp/yb-docker-build \ - && cp -v ${CURRENT_DIR}/.tmp/yb-source/build/yugabyte-*.tar.gz ${CURRENT_DIR}/.tmp/yb-docker-build/ \ - && cp -v ${CURRENT_DIR}/.docker/yugabyte-db/Dockerfile ${CURRENT_DIR}/.tmp/yb-docker-build/ \ - && cd ${CURRENT_DIR}/.tmp/yb-docker-build/ \ - && docker build \ + $(eval RELEASE_SPEC := $(shell basename $(shell cat ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source/build/latest-release) | sed 's/.tar.gz//')) + mkdir -p ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-docker-build \ + && cp -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source/build/yugabyte-*.tar.gz ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-docker-build/ \ + && cp -v ${CURRENT_DIR}/.docker/yugabyte-db/Dockerfile ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-docker-build/ \ + && cd ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-docker-build/ \ + && docker buildx build \ + --platform linux/amd64 \ + --build-arg BUILD_PLATFORM=linux/amd64 \ + --build-arg ALMALINUX_VERSION=8.8 \ + --build-arg ALPINE_VERSION=3.18 \ + --build-arg RELEASE_SPEC=${RELEASE_SPEC} \ --build-arg GID=${YB_RELEASE_DOCKER_ARG_GID} \ --build-arg GROUPNAME=${YB_RELEASE_DOCKER_ARG_GROUP} \ --build-arg UID=${YB_RELEASE_DOCKER_ARG_UID} \ @@ -126,12 +252,30 @@ endif --build-arg YB_REPOSITORY=${YB_REPOSITORY} \ -t ${YB_RELEASE_DOCKER_TAG}:${YB_RELEASE_DOCKER_VERSION} . +# Test runner: +# ------------ + +.PHONY: ybdb-tests-clang +ybdb-tests-clang: +# Use the default clang version available in the build infrastructure: + $(MAKE) ybdb-tests + +.PHONY: ybdb-tests-gcc +ybdb-tests-gcc: + $(MAKE) USE_COMPILER_TYPE=gcc${GCC_VERSION} ybdb-tests + .PHONY: ybdb-tests ybdb-tests: docker run --rm -ti \ + --platform linux/amd64 \ --cap-add=SYS_PTRACE \ -p "5433:5433" \ + -e YB_CONFIGURED_COMPILER_TYPE=${USE_COMPILER_TYPE} \ + -e YB_CONFIGURED_COMPILER_ARCH=${USE_COMPILER_ARCH} \ + -e YB_CCACHE_DIR=/yb-build-cache \ + -e LANG=en_US.UTF-8 \ -v ${CURRENT_DIR}/.tmp/yb-maven:/root/.m2 \ - -v ${CURRENT_DIR}/.tmp/yb-build:/opt/yb-build \ - -v ${CURRENT_DIR}/.tmp/yb-source:/yb-source \ - ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_BUILD_INFRASTRUCTURE} /bin/bash -c 'yb-tests.sh; /bin/bash' + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build:/opt/yb-build \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-source:/yb-source \ + -v ${CURRENT_DIR}/.tmp/${TEMP_PREFIX}/yb-build-cache:/yb-build-cache \ + ${YB_BUILD_INFRASTRUCTURE_DOCKER_TAG}:${USE_DOCKER_VER} /bin/bash -c 'yb-tests.sh; /bin/bash' diff --git a/README.md b/README.md index d306e58..ad9de51 100644 --- a/README.md +++ b/README.md @@ -6,61 +6,88 @@ Original build instructions: - [Extensions requiring installation](https://docs.yugabyte.com/latest/api/ysql/extensions/#extensions-requiring-installation) -## Build infrastructure +## Build the infrastructure -There are two build infrastructure versions: - -- the clang-based version is preferred for YugabyteDB `v2.11.2` and higher (used by default) -- the GCC-based version is preferred for versions lower than `v2.11.2` - -### clang version +Build the Docker image with the build infrastructure. The purpose of this image is to provide all required and optional software necessary to build YugabyteDB. This image is mostly based on [instructions from YugabyteDB documentation](https://docs.yugabyte.com/preview/contribute/core-database/build-from-src-almalinux/). This single build infrastructure image supports clang and gcc builds. ```sh -make ybdb-build-infrastructure-clang +make ybdb-build-infrastructure ``` -### GCC version +This target requires Docker with cross-compiling functionality support because it uses `docker buildx build`. -```sh -make ybdb-build-infrastructure-gcc -``` +### What's installed -Configuration: +- Python 3.9 with Python 3.9 _devel_ packages, +- CMake 3, +- OpenJDK 1.8.0 with OpenJDK 1.8.0 _devel_ packages, +- Ninja, +- Ccache, +- Latest Maven 3.8.x; Maven 3.9.x breaks the build, +- Clang with Clang extra tools, version from AlmaLinux package repositories, currently 15.x.x, +- GCC 11, +- a bunch of support tools to make working with the build and development process easier. -- `YB_BUILD_INFRASTRUCTURE_GCC_VERSION`: gcc version to install in the image, default `9.4.0`, used for Docker image tag version -- `YB_BUILD_INFRASTRUCTURE_GCC_PARALLELISM`: the `-j` value for gcc `make`, default `32` +## Working with the build infrastructure -## Build YugabyteDB using the infrastructure +### Clang or GCC -All commands below use the pre-built build infrastructure Docker image. Depending on the YugabyteDB version you are working with, prepare the relevant build infrastructure Docker image first. When using `clang12` infrasrtructure, no additional configuration is needed. +This single build infrastructure image supports both Clang and GCC builds. Following targets are available: -When using the GCC-based version, add `USE_BUILD_INFRASTRUCTURE=gcc9.4.0` to your `make` calls, for example: +- Clang: + - `ybdb-build-first-pass-clang`: first pass build, configure for Clang, + - `ybdb-infrastructure-shell-clang`: starts the build infrastructure in the shell mode, configure for Clang, + - `ybdb-rebuild-extensions-clang`: a shorthand command to trigger extensions build, configure for Clang, + - `ybdb-rebuild-clang`: a shorthand command to trigger full rebuild, configure for Clang, + - `ybdb-tests-clang`: a shorthand command to enter a shell configured for running tests, configure for Clang. + - `ybdb-build-docker-clang`: a shorthand command to build YugabyteDB Docker image from the Clang build artifact. -```sh -make USE_BUILD_INFRASTRUCTURE=gcc9.4.0 \ - YB_SOURCE_VERSION=v2.11.1 \ - ybdb-build-first-pass -``` +- GCC: + - `ybdb-build-first-pass-gcc`: first pass build, configure for GCC, + - `ybdb-infrastructure-shell-gcc`: starts the build infrastructure in the shell mode, configure for GCC, + - `ybdb-rebuild-extensions-gcc`: a shorthand command to trigger extensions build, configure for GCC, + - `ybdb-rebuild-gcc`: a shorthand command to trigger full rebuild, configure for GCC, + - `ybdb-tests-gcc`: a shorthand command to enter a shell configured for running tests, configure for GCC. + - `ybdb-build-docker-gcc`: a shorthand command to build YugabyteDB Docker image from the Clang build artifact. + +### Docker volumes and mount locations + +The build infrastructure uses five volumes. For four of those, the base path is `./.tmp/---` (referred further to as `$temp-root`). + +- `$temp-root/extensions` mounted at `/extensions`, +- `$temp-root/yb-build` mounted at `/opt/yb-build`, +- `$temp-root/yb-build-cache` mounted at `/yb-build-cache`; Ccache configured to use this location as a build cache, +- `$temp-root/yb-source` mounted at `/yb-source`; stores cloned Git code. + +This enables the possibility to work on various configurations with various source code versions without impacting each other. + +The fifth location is: + +- `./tmp/yb-maven` mounted at `/root/.m2`; caches downloaded Maven artifacts; shared across different builds for efficiency. ### First pass build -This target will download the sources from the request repository, check out the requested version and execute a first pass build. +This target will download the sources from the requested YugabyteDB git repository, check out the requested version, and execute a first pass build. ```sh -make ybdb-build-first-pass +make ybdb-build-first-pass-clang # for Clang +make ybdb-build-first-pass-gcc # for GCC ``` There are following configuration options available for this target: - `YB_REPOSITORY`: YugabyteDB source repository to use, default `https://github.com/yugabyte/yugabyte-db.git` -- `YB_SOURCE_VERSION`: YugabyteDB source code version: commit hash, branch name or tag name, default `v2.11.2` +- `YB_SOURCE_VERSION`: YugabyteDB source code version: commit hash, branch name or tag name, default `v2.19.0.0` + +This process will take about 3 to 5 hours even on a reasonably powerful laptop. The more CPUs, the better. A server with 40+ cores and 192 GB RAM can turn this around in about 15 minutes. ### Rebuild Recompile YugabyteDB. ```sh -make ybdb-rebuild +make ybdb-rebuild-clang # for Clang +make ybdb-rebuild-gcc # for GCC ``` ### Rebuild third-party extensions @@ -68,29 +95,32 @@ make ybdb-rebuild Regular rebuild process does not rebuild third-party extensions. To rebuild third-party extensions, run: ```sh -make ybdb-rebuild-extensions +make ybdb-rebuild-extensions-clang # for Clang +make ybdb-rebuild-extensions-gcc # for GCC ``` -This commang implies `clean`, in effect - it's a new build. +This command implies `clean`, in effect - it's a new build. ### Create a release distribution -This target creates a distribution _tar.gz_ archive from the previous first pass or rebuild result. +This target creates a _tar.gz_ distribution archive from the previous first pass or rebuild result. ```sh -make ybdb-distribution +make ybdb-distribution-clang # for Clang build +make ybdb-distribution-gcc # for GCC build ``` There are following configuration options available for this target: -- `YB_RELEASE_VERSION`: YugabyteDB release version - used for the tar.gz file name, default `2.11.2.0` results in the `yugabyte-2.11.2.0.tar.gz` archive name +- `YB_RELEASE_VERSION`: YugabyteDB release version - used for the _tar.gz_ file name, default `2.19.0.0` results in the `yugabyte-2.19.0.0.tar.gz` archive file. ### Create the Docker image This target creates a Docker image using the previously built distribution: ```sh -make ybdb-build-docker +make ybdb-build-docker-clang # for Clang build artifact +make ybdb-build-docker-gcc # for GCC build artifact ``` There are following configuration options available for this target: @@ -104,15 +134,34 @@ There are following configuration options available for this target: - `YB_RELEASE_DOCKER_VERSION`: resulting Docker image version, default the value of `YB_RELEASE_VERSION` - `YB_REPOSITORY`: used as the image label: YugabyteDB source repository to use, default `https://github.com/yugabyte/yugabyte-db.git` -- `YB_SOURCE_VERSION`: used as the image label: YugabyteDB source code version: commit hash, branch name or tag name, default `v2.11.2` +- `YB_SOURCE_VERSION`: used as the image label: YugabyteDB source code version: commit hash, branch name or tag name, default `v2.19.0.0` + +#### Verify the Docker image + +```sh +docker run --rm \ + -p 7100:7000 \ + -ti local/yugabyte-db:${YB_RELEASE_VERSION} \ + /bin/sh -c '/home/yb/bin/yb-master --master_addresses=127.0.0.1:7100 --replication_factor=1 --fs_data_dirs=/mnt/master --logtostderr=true' +``` + +The TServer will not connect to the master in this way. Find the correct IP of the master container and replace the value under `--tserver_master_addresses`. + +```sh +docker run --rm \ + -p 7100:7000 \ + -ti local/yugabyte-db:${YB_RELEASE_VERSION} \ + /bin/sh -c '/home/yb/bin/yb-tserver --tserver_master_addresses=127.0.0.1:7100 --ysql_enable_auth --fs_data_dirs=/mnt/tserver --logtostderr=true' +``` ### Running YugabyteDB tests ```sh -make ybdb-tests +make ybdb-tests-clang +make ybdb-tests-gcc ``` -This will put YugabyteDB sources in the `debug` mode to give more decent tests output. Once you have the shell available, you can run tests like this: +This will run the infrastructure in the shell mode with YugabyteDB reconfigured to the `debug` mode. Once the shell is available, run tests like this: ```sh # all C++ tests, this may take a long time: @@ -123,17 +172,19 @@ yb-tests.sh cxx test [ subtest ] yb-tests.sh java # selected Java test: yb-tests.sh java test.Class[\#testCase] +# for example: +yb-tests.sh java org.yb.pgsql.TestDropTableWithConcurrentTxn # Raw command, passes all options to the test runner, example: yb-tests.sh raw --sj --scb --java-test test.Class ``` -Regarding Java tests, you can pass any valid `yb_build.sh` options to `java` tests at the end of the command, example: +Any valid `yb_build.sh` option can be passed to `java` tests at the end of the command, example: ```sh yb-tests.sh java test.Class --sj --scb ``` -Once finished working with tests, to put the YugabyteDB in release mode, use the `ybdb-rebuild` target with relevant arguments. +Once finished working with tests, put the YugabyteDB back in release mode using the relevant `ybdb-rebuild-[clang|gcc]` target. #### Debugging the debug mode build with gdb @@ -164,9 +215,37 @@ Set some breakpoints, for example: break yb::pggate::PgTableDesc::FindColumn(int) ``` +## Compatibility + +| Tool | Intel mac | M2 mac | Linux amd64 | +| ------------------------- | --------- | ------------ | ----------- | +| Clang: first pass build | ๐Ÿ‘ | ๐Ÿ‘ | ๐Ÿ‘ | +| Clang: distribution | ๐Ÿ‘ `*` | ๐Ÿ‘ `*`, `**` | ๐Ÿ‘ `*` | +| Clang: rebuild | ๐Ÿ‘ | ๐Ÿ‘ | ๐Ÿ‘ | +| Clang: rebuild extensions | ๐Ÿ‘ | ๐Ÿ‘ | ๐Ÿ‘ | +| Clang: Java tests | ๐Ÿ‘ | ๐Ÿ‘Ž | ๐Ÿ‘ | +| Clang: C++ tests | ๐Ÿ‘ `***` | ๐Ÿ‘Ž | ๐Ÿ‘ | +| Clang: Docker image build | ๐Ÿ‘ | ๐Ÿ‘ | ๐Ÿ‘ | +| GCC: first pass build | ๐Ÿ‘ | ๐Ÿ‘ | ๐Ÿ‘ | +| GCC: distribution | ๐Ÿ‘ `*` | ๐Ÿ‘Ž | ๐Ÿ‘ | +| GCC: rebuild | ๐Ÿ‘ | ๐Ÿ‘ | ๐Ÿ‘ | +| GCC: rebuild extensions | ๐Ÿ‘ | ๐Ÿ‘ | ๐Ÿ‘ | +| GCC: Java tests | ๐Ÿ‘ | ๐Ÿ‘Ž | ๐Ÿ‘ | +| GCC: C++ tests | ๐Ÿ‘ `***` | ๐Ÿ‘Ž | ๐Ÿ‘ | +| GCC: Docker image build | ๐Ÿ‘ | ๐Ÿ‘Ž `****` | ๐Ÿ‘ | + +- `*`: requires `python/yugabyte/library_packager.py.diff` +- `**`: requires `yugabyted-ui/build.sh.diff` +- `***`: some individual tests fail but tests execute +- `****`: distribution cannot be built so the image cannot be built + ## Building YugabyteDB with Postgres extensions -The first pass build and rebuild allows compilation of extensions together with YugabyteDB. Such compilations result in distributions with extension libraries already installed in the package and share directories. It is possible to `ybdb-rebuild` with extensions which did not exist in first pass build. To add extensions to the compilation process, for every extension: +The first pass build and rebuild allows extensions compilation together with YugabyteDB. + +Such compilations result in distributions with extension libraries already installed in the package and share directories. It is possible to `ybdb-rebuild-[clang|gcc]` with extensions which did not exist in first pass build. + +To add extensions to the compilation process, for every extension: ```sh mkdir -p .tmp/extensions/extension-name @@ -174,4 +253,387 @@ cd .tmp/extensions/extension-name git clone . ``` -Then simply execute first pass build or rebuild. +Then execute first pass build or rebuild. + +## KNown problems and caveats + +### General build infrastructure + +#### OpenJDK 11 is used to compile Java bits + +Java 8 and Docker do not play nicely. On the M2 mac Java 8 based build hangs at random places. CPU gets pegged at 200% and memory usage goes through the roof. Compiling with Java 11 to Java 8 target works and finishes successfully. + +#### `patchelf` is installed in the infrastructure image + +During a release build the `yugabyted-ui/build.sh` program uses a system-wide `patchelf` and `ldd`. `ldd` comes preinstalled, `patchelf` needs to be added. An alternative would be to configure the `$PATH` so that _linuxbrew-provided_ binaries are in effect. Preferably the `build.sh` program should be modified to pick up the path from the downloaded _linuxbrew_. In reality, maintaining another patch is a stretch because installing `patchelf` is straightforward. Tracking issue: https://github.com/yugabyte/yugabyte-db/issues/18258. + +#### `yugabyted-ui/build.sh` doesn't call `ldd` + +Indeed. It's a no-op command printing output to the screen. It's pointless. Furthermore, it is not working on M1/M2 mac: + +``` +[2023-07-19T19:56:23 build.sh:48 main] Yugabyted UI Binary generated successfully at /tmp/yugabyted-ui/gobin/yugabyted-ui +Running ldd on /tmp/yugabyted-ui/gobin/yugabyted-ui +ldd: exited with unknown exit code (139) +``` + +Without the call to `ldd`, with the supplied `yugabyted-ui/build.sh.diff` patch applied: + +``` +[2023-07-19T20:12:36 build.sh:48 main] Yugabyted UI Binary generated successfully at /yb-source/build/release-clang15-linuxbrew-dynamic-ninja/gobin/yugabyted-ui +[2023-07-19T20:12:36 build.sh:57 main] Skipping ldd on generated library because it doesn't work on M1/M2 mac and the command is irrelevant anyway +[2023-07-19T20:12:36 build.sh:61 main] /yb-source/build/release-clang15-linuxbrew-dynamic-ninja/gobin/yugabyted-ui is correctly configured with the shared library interpreter: /lib64/ld-linux-x86-64.so.2 +``` + +Works on M1/M2 mac for Clang build. + +### Distribution + +#### M2 mac: GCC distribution doesn't work + +GCC build doesn't work due to errors in distribution creation, after `yugabyted-ui` build, where `ldd` is used extensively. Errors look like this: + +``` +Traceback (most recent call last): + File "/yb-source/python/yugabyte/yb_release_core_db.py", line 444, in + main() + File "/yb-source/python/yugabyte/yb_release_core_db.py", line 397, in main + library_packager.package_binaries() + File "/yb-source/python/yugabyte/library_packager.py", line 463, in package_binaries + deps = self.find_elf_dependencies(executable) + File "/yb-source/python/yugabyte/library_packager.py", line 335, in find_elf_dependencies + raise RuntimeError(ldd_result.error_msg) +RuntimeError: Non-zero exit code 1 from: /usr/bin/ldd /yb-source/build/release-gcc11-dynamic-ninja/bin/ldb ; stdout: '' stderr: 'qemu: uncaught target signal 11 (Segmentation fault) - core dumped +ldd: exited with unknown exit code (139)' +``` + +Apparently this is due to an upstream qemu issue: https://github.com/docker/for-mac/issues/5123. + +### Tests + +#### Java test execution slow due to compilation + +Run Java tests with `--sj` flag unless you have modified the actual Java class under test. This will skip Java compilation and use previously compiled classes. For example: + +```sh +yb-tests.sh raw --sj --java-test org.yb.pgsql.TestDropTableWithConcurrentTxn +``` + +The test execution speed can be improved further: + +```sh +# Skip Java and C++ compilation, this is useful when running a test repetitively, for example benchmarking: +yb-tests.sh raw --sj --scb --java-test org.yb.pgsql.TestDropTableWithConcurrentTxn +``` + +#### M2 mac: yb-tests.sh doesn't work + +C++ and Java tests don't work due to the following precondition failing: + +``` +[2246/2246] cd /yb-source/build/debug-gcc11-dynamic-ninja/src/yb/yql/pgwrapper && /usr/bin/cmake -E env YB_BUILD_ROOT=/yb-source/build/debug-gcc11-dynamic-ninja /yb-source/build-support/gen_initial_sys_catalog_snapshot_wrapper +FAILED: src/yb/yql/pgwrapper/CMakeFiles/initial_sys_catalog_snapshot +cd /yb-source/build/debug-gcc11-dynamic-ninja/src/yb/yql/pgwrapper && /usr/bin/cmake -E env YB_BUILD_ROOT=/yb-source/build/debug-gcc11-dynamic-ninja /yb-source/build-support/gen_initial_sys_catalog_snapshot_wrapper +2023-07-19 21:50:01,547 [gen_initial_sys_catalog_snapshot.py:55 INFO] Starting creating initial system catalog snapshot data +2023-07-19 21:50:01,550 [gen_initial_sys_catalog_snapshot.py:56 INFO] Logging to: /yb-source/build/debug-gcc11-dynamic-ninja/create_initial_sys_catalog_snapshot.err + +Standard output from external program {{ '/yb-source/build-support/run-test.sh' /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper/create_initial_sys_catalog_snapshot }} running in '/yb-source/build/debug-gcc11-dynamic-ninja', saving stdout to {{ /yb-source/build/debug-gcc11-dynamic-ninja/create_initial_sys_catalog_snapshot.out }}, stderr to {{ /yb-source/build/debug-gcc11-dynamic-ninja/create_initial_sys_catalog_snapshot.err }}: +Test is running on host 1cd8f50da476, arguments: /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper/create_initial_sys_catalog_snapshot +I0719 21:50:06.167414 445191 run-with-timeout.cc:133] Running on host 1cd8f50da476 +I0719 21:50:06.170686 445191 run-with-timeout.cc:140] PATH environment variable: /yb-source/build/venv/bin:/yb-source/build/venv/bin:/opt/rh/gcc-toolset-11/root/usr/bin:/yb-source/build/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +I0719 21:50:06.171424 445191 run-with-timeout.cc:148] Starting child process with a timeout of 1201 seconds +I0719 21:50:06.181224 445191 run-with-timeout.cc:157] Child process pid: 445209 +Note: Google Test filter = CreateInitialSysCatalogSnapshotTest.CreateInitialSysCatalogSnapshot +[==========] Running 1 test from 1 test suite. +[----------] Global test environment set-up. +[----------] 1 test from CreateInitialSysCatalogSnapshotTest +[ RUN ] CreateInitialSysCatalogSnapshotTest.CreateInitialSysCatalogSnapshot +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0719 21:50:08.390722 445209 env_posix.cc:1224] Create test dir failed: Already present (yb/util/env_posix.cc:1085): /tmp/yb_test.tmp.3879.20967.18980.pid445009: File exists (system error 17) +... +Standard error from external program {{ '/yb-source/build-support/run-test.sh' /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper/create_initial_sys_catalog_snapshot }} running in '/yb-source/build/debug-gcc11-dynamic-ninja', saving stdout to {{ /yb-source/build/debug-gcc11-dynamic-ninja/create_initial_sys_catalog_snapshot.out }}, stderr to {{ /yb-source/build/debug-gcc11-dynamic-ninja/create_initial_sys_catalog_snapshot.err }}: +Details of third-party dependencies: + YB_THIRDPARTY_DIR: /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20230418053213-a675ed2153-almalinux8-x86_64-gcc11 (from environment) + YB_THIRDPARTY_URL: https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20230418053213-a675ed2153-almalinux8-x86_64-gcc11/yugabyte-db-thirdparty-v20230418053213-a675ed2153-almalinux8-x86_64-gcc11.tar.gz (from environment) + YB_DOWNLOAD_THIRDPARTY: 1 + NO_REBUILD_THIRDPARTY: 1 ++ /yb-source/build/debug-gcc11-dynamic-ninja/bin/run-with-timeout 1201 /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper/create_initial_sys_catalog_snapshot --initial_sys_catalog_snapshot_dest_path=/yb-source/build/debug-gcc11-dynamic-ninja/share/initial_sys_catalog_snapshot --gtest_output=xml:/yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml --gtest_filter=CreateInitialSysCatalogSnapshotTest.CreateInitialSysCatalogSnapshot +[2023-07-19T22:10:08 common-test-env.sh:1187 did_test_succeed] Test failure reason: exit code: 1 +2023-07-19 22:10:08,664 [rewrite_test_log.py:173 INFO] test_host_name = 1cd8f50da476 +2023-07-19 22:10:08,668 [rewrite_test_log.py:173 INFO] m1_peer_id = d28ad8144cd94cea9c6b34bfa78a7aad +2023-07-19 22:10:08,668 [rewrite_test_log.py:173 INFO] sys_catalog_tablet_id = 00000000000000000000000000000000 +2023-07-19 22:10:08,669 [rewrite_test_log.py:173 INFO] BUILD_ROOT = /yb-source/build/debug-gcc11-dynamic-ninja +2023-07-19 22:10:08,669 [rewrite_test_log.py:173 INFO] YB_SRC_ROOT = /yb-source +2023-07-19 22:10:08,670 [rewrite_test_log.py:173 INFO] TEST_TMPDIR = /tmp/yb_test.tmp.3879.20967.18980.pid445009 + +TEST FAILURE +Test command: /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper/create_initial_sys_catalog_snapshot --initial_sys_catalog_snapshot_dest_path=/yb-source/build/debug-gcc11-dynamic-ninja/share/initial_sys_catalog_snapshot --gtest_output=xml:/yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml --gtest_filter=CreateInitialSysCatalogSnapshotTest.CreateInitialSysCatalogSnapshot +Test exit status: 1 +Log path: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.log +Found a core file at '/tmp/yb_test.tmp.3879.20967.18980.pid445009/core', backtrace: ++ echo '' ++ gdb -q -n -ex bt -ex 'thread apply all bt' -batch /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper/create_initial_sys_catalog_snapshot /tmp/yb_test.tmp.3879.20967.18980.pid445009/core ++ grep -Ev '^\[New LWP [0-9]+\]$' ++ /yb-source/python/yugabyte/dedup_thread_stacks.py ++ tee -a /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.log + +warning: Can't open file /usr/bin/qemu-x86_64 during file-backed mapping note processing + +warning: core file may not match specified executable file. + +warning: Selected architecture i386:x86-64 is not compatible with reported target architecture aarch64 + +warning: Architecture rejected target-supplied description + +warning: Unexpected size of section `.reg/445209' in core file. + +warning: Unexpected size of section `.reg2/445209' in core file. +Core was generated by `/usr/bin/qemu-x86_64 /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper'. +Program terminated with signal SIGQUIT, Quit. + +warning: Unexpected size of section `.reg/445209' in core file. + +warning: Unexpected size of section `.reg2/445209' in core file. +#0 0x707061727767702d in ?? () +[Current thread is 1 (LWP 445209)] +#0 0x707061727767702d in ?? () +Backtrace stopped: Cannot access memory at address 0x3 + +warning: Unexpected size of section `.reg/445243' in core file. +warning: Unexpected size of section `.reg2/445243' in core file. +#0 0x0000000000f10398 in ?? () +#1 0x0000000000000000 in ?? () + +warning: Unexpected size of section `.reg/445244' in core file. +warning: Unexpected size of section `.reg2/445244' in core file. +#0 0x0000000000f10398 in ?? () +#1 0x0000000000000000 in ?? () + +warning: Unexpected size of section `.reg/445245' in core file. +warning: Unexpected size of section `.reg2/445245' in core file. +#0 0x0000000000000008 in ?? () +Backtrace stopped: Cannot access memory at address 0x0 + +warning: Unexpected size of section `.reg/445241' in core file. +warning: Unexpected size of section `.reg2/445241' in core file. +#0 0x0000000000f10398 in ?? () +#1 0x0000000000000000 in ?? () + +warning: Unexpected size of section `.reg/445242' in core file. +warning: Unexpected size of section `.reg2/445242' in core file. +#0 0x0000000000000006 in ?? () +#1 0x0000000000000000 in ?? () + +warning: Unexpected size of section `.reg/445211' in core file. +warning: Unexpected size of section `.reg2/445211' in core file. +#0 0x0000ffffad7fc2c0 in __kernel_clock_gettime () +#1 0x00000001ffffffff in ?? () +#2 0x0000000100000000 in ?? () +#3 0x0000000000000001 in ?? () +#4 0x0000000000000000 in ?? () + +Backtrace stopped: Cannot access memory at address 0x3 +Thread 1 (LWP 445209) +#0 0x707061727767702d in ?? () +Thread 2 (LWP 445211), 3 (LWP 445242), 4 (LWP 445241), 5 (LWP 445245), 6 (LWP 445244), 7 (LWP 445243) + + +tests-pgwrapper/create_initial_sys_catalog_snapshot failed to produce an XML output file at /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +Generating an XML output file using parse_test_failure.py: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +[2023-07-19T22:10:11 common-test-env.sh:806 handle_cxx_test_xml_output] Test failed, updating /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml ++ /yb-source/python/yugabyte/update_test_result_xml.py --result-xml /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml --mark-as-failed true +2023-07-19 22:10:11,943 [postprocess_test_result.py:187 INFO] Log path: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.log +2023-07-19 22:10:11,946 [postprocess_test_result.py:188 INFO] JUnit XML path: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +2023-07-19 22:10:14,868 [postprocess_test_result.py:316 INFO] Wrote JSON test report file: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot_test_report.json +(end of standard error) +Traceback (most recent call last): + File "/yb-source/python/yugabyte/gen_initial_sys_catalog_snapshot.py", line 83, in + main() + File "/yb-source/python/yugabyte/gen_initial_sys_catalog_snapshot.py", line 74, in main + raise RuntimeError("initdb failed in %.1f sec" % elapsed_time_sec) +RuntimeError: initdb failed in 1213.4 sec +ninja: build stopped: subcommand failed. + +real 160m43.874s +user 777m47.802s +sys 43m12.371s +[2023-07-19T22:10:15 yb_build.sh:589 run_cxx_build] C++ build finished with exit code 1 (build type: debug, compiler: gcc11). Timing information is available above. +``` + +A similar error appears on any test attempt due to: + +``` +[7/8] cd /yb-source/build/debug-clang15-dynamic-ninja/src/yb/yql/pgwrapper && /usr/bin/cmake -E env YB_BUILD_ROOT=/yb-source/build/debug-clang15-dynamic-ninja /yb-source/build-support/gen_initial_sys_catalog_snapshot_wrapper +``` + +Neither C++, nor Java tests are working: + +``` +TEST FAILURE +Test command: /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper/create_initial_sys_catalog_snapshot --initial_sys_catalog_snapshot_dest_path=/yb-source/build/debug-gcc11-dynamic-ninja/share/initial_sys_catalog_snapshot --gtest_output=xml:/yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml --gtest_filter=CreateInitialSysCatalogSnapshotTest.CreateInitialSysCatalogSnapshot +Test exit status: 1 +Log path: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.log +Found a core file at '/tmp/yb_test.tmp.3879.20967.18980.pid445009/core', backtrace: ++ echo '' ++ gdb -q -n -ex bt -ex 'thread apply all bt' -batch /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper/create_initial_sys_catalog_snapshot /tmp/yb_test.tmp.3879.20967.18980.pid445009/core ++ grep -Ev '^\[New LWP [0-9]+\]$' ++ /yb-source/python/yugabyte/dedup_thread_stacks.py ++ tee -a /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.log + +warning: Can't open file /usr/bin/qemu-x86_64 during file-backed mapping note processing + +warning: core file may not match specified executable file. + +warning: Selected architecture i386:x86-64 is not compatible with reported target architecture aarch64 + +warning: Architecture rejected target-supplied description + +warning: Unexpected size of section `.reg/445209' in core file. + +warning: Unexpected size of section `.reg2/445209' in core file. +Core was generated by `/usr/bin/qemu-x86_64 /yb-source/build/debug-gcc11-dynamic-ninja/tests-pgwrapper'. +Program terminated with signal SIGQUIT, Quit. + +warning: Unexpected size of section `.reg/445209' in core file. + +warning: Unexpected size of section `.reg2/445209' in core file. +#0 0x707061727767702d in ?? () +[Current thread is 1 (LWP 445209)] +#0 0x707061727767702d in ?? () +Backtrace stopped: Cannot access memory at address 0x3 + +warning: Unexpected size of section `.reg/445243' in core file. +warning: Unexpected size of section `.reg2/445243' in core file. +#0 0x0000000000f10398 in ?? () +#1 0x0000000000000000 in ?? () + +warning: Unexpected size of section `.reg/445244' in core file. +warning: Unexpected size of section `.reg2/445244' in core file. +#0 0x0000000000f10398 in ?? () +#1 0x0000000000000000 in ?? () + +warning: Unexpected size of section `.reg/445245' in core file. +warning: Unexpected size of section `.reg2/445245' in core file. +#0 0x0000000000000008 in ?? () +Backtrace stopped: Cannot access memory at address 0x0 + +warning: Unexpected size of section `.reg/445241' in core file. +warning: Unexpected size of section `.reg2/445241' in core file. +#0 0x0000000000f10398 in ?? () +#1 0x0000000000000000 in ?? () + +warning: Unexpected size of section `.reg/445242' in core file. +warning: Unexpected size of section `.reg2/445242' in core file. +#0 0x0000000000000006 in ?? () +#1 0x0000000000000000 in ?? () + +warning: Unexpected size of section `.reg/445211' in core file. +warning: Unexpected size of section `.reg2/445211' in core file. +#0 0x0000ffffad7fc2c0 in __kernel_clock_gettime () +#1 0x00000001ffffffff in ?? () +#2 0x0000000100000000 in ?? () +#3 0x0000000000000001 in ?? () +#4 0x0000000000000000 in ?? () + +Backtrace stopped: Cannot access memory at address 0x3 +Thread 1 (LWP 445209) +#0 0x707061727767702d in ?? () +Thread 2 (LWP 445211), 3 (LWP 445242), 4 (LWP 445241), 5 (LWP 445245), 6 (LWP 445244), 7 (LWP 445243) + + +tests-pgwrapper/create_initial_sys_catalog_snapshot failed to produce an XML output file at /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +Generating an XML output file using parse_test_failure.py: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +[2023-07-19T22:10:11 common-test-env.sh:806 handle_cxx_test_xml_output] Test failed, updating /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml ++ /yb-source/python/yugabyte/update_test_result_xml.py --result-xml /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml --mark-as-failed true +2023-07-19 22:10:11,943 [postprocess_test_result.py:187 INFO] Log path: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.log +2023-07-19 22:10:11,946 [postprocess_test_result.py:188 INFO] JUnit XML path: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +2023-07-19 22:10:14,868 [postprocess_test_result.py:316 INFO] Wrote JSON test report file: /yb-source/build/debug-gcc11-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot_test_report.json +(end of standard error) +Traceback (most recent call last): + File "/yb-source/python/yugabyte/gen_initial_sys_catalog_snapshot.py", line 83, in + main() + File "/yb-source/python/yugabyte/gen_initial_sys_catalog_snapshot.py", line 74, in main + raise RuntimeError("initdb failed in %.1f sec" % elapsed_time_sec) +RuntimeError: initdb failed in 1213.4 sec +ninja: build stopped: subcommand failed. +``` + +``` +[INFO] ------------------------------------------------------- +[INFO] T E S T S +[INFO] ------------------------------------------------------- +[INFO] Running org.yb.pgsql.TestDropTableWithConcurrentTxn +[ERROR] Tests run: 2, Failures: 0, Errors: 2, Skipped: 0, Time elapsed: 1,816.954 s <<< FAILURE! - in org.yb.pgsql.TestDropTableWithConcurrentTxn +[ERROR] testDmlTxnDropWithReadCommitted(org.yb.pgsql.TestDropTableWithConcurrentTxn) Time elapsed: 612.905 s <<< ERROR! +java.lang.Exception: Operation timed out after 600000ms + +[ERROR] testDmlTxnDrop(org.yb.pgsql.TestDropTableWithConcurrentTxn) Time elapsed: 1,200.09 s <<< ERROR! +org.junit.runners.model.TestTimedOutException: test timed out after 1200 seconds + at app//org.yb.pgsql.TestDropTableWithConcurrentTxn.runDmlTxnWithDropOnCurrentResource(TestDropTableWithConcurrentTxn.java:180) + at app//org.yb.pgsql.TestDropTableWithConcurrentTxn.testDmlTxnDropInternal(TestDropTableWithConcurrentTxn.java:384) + at app//org.yb.pgsql.TestDropTableWithConcurrentTxn.testDmlTxnDrop(TestDropTableWithConcurrentTxn.java:398) + +[INFO] +[INFO] Results: +[INFO] +[ERROR] Errors: +[ERROR] TestDropTableWithConcurrentTxn.testDmlTxnDrop:398->testDmlTxnDropInternal:384->runDmlTxnWithDropOnCurrentResource:180 ยป TestTimedOut +[ERROR] TestDropTableWithConcurrentTxn>BasePgSQLTest.initPostgresBefore:306 ยป Operati... +[INFO] +[ERROR] Tests run: 2, Failures: 0, Errors: 2, Skipped: 0 +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] BUILD FAILURE +[INFO] ------------------------------------------------------------------------ +[INFO] Total time: 30:34 min +[INFO] Finished at: 2023-07-19T23:05:17Z +[INFO] ------------------------------------------------------------------------ +[ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:2.22.2:test (default-cli) on project yb-pgsql: There are test failures. +[ERROR] +[ERROR] Please refer to /yb-source/java/yb-pgsql/target/surefire-reports_org.yb.pgsql.TestDropTableWithConcurrentTxn for the individual test results. +[ERROR] Please refer to dump files (if any exist) [date].dump, [date]-jvmRun[N].dump and [date].dumpstream. +[ERROR] -> [Help 1] +[ERROR] +[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. +[ERROR] Re-run Maven using the -X switch to enable full debug logging. +[ERROR] +[ERROR] For more information about the errors and possible solutions, please read the following articles: +[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException + +real 30m39.960s +user 82m14.171s +sys 3m4.636s +[2023-07-19T23:05:17 common-test-env.sh:1661 run_java_test] Maven exited with code 1 +[2023-07-19T23:05:20 common-test-env.sh:1691 run_java_test] Filtered JUnit XML file '/yb-source/java/yb-pgsql/target/surefire-reports_org.yb.pgsql.TestDropTableWithConcurrentTxn/TEST-org.yb.pgsql.TestDropTableWithConcurrentTxn.xml' is valid +./build-support/common-test-env.sh: line 1041: TEST_TMPDIR: unbound variable +[2023-07-19T23:05:20 common-test-env.sh:1051 rewrite_test_log] Test log rewrite script failed with exit code 1 +2023-07-19 23:05:20,783 [postprocess_test_result.py:187 INFO] Log path: /yb-source/java/yb-pgsql/target/surefire-reports_org.yb.pgsql.TestDropTableWithConcurrentTxn/org.yb.pgsql.TestDropTableWithConcurrentTxn-output.txt +2023-07-19 23:05:20,786 [postprocess_test_result.py:188 INFO] JUnit XML path: /yb-source/java/yb-pgsql/target/surefire-reports_org.yb.pgsql.TestDropTableWithConcurrentTxn/TEST-org.yb.pgsql.TestDropTableWithConcurrentTxn.xml +2023-07-19 23:05:20,788 [postprocess_test_result.py:201 INFO] Externally specified class name: org.yb.pgsql.TestDropTableWithConcurrentTxn +2023-07-19 23:05:27,753 [postprocess_test_result.py:316 INFO] Wrote JSON test report file: /yb-source/java/yb-pgsql/target/surefire-reports_org.yb.pgsql.TestDropTableWithConcurrentTxn/TEST-org.yb.pgsql.TestDropTableWithConcurrentTxn_test_report.json +``` + +This problem also applies to Clang based builds: + +``` +tests-pgwrapper/create_initial_sys_catalog_snapshot failed to produce an XML output file at /yb-source/build/debug-clang15-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +Generating an XML output file using parse_test_failure.py: /yb-source/build/debug-clang15-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +[2023-07-20T01:36:34 common-test-env.sh:806 handle_cxx_test_xml_output] Test failed, updating /yb-source/build/debug-clang15-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml ++ /yb-source/python/yugabyte/update_test_result_xml.py --result-xml /yb-source/build/debug-clang15-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml --mark-as-failed true +2023-07-20 01:36:35,181 [postprocess_test_result.py:187 INFO] Log path: /yb-source/build/debug-clang15-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.log +2023-07-20 01:36:35,184 [postprocess_test_result.py:188 INFO] JUnit XML path: /yb-source/build/debug-clang15-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot.xml +2023-07-20 01:36:38,127 [postprocess_test_result.py:316 INFO] Wrote JSON test report file: /yb-source/build/debug-clang15-dynamic-ninja/yb-test-logs/tests-pgwrapper__create_initial_sys_catalog_snapshot/CreateInitialSysCatalogSnapshotTest_CreateInitialSysCatalogSnapshot_test_report.json +(end of standard error) +Traceback (most recent call last): + File "/yb-source/python/yugabyte/gen_initial_sys_catalog_snapshot.py", line 83, in + main() + File "/yb-source/python/yugabyte/gen_initial_sys_catalog_snapshot.py", line 74, in main + raise RuntimeError("initdb failed in %.1f sec" % elapsed_time_sec) +RuntimeError: initdb failed in 1212.1 sec +ninja: build stopped: subcommand failed. + +real 128m18.551s +user 681m26.286s +sys 31m2.172s +[2023-07-20T01:36:38 yb_build.sh:589 run_cxx_build] C++ build finished with exit code 1 (build type: debug, compiler: clang15). Timing information is available above. +```