From 3f8dcbd5ea5f738d258595513629dc2ac289e10e Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 10 Jun 2021 11:24:27 +0800 Subject: [PATCH] re-cherry-pick #2074 #2070 to release-5.0 (#2122) --- .ci/build.groovy | 72 ++++++++++++ .ci/integration_test.groovy | 26 ++--- .ci/util.groovy | 97 +++++++++++------ CMakeLists.txt | 17 ++- cmake/find_ccache.cmake | 19 ++-- contrib/poco | 2 +- dbms/src/Common/UInt128.h | 9 ++ dbms/src/Core/Types.h | 4 + dbms/src/DataStreams/PKColumnIterator.hpp | 12 +- dbms/src/Functions/GeoUtils.h | 4 + .../Storages/DeltaMerge/DeltaMergeStore.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.h | 7 ++ dbms/src/Storages/Page/PageUtil.cpp | 2 + release-centos7/build/build-tiflash-ci.sh | 103 ++++++++++++++---- .../build/build-tiflash-release.sh | 3 +- release-centos7/build/env.sh | 3 + release-centos7/build/fetch-ci-build.sh | 35 ++++++ release-centos7/build/upload-ci-build.sh | 20 ++++ release-darwin/build/build-tiflash-release.sh | 4 + tests/docker/gtest.yaml | 3 +- tests/docker/mock-test-dt.yaml | 7 +- tests/docker/mock-test-tmt.yaml | 7 +- tests/docker/run.sh | 12 +- tests/docker/tiflash-dt.yaml | 7 +- tests/docker/tiflash-tagged-image.yaml | 27 +++++ 25 files changed, 405 insertions(+), 99 deletions(-) create mode 100644 .ci/build.groovy create mode 100644 release-centos7/build/env.sh create mode 100755 release-centos7/build/fetch-ci-build.sh create mode 100755 release-centos7/build/upload-ci-build.sh create mode 100644 tests/docker/tiflash-tagged-image.yaml diff --git a/.ci/build.groovy b/.ci/build.groovy new file mode 100644 index 00000000000..139dec9317f --- /dev/null +++ b/.ci/build.groovy @@ -0,0 +1,72 @@ +catchError { + def util = load('util.groovy') + + def label = "build-tics" + + def tiflashTag = ({ + def m = ghprbCommentBody =~ /tiflash\s*=\s*([^\s\\]+)(\s|\\|$)/ + if (m) { + return "${m.group(1)}" + } + return params.tiflashTag ?: ghprbTargetBranch ?: 'master' + }).call() + + podTemplate(name: label, label: label, instanceCap: 5, + workspaceVolume: emptyDirWorkspaceVolume(memory: true), + nodeSelector: 'role_type=slave', + containers: [ + containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true), + containerTemplate(name: 'docker', image: 'hub.pingcap.net/zyguan/docker:build-essential', + alwaysPullImage: true, envVars: [ + envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375'), + ], ttyEnabled: true, command: 'cat'), + containerTemplate(name: 'builder', image: 'hub.pingcap.net/tiflash/tiflash-builder-ci', + alwaysPullImage: true, ttyEnabled: true, command: 'cat', + resourceRequestCpu: '4000m', resourceRequestMemory: '8Gi', + resourceLimitCpu: '10000m', resourceLimitMemory: '30Gi'), + ]) { + + node(label) { + + dir("tics") { + stage("Checkout") { + container("docker") { + sh """ + archive_url=${FILE_SERVER_URL}/download/builds/pingcap/tics/cache/tics-repo_latest.tar.gz + if [ ! -d contrib ]; then curl -sL \$archive_url | tar -zx --strip-components=1 || true; fi + """ + sh "chown -R 1000:1000 ./" + // sh "if ! grep -q hub.pingcap.net /etc/hosts ; then echo '172.16.10.5 hub.pingcap.net' >> /etc/hosts; fi" + } + util.checkoutTiCSFull("${ghprbActualCommit}", "${ghprbPullId}") + } + stage("Build & Upload") { + timeout(time: 70, unit: 'MINUTES') { + container("builder") { + sh "NPROC=5 BUILD_BRANCH=${ghprbTargetBranch} release-centos7/build/build-tiflash-ci.sh" + sh "PULL_ID=${ghprbPullId} COMMIT_HASH=${ghprbActualCommit} release-centos7/build/upload-ci-build.sh" + } + } + } + } + } + } +} + + +stage('Summary') { + def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) + def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" + + "${ghprbPullLink}" + "\n" + + "${ghprbPullDescription}" + "\n" + + "Build Result: `${currentBuild.currentResult}`" + "\n" + + "Elapsed Time: `${duration} mins` " + "\n" + + "${env.RUN_DISPLAY_URL}" + + echo "${msg}" + + if (currentBuild.currentResult != "SUCCESS") { + echo "Send slack here ..." + slackSend channel: '#jenkins-ci', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${msg}" + } +} diff --git a/.ci/integration_test.groovy b/.ci/integration_test.groovy index e5d9e499fc9..c392a840204 100644 --- a/.ci/integration_test.groovy +++ b/.ci/integration_test.groovy @@ -9,18 +9,8 @@ catchError { return params.ghprbTargetBranch ?: 'master' }).call() - echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}" - - stage("Wait for images") { - util.runClosure("wait-for-images") { - timeout(time: 60, unit: 'MINUTES') { - container("docker") { - sh """ - while ! docker pull hub.pingcap.net/tiflash/tics:${params.ghprbActualCommit}; do sleep 60; done - """ - } - } - } + stage("Wait for ci build") { + echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}" } node("${GO_BUILD_SLAVE}") { @@ -33,13 +23,20 @@ catchError { cp -R /nfs/cache/git/src-tics.tar.gz* ./ mkdir -p ${curws}/tics tar -xzf src-tics.tar.gz -C ${curws}/tics --strip-components=1 - """ + """ } } } dir("${curws}/tics") { util.checkoutTiCS("${params.ghprbActualCommit}", "${params.ghprbPullId}") } + timeout(time: 60, unit: 'MINUTES') { + container("golang") { + sh """ + COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=${curws}/tics/tests/.build bash -e ${curws}/tics/release-centos7/build/fetch-ci-build.sh + """ + } + } } stash includes: "tics/**", name: "git-code-tics", useDefaultExcludes: false } @@ -74,10 +71,9 @@ catchError { stage('Summary') { def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def msg = "Build Result: `${currentBuild.currentResult}`" + "\n" + + def msg = "Result: `${currentBuild.currentResult}`" + "\n" + "Elapsed Time: `${duration} mins`" + "\n" + "${env.RUN_DISPLAY_URL}" echo "${msg}" - } diff --git a/.ci/util.groovy b/.ci/util.groovy index 4fc13dd8b47..016428afd4d 100644 --- a/.ci/util.groovy +++ b/.ci/util.groovy @@ -1,26 +1,30 @@ +def doCheckout(commit, refspec) { + checkout(changelog: false, poll: false, scm: [ + $class : "GitSCM", + branches : [ + [name: "${commit}"], + ], + userRemoteConfigs: [ + [ + url : "git@github.com:pingcap/tics.git", + refspec : refspec, + credentialsId: "github-sre-bot-ssh", + ] + ], + extensions : [ + [$class: 'PruneStaleBranch'], + [$class: 'CleanBeforeCheckout'], + ], + ]) +} + def checkoutTiCS(commit, pullId) { def refspec = "+refs/heads/*:refs/remotes/origin/*" if (pullId) { refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*" } try { - checkout(changelog: false, poll: false, scm: [ - $class : "GitSCM", - branches : [ - [name: "${commit}"], - ], - userRemoteConfigs: [ - [ - url : "git@github.com:pingcap/tics.git", - refspec : refspec, - credentialsId: "github-sre-bot-ssh", - ] - ], - extensions : [ - [$class: 'PruneStaleBranch'], - [$class: 'CleanBeforeCheckout'], - ], - ]) + doCheckout(commit, refspec) } catch (info) { retry(2) { echo "checkout failed, retry.." @@ -29,27 +33,42 @@ def checkoutTiCS(commit, pullId) { echo ".git already exist or not a valid git dir. Delete dir..." deleteDir() } - checkout(changelog: false, poll: false, scm: [ - $class : "GitSCM", - branches : [ - [name: "${commit}"], - ], - userRemoteConfigs: [ - [ - url : "git@github.com:pingcap/tics.git", - refspec : refspec, - credentialsId: "github-sre-bot-ssh", - ] - ], - extensions : [ - [$class: 'PruneStaleBranch'], - [$class: 'CleanBeforeCheckout'], - ], - ]) + doCheckout(commit, refspec) } } } +def checkoutTiCSFull(commit, pullId) { + def refspec = "+refs/heads/*:refs/remotes/origin/*" + if (pullId) { + refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*" + } + checkout(changelog: false, poll: false, scm: [ + $class : "GitSCM", + branches : [ + [name: "${commit}"], + ], + userRemoteConfigs : [ + [ + url : "git@github.com:pingcap/tics.git", + refspec : refspec, + credentialsId: "github-sre-bot-ssh", + ] + ], + extensions : [ + [$class : 'SubmoduleOption', + disableSubmodules : false, + parentCredentials : true, + recursiveSubmodules: true, + trackingSubmodules : false, + reference : ''], + [$class: 'PruneStaleBranch'], + [$class: 'CleanBeforeCheckout'], + ], + doGenerateSubmoduleConfigurations: false, + ]) +} + def runClosure(label, Closure body) { podTemplate(name: label, label: label, instanceCap: 15, containers: [ containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true, @@ -70,6 +89,16 @@ def runTest(label, testPath, tidbBranch) { runClosure(label) { stage("Unstash") { unstash 'git-code-tics' + dir("tics") { + timeout(time: 5, unit: 'MINUTES') { + container("docker") { + sh """ + pwd + DOWNLOAD_TAR=true COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=./tests/.build bash -e release-centos7/build/fetch-ci-build.sh + """ + } + } + } } dir(testPath) { stage("Test") { diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c1c6a368f7..7e3d9857607 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ endif () # Write compile_commands.json set(CMAKE_EXPORT_COMPILE_COMMANDS 1) +option (USE_CCACHE "Set to OFF to disable ccache" ON) include (cmake/find_ccache.cmake) if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") @@ -148,14 +149,24 @@ if (NOT ARCH_ARM) set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3") endif () +option (DEBUG_WITHOUT_DEBUG_INFO "Set to ON to build dev target without debug info (remove flag `-g` in order to accelerate compiling speed and reduce target binary size)" OFF) +if (DEBUG_WITHOUT_DEBUG_INFO) + string(REPLACE "-g" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") + string(REPLACE "-g" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") + message (STATUS "Build debug target without debug info") +else () + set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g3 -ggdb3") + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g3 -ggdb3") +endif () + set (CMAKE_BUILD_COLOR_MAKEFILE ON) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}") set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${CMAKE_CXX_FLAGS_ADD}") -set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}") +set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fno-inline ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_FLAGS_ADD}") -set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -fno-inline ${CMAKE_C_FLAGS_ADD}") if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND ARCH_FREEBSD)) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++") @@ -228,7 +239,7 @@ if (UNBUNDLED OR NOT (ARCH_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () -message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}") +message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED}") include(GNUInstallDirs) diff --git a/cmake/find_ccache.cmake b/cmake/find_ccache.cmake index 22372c3cbfe..eef203f9cf1 100644 --- a/cmake/find_ccache.cmake +++ b/cmake/find_ccache.cmake @@ -1,14 +1,19 @@ find_program (CCACHE_FOUND ccache) -if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache") - execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION) - string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION} ) +if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache") + execute_process (COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION) + execute_process (COMMAND ${CCACHE_FOUND} "-p" OUTPUT_VARIABLE CCACHE_CONFIG) + string (REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION}) if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - #message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}") - set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) - set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) + message (STATUS "Using ccache: ${CCACHE_FOUND}, version ${CCACHE_VERSION}") + message (STATUS "Show ccache config:") + message ("${CCACHE_CONFIG}") + set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) + set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) else () - message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118") + message (STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118") endif () +else () + message (STATUS "Not using ccache ${CCACHE_FOUND}, USE_CCACHE=${USE_CCACHE}") endif () diff --git a/contrib/poco b/contrib/poco index f22d49ee709..00fd749ea7b 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit f22d49ee709867cdd2a789cc581449ee25f42a7f +Subproject commit 00fd749ea7b244172fa173c9a8c240d5c9b513ab diff --git a/dbms/src/Common/UInt128.h b/dbms/src/Common/UInt128.h index 3481640529d..1080a744e94 100644 --- a/dbms/src/Common/UInt128.h +++ b/dbms/src/Common/UInt128.h @@ -193,16 +193,25 @@ template <> struct is_signed static constexpr bool value = false; }; +template <> +inline constexpr bool is_signed_v = is_signed::value; + template <> struct is_unsigned { static constexpr bool value = true; }; +template <> +inline constexpr bool is_unsigned_v = is_unsigned::value; + template <> struct is_integral { static constexpr bool value = true; }; +template <> +inline constexpr bool is_integral_v = is_integral::value; + // Operator +, -, /, *, % aren't implemented so it's not an arithmetic type template <> struct is_arithmetic { diff --git a/dbms/src/Core/Types.h b/dbms/src/Core/Types.h index da8db874e65..fdae3a60e49 100644 --- a/dbms/src/Core/Types.h +++ b/dbms/src/Core/Types.h @@ -30,6 +30,10 @@ struct numeric_limits<__int128_t> #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#if defined(__clang__) +#pragma GCC diagnostic ignored "-Wunknown-warning-option" +#pragma GCC diagnostic ignored "-Wdeprecated-copy" +#endif #include #pragma GCC diagnostic pop diff --git a/dbms/src/DataStreams/PKColumnIterator.hpp b/dbms/src/DataStreams/PKColumnIterator.hpp index 23682cc2ab0..d70c228c87e 100644 --- a/dbms/src/DataStreams/PKColumnIterator.hpp +++ b/dbms/src/DataStreams/PKColumnIterator.hpp @@ -14,8 +14,7 @@ struct PKColumnIterator : public std::iterator diff --git a/dbms/src/Functions/GeoUtils.h b/dbms/src/Functions/GeoUtils.h index d1267b18769..9ce40d558eb 100644 --- a/dbms/src/Functions/GeoUtils.h +++ b/dbms/src/Functions/GeoUtils.h @@ -15,6 +15,10 @@ #endif #pragma GCC diagnostic ignored "-Wunused-parameter" +#if defined(__clang__) +#pragma GCC diagnostic ignored "-Wunknown-warning-option" +#pragma GCC diagnostic ignored "-Wdeprecated-copy" +#endif #include diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 0b1516e3235..d8b51a020b7 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -1734,7 +1734,7 @@ void DeltaMergeStore::applyAlters(const AlterCommands & commands, original_table_columns.swap(new_original_table_columns); store_columns.swap(new_store_columns); - std::atomic_store(&original_table_header, std::make_shared(toEmptyBlock(original_table_columns))); + std::atomic_store(&original_table_header, std::make_shared(toEmptyBlock(original_table_columns))); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index d7d72dbc2de..2441c3d1429 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -17,10 +17,17 @@ #include #include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#if defined(__clang__) +#pragma GCC diagnostic ignored "-Wunknown-warning-option" +#pragma GCC diagnostic ignored "-Wdeprecated-copy" +#endif #include #include #include #include +#pragma GCC diagnostic pop namespace TiDB { diff --git a/dbms/src/Storages/Page/PageUtil.cpp b/dbms/src/Storages/Page/PageUtil.cpp index bf33bbe4930..1a7ee25714f 100644 --- a/dbms/src/Storages/Page/PageUtil.cpp +++ b/dbms/src/Storages/Page/PageUtil.cpp @@ -75,6 +75,7 @@ void writeFile(WritableFilePtr & file, UInt64 offset, char * data, size_t to_wri res = file->pwrite(data + bytes_written, to_write - bytes_written, offset + bytes_written); } +#ifndef NDEBUG // Can inject failpoint under debug mode fiu_do_on(FailPoints::force_set_page_file_write_errno, { if (enable_failpoint) @@ -83,6 +84,7 @@ void writeFile(WritableFilePtr & file, UInt64 offset, char * data, size_t to_wri errno = ENOSPC; } }); +#endif if ((-1 == res || 0 == res) && errno != EINTR) { ProfileEvents::increment(ProfileEvents::PSMWriteFailed); diff --git a/release-centos7/build/build-tiflash-ci.sh b/release-centos7/build/build-tiflash-ci.sh index 16cdca2ba01..9458c50d5e6 100755 --- a/release-centos7/build/build-tiflash-ci.sh +++ b/release-centos7/build/build-tiflash-ci.sh @@ -1,28 +1,79 @@ #!/bin/bash +command -v ccache > /dev/null 2>&1 +if [[ $? != 0 ]]; then + echo "try to install ccache" + wget http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/ccache.x86_64.rpm + rpm -Uvh ccache.x86_64.rpm +else + echo "ccache has been installed" +fi + set -ueox pipefail SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" SRCPATH=${1:-$(cd $SCRIPTPATH/../..; pwd -P)} -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} +CI_CCACHE_USED_SRCPATH="/build/tics" +export INSTALL_DIR=${INSTALL_DIR:-"$SRCPATH/release-centos7/tiflash"} + +if [[ ${CI_CCACHE_USED_SRCPATH} != ${SRCPATH} ]]; then + rm -rf "${CI_CCACHE_USED_SRCPATH}" + mkdir -p /build && cd /build + cp -r ${SRCPATH} ${CI_CCACHE_USED_SRCPATH} + sh ${CI_CCACHE_USED_SRCPATH}/release-centos7/build/build-tiflash-ci.sh + exit 0 +fi +NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} ENABLE_TEST=${ENABLE_TEST:-1} ENABLE_EMBEDDED_COMPILER="FALSE" CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} - +UPDATE_CCACHE=${UPDATE_CCACHE:-false} +BUILD_BRANCH=${BUILD_BRANCH:-master} +BUILD_UPDATE_DEBUG_CI_CCACHE=${BUILD_UPDATE_DEBUG_CI_CCACHE:-false} +CCACHE_REMOTE_TAR="${BUILD_BRANCH}-${CMAKE_BUILD_TYPE}.tar" +CCACHE_REMOTE_TAR=$(echo "${CCACHE_REMOTE_TAR}" | tr 'A-Z' 'a-z') if [[ "${CMAKE_BUILD_TYPE}" != "Debug" ]]; then ENABLE_TEST=0 fi +# https://cd.pingcap.net/blue/organizations/jenkins/build_tiflash_multi_branch/activity/ +# Each time after a new commit merged into target branch, a task about nightly build will be triggered. +# BUILD_UPDATE_DEBUG_CI_CCACHE is set true in order to build and upload ccache. +if [[ "${BUILD_UPDATE_DEBUG_CI_CCACHE}" != "false" ]]; then + echo "====== begin to build & upload ccache for ci debug build ======" + UPDATE_CCACHE=true NPROC=${NPROC} BUILD_BRANCH=${BUILD_BRANCH} CMAKE_BUILD_TYPE=Debug BUILD_UPDATE_DEBUG_CI_CCACHE=false sh ${SRCPATH}/release-centos7/build/build-tiflash-ci.sh + echo "====== finish build & upload ccache for ci debug build ======" +fi -set -xe - -install_dir="$SRCPATH/release-centos7/tiflash" -if [ -d "$install_dir" ]; then rm -rf "$install_dir"/*; else mkdir -p "$install_dir"; fi +rm -rf "${INSTALL_DIR}" +mkdir -p "${INSTALL_DIR}" + +USE_CCACHE=ON +rm -rf "${SRCPATH}/.ccache" +cache_file="${SRCPATH}/ccache.tar" +rm -rf "${cache_file}" +curl -o "${cache_file}" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR} +cache_size=`ls -l "${cache_file}" | awk '{print $5}'` +min_size=$((1024000)) +if [[ ${cache_size} -gt ${min_size} ]]; then + echo "try to use ccache to accelerate compile speed" + cd "${SRCPATH}" + tar -xf ccache.tar +fi +ccache -o cache_dir="${SRCPATH}/.ccache" +ccache -o max_size=2G +ccache -o limit_multiple=0.99 +ccache -o hash_dir=false +ccache -o compression=true +ccache -o compression_level=6 +if [[ ${UPDATE_CCACHE} == "false" ]]; then + ccache -o read_only=true +fi +ccache -z pushd ${SRCPATH}/cluster_manage/ rm -rf ./dist ./release.sh -cp -r dist/flash_cluster_manager "$install_dir"/flash_cluster_manager popd if [ -d "$SRCPATH/contrib/kvproto" ]; then @@ -62,29 +113,41 @@ fi chmod 0731 "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" build_dir="$SRCPATH/release-centos7/build-release" +rm -rf ${build_dir} mkdir -p $build_dir && cd $build_dir cmake "$SRCPATH" \ -DENABLE_EMBEDDED_COMPILER=$ENABLE_EMBEDDED_COMPILER \ -DENABLE_TESTS=$ENABLE_TEST \ - -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE + -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ + -DUSE_CCACHE=${USE_CCACHE} \ + -DDEBUG_WITHOUT_DEBUG_INFO=ON make -j $NPROC tiflash -# Reduce binary size by compressing. -objcopy --compress-debug-sections=zlib-gnu "$build_dir/dbms/src/Server/tiflash" - -cp -f "$build_dir/dbms/src/Server/tiflash" "$install_dir/tiflash" -cp -f "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" "$install_dir/libtiflash_proxy.so" - # copy gtest binary under Debug mode if [[ "${CMAKE_BUILD_TYPE}" = "Debug" && ${ENABLE_TEST} -ne 0 ]]; then #ctest -V -j $(nproc || grep -c ^processor /proc/cpuinfo) make -j ${NPROC} gtests_dbms gtests_libcommon - cp -f "$build_dir/dbms/gtests_dbms" "$install_dir/" - cp -f "$build_dir/libs/libcommon/src/tests/gtests_libcommon" "$install_dir/" + cp -f "$build_dir/dbms/gtests_dbms" "${INSTALL_DIR}/" + cp -f "$build_dir/libs/libcommon/src/tests/gtests_libcommon" "${INSTALL_DIR}/" fi -ldd "$install_dir/tiflash" -cd "$install_dir" -chrpath -d libtiflash_proxy.so "$install_dir/tiflash" -ldd "$install_dir/tiflash" +ccache -s + +if [[ ${UPDATE_CCACHE} == "true" ]]; then + cd ${SRCPATH} + rm -rf ccache.tar + tar -cf ccache.tar .ccache + curl -F builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR}=@ccache.tar http://fileserver.pingcap.net/upload +fi + +# Reduce binary size by compressing. +objcopy --compress-debug-sections=zlib-gnu "$build_dir/dbms/src/Server/tiflash" +cp -r "${SRCPATH}/cluster_manage/dist/flash_cluster_manager" "${INSTALL_DIR}"/flash_cluster_manager +cp -f "$build_dir/dbms/src/Server/tiflash" "${INSTALL_DIR}/tiflash" +cp -f "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" "${INSTALL_DIR}/libtiflash_proxy.so" +ldd "${INSTALL_DIR}/tiflash" +cd "${INSTALL_DIR}" +chrpath -d libtiflash_proxy.so "${INSTALL_DIR}/tiflash" +ldd "${INSTALL_DIR}/tiflash" +ls -lh "${INSTALL_DIR}" diff --git a/release-centos7/build/build-tiflash-release.sh b/release-centos7/build/build-tiflash-release.sh index fdc3052a641..7a6a5e9db8f 100755 --- a/release-centos7/build/build-tiflash-release.sh +++ b/release-centos7/build/build-tiflash-release.sh @@ -50,7 +50,8 @@ cmake "$SRCPATH" ${DEFINE_CMAKE_PREFIX_PATH} \ -DENABLE_MYSQL=OFF \ -DENABLE_TESTING=OFF \ -DENABLE_TESTS=OFF \ - -Wno-dev + -Wno-dev \ + -DUSE_CCACHE=OFF make -j $NPROC tiflash diff --git a/release-centos7/build/env.sh b/release-centos7/build/env.sh new file mode 100644 index 00000000000..8867709d0f8 --- /dev/null +++ b/release-centos7/build/env.sh @@ -0,0 +1,3 @@ +TIFLASH_CI_BUILD_URI_PREFIX="builds/pingcap/tiflash/ci-cache/tmp/pr-build" +TIFLASH_CI_BUILD_PRE_FIX="http://fileserver.pingcap.net/download/${TIFLASH_CI_BUILD_URI_PREFIX}" +CI_BUILD_INFO_PRE_FIX="commit hash value" diff --git a/release-centos7/build/fetch-ci-build.sh b/release-centos7/build/fetch-ci-build.sh new file mode 100755 index 00000000000..1d5d9c3be9f --- /dev/null +++ b/release-centos7/build/fetch-ci-build.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -ueox pipefail + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +source ${SCRIPTPATH}/env.sh + +DOWNLOAD_TAR=${DOWNLOAD_TAR:-false} + +rm -rf "${TAR_PATH}" +mkdir -p "${TAR_PATH}" +cd "${TAR_PATH}" +TAR_BIN_URL="${TIFLASH_CI_BUILD_PRE_FIX}/${PULL_ID}/tiflash.tar.gz" +COMMIT_HASH_FILE_URL="${TIFLASH_CI_BUILD_PRE_FIX}/${PULL_ID}/commit-hash" +while [[ true ]]; do + curl -o ./commit-hash ${COMMIT_HASH_FILE_URL} + if [[ `grep -c "${CI_BUILD_INFO_PRE_FIX}" ./commit-hash` -ne '0' ]];then + COMMIT_HASH_VAL=$(tail -n -1 ./commit-hash) + if [[ ${COMMIT_HASH_VAL} == ${COMMIT_HASH} ]]; then + if [[ ${DOWNLOAD_TAR} == "true" ]]; then + curl -o ./tiflash.tar.gz ${TAR_BIN_URL} + MD5_SUM=$(md5sum ./tiflash.tar.gz | awk '{ print $1 }') + EXPECT_MD5_SUM=$(tail -n +2 ./commit-hash | head -n 1) + if [[ ${MD5_SUM} != ${EXPECT_MD5_SUM} ]]; then + echo "wrong md5sum got ${MD5_SUM} but expect ${EXPECT_MD5_SUM}" + exit -1 + fi + tar -zxf tiflash.tar.gz + fi + break + fi + fi + echo "fail to get ci build tiflash binary, sleep 60s" + sleep 60 +done diff --git a/release-centos7/build/upload-ci-build.sh b/release-centos7/build/upload-ci-build.sh new file mode 100755 index 00000000000..929e8c2ccc4 --- /dev/null +++ b/release-centos7/build/upload-ci-build.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -ueox pipefail + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +SRCPATH=${1:-$(cd $SCRIPTPATH/../..; pwd -P)} +source ${SCRIPTPATH}/env.sh +TAR_BIN_URI="${TIFLASH_CI_BUILD_URI_PREFIX}/${PULL_ID}/tiflash.tar.gz" +COMMIT_HASH_FILE_URI="${TIFLASH_CI_BUILD_URI_PREFIX}/${PULL_ID}/commit-hash" + +build_dir="$SRCPATH/release-centos7" +cd ${build_dir} +tar -zcf tiflash.tar.gz ./tiflash +MD5_SUM=$(md5sum ./tiflash.tar.gz | awk '{ print $1 }') +echo "${CI_BUILD_INFO_PRE_FIX}" > ./commit-hash +echo "${MD5_SUM}" >> ./commit-hash +echo "${COMMIT_HASH}" >> ./commit-hash + +curl -F ${TAR_BIN_URI}=@tiflash.tar.gz http://fileserver.pingcap.net/upload +curl -F ${COMMIT_HASH_FILE_URI}=@commit-hash http://fileserver.pingcap.net/upload diff --git a/release-darwin/build/build-tiflash-release.sh b/release-darwin/build/build-tiflash-release.sh index 30ab2816748..dabea816af6 100755 --- a/release-darwin/build/build-tiflash-release.sh +++ b/release-darwin/build/build-tiflash-release.sh @@ -47,3 +47,7 @@ cp -f "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.dylib" "$install_dir/li FILE="$install_dir/tiflash" otool -L "$FILE" + +set +e +echo "show ccache stats" +ccache -s \ No newline at end of file diff --git a/tests/docker/gtest.yaml b/tests/docker/gtest.yaml index 01bb32d9290..be2edca2a68 100644 --- a/tests/docker/gtest.yaml +++ b/tests/docker/gtest.yaml @@ -3,9 +3,10 @@ version: '2.3' services: # tics-gtest container is for gtest cases tics-gtest: - image: hub.pingcap.net/tiflash/tics:${TAG:-master} + image: hub.pingcap.net/tiflash/tiflash-ci-base volumes: - ./log/tics-gtest:/tmp/tiflash/log - ..:/tests - ../docker/_env.sh:/tests/_env.sh + - ../.build/tiflash:/tiflash entrypoint: sleep infinity # just wait diff --git a/tests/docker/mock-test-dt.yaml b/tests/docker/mock-test-dt.yaml index 70e712b270b..c5770ba7db2 100644 --- a/tests/docker/mock-test-dt.yaml +++ b/tests/docker/mock-test-dt.yaml @@ -3,7 +3,7 @@ version: '2.3' services: # tics0 container is for tests under mutable-test && delta-merge-test directory tics0: - image: hub.pingcap.net/tiflash/tics:${TAG:-master} + image: hub.pingcap.net/tiflash/tiflash-ci-base volumes: - ./config/tics_dt.toml:/config.toml:ro - ./config/users.toml:/users.toml:ro @@ -11,6 +11,9 @@ services: - ./log/tics_dt:/tmp/tiflash/log - ..:/tests - ../docker/_env.sh:/tests/_env.sh - command: + - ../.build/tiflash:/tiflash + entrypoint: + - /tiflash/tiflash + - server - --config-file - /config.toml diff --git a/tests/docker/mock-test-tmt.yaml b/tests/docker/mock-test-tmt.yaml index 6d240db4849..633c4936b6f 100644 --- a/tests/docker/mock-test-tmt.yaml +++ b/tests/docker/mock-test-tmt.yaml @@ -3,7 +3,7 @@ version: '2.3' services: # tics0 container is for tests under mutable-test && delta-merge-test directory tics0: - image: hub.pingcap.net/tiflash/tics:${TAG:-master} + image: hub.pingcap.net/tiflash/tiflash-ci-base volumes: - ./config/tics_tmt.toml:/config.toml:ro - ./config/users.toml:/users.toml:ro @@ -11,6 +11,9 @@ services: - ./log/tics_tmt:/tmp/tiflash/log - ..:/tests - ../docker/_env.sh:/tests/_env.sh - command: + - ../.build/tiflash:/tiflash + entrypoint: + - /tiflash/tiflash + - server - --config-file - /config.toml diff --git a/tests/docker/run.sh b/tests/docker/run.sh index 645d494c9b0..c364ead9886 100644 --- a/tests/docker/run.sh +++ b/tests/docker/run.sh @@ -59,22 +59,18 @@ fi # Stop all docker instances if exist. docker-compose \ - -f gtest.yaml \ -f cluster.yaml \ - -f cluster_new_collation.yaml \ - -f tiflash-dt.yaml \ - -f mock-test-dt.yaml \ - -f cluster_tidb_fail_point.yaml \ + -f tiflash-tagged-image.yaml \ down rm -rf ./data ./log #################################### TIDB-CI ONLY #################################### # run fullstack-tests (for engine DeltaTree) -docker-compose -f cluster.yaml -f tiflash-dt.yaml up -d +docker-compose -f cluster.yaml -f tiflash-tagged-image.yaml up -d wait_env dt -docker-compose -f cluster.yaml -f tiflash-dt.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh tidb-ci/fullstack-test true && ./run-test.sh tidb-ci/fullstack-test-dt' -docker-compose -f cluster.yaml -f tiflash-dt.yaml down +docker-compose -f cluster.yaml -f tiflash-tagged-image.yaml exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh tidb-ci/fullstack-test true && ./run-test.sh tidb-ci/fullstack-test-dt' +docker-compose -f cluster.yaml -f tiflash-tagged-image.yaml down rm -rf ./data ./log [[ "$TIDB_CI_ONLY" -eq 1 ]] && exit diff --git a/tests/docker/tiflash-dt.yaml b/tests/docker/tiflash-dt.yaml index acefd09e5cb..01b5a31664d 100644 --- a/tests/docker/tiflash-dt.yaml +++ b/tests/docker/tiflash-dt.yaml @@ -4,7 +4,7 @@ services: # for tests under fullstack-test directory # (engine DeltaTree) tiflash0: - image: hub.pingcap.net/tiflash/tics:${TAG:-master} + image: hub.pingcap.net/tiflash/tiflash-ci-base volumes: - ./config/tiflash_dt.toml:/config.toml:ro - ./config/tiflash-users.toml:/users.toml:ro @@ -17,7 +17,10 @@ services: - ./config/cipher-file-256:/cipher-file-256:ro - ./data/proxy:/data - ./log/proxy:/log - command: + - ../.build/tiflash:/tiflash + entrypoint: + - /tiflash/tiflash + - server - --config-file - /config.toml restart: on-failure diff --git a/tests/docker/tiflash-tagged-image.yaml b/tests/docker/tiflash-tagged-image.yaml new file mode 100644 index 00000000000..acefd09e5cb --- /dev/null +++ b/tests/docker/tiflash-tagged-image.yaml @@ -0,0 +1,27 @@ +version: '2.3' + +services: + # for tests under fullstack-test directory + # (engine DeltaTree) + tiflash0: + image: hub.pingcap.net/tiflash/tics:${TAG:-master} + volumes: + - ./config/tiflash_dt.toml:/config.toml:ro + - ./config/tiflash-users.toml:/users.toml:ro + - ./data/tiflash:/tmp/tiflash/data + - ./log/tiflash:/tmp/tiflash/log + - ..:/tests + - ../docker/_env.sh:/tests/_env.sh + - ./log/tiflash-cluster-manager:/tmp/tiflash/data/tmp + - ./config/proxy.toml:/proxy.toml:ro + - ./config/cipher-file-256:/cipher-file-256:ro + - ./data/proxy:/data + - ./log/proxy:/log + command: + - --config-file + - /config.toml + restart: on-failure + depends_on: + - "pd0" + - "tikv0" +