Skip to content

Commit

Permalink
re-cherry-pick #2074 #2070 to release-5.0 (#2122)
Browse files Browse the repository at this point in the history
  • Loading branch information
solotzg authored Jun 10, 2021
1 parent 4410e7b commit 3f8dcbd
Show file tree
Hide file tree
Showing 25 changed files with 405 additions and 99 deletions.
72 changes: 72 additions & 0 deletions .ci/build.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
catchError {
def util = load('util.groovy')

def label = "build-tics"

def tiflashTag = ({
def m = ghprbCommentBody =~ /tiflash\s*=\s*([^\s\\]+)(\s|\\|$)/
if (m) {
return "${m.group(1)}"
}
return params.tiflashTag ?: ghprbTargetBranch ?: 'master'
}).call()

podTemplate(name: label, label: label, instanceCap: 5,
workspaceVolume: emptyDirWorkspaceVolume(memory: true),
nodeSelector: 'role_type=slave',
containers: [
containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true),
containerTemplate(name: 'docker', image: 'hub.pingcap.net/zyguan/docker:build-essential',
alwaysPullImage: true, envVars: [
envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375'),
], ttyEnabled: true, command: 'cat'),
containerTemplate(name: 'builder', image: 'hub.pingcap.net/tiflash/tiflash-builder-ci',
alwaysPullImage: true, ttyEnabled: true, command: 'cat',
resourceRequestCpu: '4000m', resourceRequestMemory: '8Gi',
resourceLimitCpu: '10000m', resourceLimitMemory: '30Gi'),
]) {

node(label) {

dir("tics") {
stage("Checkout") {
container("docker") {
sh """
archive_url=${FILE_SERVER_URL}/download/builds/pingcap/tics/cache/tics-repo_latest.tar.gz
if [ ! -d contrib ]; then curl -sL \$archive_url | tar -zx --strip-components=1 || true; fi
"""
sh "chown -R 1000:1000 ./"
// sh "if ! grep -q hub.pingcap.net /etc/hosts ; then echo '172.16.10.5 hub.pingcap.net' >> /etc/hosts; fi"
}
util.checkoutTiCSFull("${ghprbActualCommit}", "${ghprbPullId}")
}
stage("Build & Upload") {
timeout(time: 70, unit: 'MINUTES') {
container("builder") {
sh "NPROC=5 BUILD_BRANCH=${ghprbTargetBranch} release-centos7/build/build-tiflash-ci.sh"
sh "PULL_ID=${ghprbPullId} COMMIT_HASH=${ghprbActualCommit} release-centos7/build/upload-ci-build.sh"
}
}
}
}
}
}
}


stage('Summary') {
def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" +
"${ghprbPullLink}" + "\n" +
"${ghprbPullDescription}" + "\n" +
"Build Result: `${currentBuild.currentResult}`" + "\n" +
"Elapsed Time: `${duration} mins` " + "\n" +
"${env.RUN_DISPLAY_URL}"

echo "${msg}"

if (currentBuild.currentResult != "SUCCESS") {
echo "Send slack here ..."
slackSend channel: '#jenkins-ci', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${msg}"
}
}
26 changes: 11 additions & 15 deletions .ci/integration_test.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,8 @@ catchError {
return params.ghprbTargetBranch ?: 'master'
}).call()

echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}"

stage("Wait for images") {
util.runClosure("wait-for-images") {
timeout(time: 60, unit: 'MINUTES') {
container("docker") {
sh """
while ! docker pull hub.pingcap.net/tiflash/tics:${params.ghprbActualCommit}; do sleep 60; done
"""
}
}
}
stage("Wait for ci build") {
echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}"
}

node("${GO_BUILD_SLAVE}") {
Expand All @@ -33,13 +23,20 @@ catchError {
cp -R /nfs/cache/git/src-tics.tar.gz* ./
mkdir -p ${curws}/tics
tar -xzf src-tics.tar.gz -C ${curws}/tics --strip-components=1
"""
"""
}
}
}
dir("${curws}/tics") {
util.checkoutTiCS("${params.ghprbActualCommit}", "${params.ghprbPullId}")
}
timeout(time: 60, unit: 'MINUTES') {
container("golang") {
sh """
COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=${curws}/tics/tests/.build bash -e ${curws}/tics/release-centos7/build/fetch-ci-build.sh
"""
}
}
}
stash includes: "tics/**", name: "git-code-tics", useDefaultExcludes: false
}
Expand Down Expand Up @@ -74,10 +71,9 @@ catchError {

stage('Summary') {
def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
def msg = "Build Result: `${currentBuild.currentResult}`" + "\n" +
def msg = "Result: `${currentBuild.currentResult}`" + "\n" +
"Elapsed Time: `${duration} mins`" + "\n" +
"${env.RUN_DISPLAY_URL}"

echo "${msg}"

}
97 changes: 63 additions & 34 deletions .ci/util.groovy
Original file line number Diff line number Diff line change
@@ -1,26 +1,30 @@
def doCheckout(commit, refspec) {
checkout(changelog: false, poll: false, scm: [
$class : "GitSCM",
branches : [
[name: "${commit}"],
],
userRemoteConfigs: [
[
url : "git@github.com:pingcap/tics.git",
refspec : refspec,
credentialsId: "github-sre-bot-ssh",
]
],
extensions : [
[$class: 'PruneStaleBranch'],
[$class: 'CleanBeforeCheckout'],
],
])
}

def checkoutTiCS(commit, pullId) {
def refspec = "+refs/heads/*:refs/remotes/origin/*"
if (pullId) {
refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*"
}
try {
checkout(changelog: false, poll: false, scm: [
$class : "GitSCM",
branches : [
[name: "${commit}"],
],
userRemoteConfigs: [
[
url : "git@github.com:pingcap/tics.git",
refspec : refspec,
credentialsId: "github-sre-bot-ssh",
]
],
extensions : [
[$class: 'PruneStaleBranch'],
[$class: 'CleanBeforeCheckout'],
],
])
doCheckout(commit, refspec)
} catch (info) {
retry(2) {
echo "checkout failed, retry.."
Expand All @@ -29,27 +33,42 @@ def checkoutTiCS(commit, pullId) {
echo ".git already exist or not a valid git dir. Delete dir..."
deleteDir()
}
checkout(changelog: false, poll: false, scm: [
$class : "GitSCM",
branches : [
[name: "${commit}"],
],
userRemoteConfigs: [
[
url : "git@github.com:pingcap/tics.git",
refspec : refspec,
credentialsId: "github-sre-bot-ssh",
]
],
extensions : [
[$class: 'PruneStaleBranch'],
[$class: 'CleanBeforeCheckout'],
],
])
doCheckout(commit, refspec)
}
}
}

def checkoutTiCSFull(commit, pullId) {
def refspec = "+refs/heads/*:refs/remotes/origin/*"
if (pullId) {
refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*"
}
checkout(changelog: false, poll: false, scm: [
$class : "GitSCM",
branches : [
[name: "${commit}"],
],
userRemoteConfigs : [
[
url : "git@github.com:pingcap/tics.git",
refspec : refspec,
credentialsId: "github-sre-bot-ssh",
]
],
extensions : [
[$class : 'SubmoduleOption',
disableSubmodules : false,
parentCredentials : true,
recursiveSubmodules: true,
trackingSubmodules : false,
reference : ''],
[$class: 'PruneStaleBranch'],
[$class: 'CleanBeforeCheckout'],
],
doGenerateSubmoduleConfigurations: false,
])
}

def runClosure(label, Closure body) {
podTemplate(name: label, label: label, instanceCap: 15, containers: [
containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true,
Expand All @@ -70,6 +89,16 @@ def runTest(label, testPath, tidbBranch) {
runClosure(label) {
stage("Unstash") {
unstash 'git-code-tics'
dir("tics") {
timeout(time: 5, unit: 'MINUTES') {
container("docker") {
sh """
pwd
DOWNLOAD_TAR=true COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=./tests/.build bash -e release-centos7/build/fetch-ci-build.sh
"""
}
}
}
}
dir(testPath) {
stage("Test") {
Expand Down
17 changes: 14 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ endif ()
# Write compile_commands.json
set(CMAKE_EXPORT_COMPILE_COMMANDS 1)

option (USE_CCACHE "Set to OFF to disable ccache" ON)
include (cmake/find_ccache.cmake)

if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
Expand Down Expand Up @@ -148,14 +149,24 @@ if (NOT ARCH_ARM)
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3")
endif ()

option (DEBUG_WITHOUT_DEBUG_INFO "Set to ON to build dev target without debug info (remove flag `-g` in order to accelerate compiling speed and reduce target binary size)" OFF)
if (DEBUG_WITHOUT_DEBUG_INFO)
string(REPLACE "-g" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
string(REPLACE "-g" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
message (STATUS "Build debug target without debug info")
else ()
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g3 -ggdb3")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g3 -ggdb3")
endif ()

set (CMAKE_BUILD_COLOR_MAKEFILE ON)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")

set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -fno-inline ${CMAKE_C_FLAGS_ADD}")

if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND ARCH_FREEBSD))
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
Expand Down Expand Up @@ -228,7 +239,7 @@ if (UNBUNDLED OR NOT (ARCH_LINUX OR APPLE) OR ARCH_32)
option (NO_WERROR "Disable -Werror compiler option" ON)
endif ()

message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED}")

include(GNUInstallDirs)

Expand Down
19 changes: 12 additions & 7 deletions cmake/find_ccache.cmake
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@

find_program (CCACHE_FOUND ccache)
if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache")
execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION} )
if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache")
execute_process (COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
execute_process (COMMAND ${CCACHE_FOUND} "-p" OUTPUT_VARIABLE CCACHE_CONFIG)
string (REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})

if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
#message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
message (STATUS "Using ccache: ${CCACHE_FOUND}, version ${CCACHE_VERSION}")
message (STATUS "Show ccache config:")
message ("${CCACHE_CONFIG}")
set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
else ()
message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
message (STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
endif ()
else ()
message (STATUS "Not using ccache ${CCACHE_FOUND}, USE_CCACHE=${USE_CCACHE}")
endif ()
9 changes: 9 additions & 0 deletions dbms/src/Common/UInt128.h
Original file line number Diff line number Diff line change
Expand Up @@ -193,16 +193,25 @@ template <> struct is_signed<DB::UInt128>
static constexpr bool value = false;
};

template <>
inline constexpr bool is_signed_v<DB::UInt128> = is_signed<DB::UInt128>::value;

template <> struct is_unsigned<DB::UInt128>
{
static constexpr bool value = true;
};

template <>
inline constexpr bool is_unsigned_v<DB::UInt128> = is_unsigned<DB::UInt128>::value;

template <> struct is_integral<DB::UInt128>
{
static constexpr bool value = true;
};

template <>
inline constexpr bool is_integral_v<DB::UInt128> = is_integral<DB::UInt128>::value;

// Operator +, -, /, *, % aren't implemented so it's not an arithmetic type
template <> struct is_arithmetic<DB::UInt128>
{
Expand Down
4 changes: 4 additions & 0 deletions dbms/src/Core/Types.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ struct numeric_limits<__int128_t>

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#if defined(__clang__)
#pragma GCC diagnostic ignored "-Wunknown-warning-option"
#pragma GCC diagnostic ignored "-Wdeprecated-copy"
#endif
#include <boost/multiprecision/cpp_int.hpp>
#pragma GCC diagnostic pop

Expand Down
12 changes: 10 additions & 2 deletions dbms/src/DataStreams/PKColumnIterator.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@ struct PKColumnIterator : public std::iterator<std::random_access_iterator_tag,

PKColumnIterator & operator=(const PKColumnIterator & itr)
{
pos = itr.pos;
column = itr.column;
copy(itr);
return *this;
}

Expand All @@ -25,10 +24,19 @@ struct PKColumnIterator : public std::iterator<std::random_access_iterator_tag,

PKColumnIterator(const int pos_, const IColumn * column_) : pos(pos_), column(column_) {}

PKColumnIterator(const PKColumnIterator & itr) { copy(itr); }

void operator+=(size_t n) { pos += n; }

size_t pos;
const IColumn * column;

private:
inline void copy(const PKColumnIterator & itr)
{
pos = itr.pos;
column = itr.column;
}
};

template<typename HandleType>
Expand Down
Loading

0 comments on commit 3f8dcbd

Please sign in to comment.