pingcap · ti-srebot · Jun 3, 2021 · May 28, 2021 · Jun 2, 2021 · Jun 2, 2021
diff --git a/.ci/build.groovy b/.ci/build.groovy
@@ -0,0 +1,72 @@
+catchError {
+    def util = load('util.groovy')
+
+    def label = "build-tics"
+
+    def tiflashTag = ({
+        def m = ghprbCommentBody =~ /tiflash\s*=\s*([^\s\\]+)(\s|\\|$)/
+        if (m) {
+            return "${m.group(1)}"
+        }
+        return params.tiflashTag ?: ghprbTargetBranch ?: 'master'
+    }).call()
+
+    podTemplate(name: label, label: label, instanceCap: 5,
+            workspaceVolume: emptyDirWorkspaceVolume(memory: true),
+            nodeSelector: 'role_type=slave',
+            containers: [
+                    containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true),
+                    containerTemplate(name: 'docker', image: 'hub.pingcap.net/zyguan/docker:build-essential',
+                            alwaysPullImage: true, envVars: [
+                            envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375'),
+                    ], ttyEnabled: true, command: 'cat'),
+                    containerTemplate(name: 'builder', image: 'hub.pingcap.net/tiflash/tiflash-builder-ci',
+                            alwaysPullImage: true, ttyEnabled: true, command: 'cat',
+                            resourceRequestCpu: '4000m', resourceRequestMemory: '8Gi',
+                            resourceLimitCpu: '10000m', resourceLimitMemory: '30Gi'),
+            ]) {
+
+        node(label) {
+
+            dir("tics") {
+                stage("Checkout") {
+                    container("docker") {
+                        sh """
+                        archive_url=${FILE_SERVER_URL}/download/builds/pingcap/tics/cache/tics-repo_latest.tar.gz
+                        if [ ! -d contrib ]; then curl -sL \$archive_url | tar -zx --strip-components=1 || true; fi
+                        """
+                        sh "chown -R 1000:1000 ./"
+                        // sh "if ! grep -q hub.pingcap.net /etc/hosts ; then echo '172.16.10.5 hub.pingcap.net' >> /etc/hosts; fi"
+                    }
+                    util.checkoutTiCSFull("${ghprbActualCommit}", "${ghprbPullId}")
+                }
+                stage("Build & Upload") {
+                    timeout(time: 70, unit: 'MINUTES') {
+                        container("builder") {
+                            sh "NPROC=5 BUILD_BRANCH=${ghprbTargetBranch} release-centos7/build/build-tiflash-ci.sh"
+                            sh "PULL_ID=${ghprbPullId} COMMIT_HASH=${ghprbActualCommit} release-centos7/build/upload-ci-build.sh"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+stage('Summary') {
+    def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
+    def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" +
+            "${ghprbPullLink}" + "\n" +
+            "${ghprbPullDescription}" + "\n" +
+            "Build Result: `${currentBuild.currentResult}`" + "\n" +
+            "Elapsed Time: `${duration} mins` " + "\n" +
+            "${env.RUN_DISPLAY_URL}"
+
+    echo "${msg}"
+
+    if (currentBuild.currentResult != "SUCCESS") {
+        echo "Send slack here ..."
+        slackSend channel: '#jenkins-ci', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${msg}"
+    }
+}
diff --git a/.ci/integration_test.groovy b/.ci/integration_test.groovy
@@ -9,18 +9,8 @@ catchError {
         return params.ghprbTargetBranch ?: 'master'
     }).call()
 
-    echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}"
-
-    stage("Wait for images") {
-        util.runClosure("wait-for-images") {
-            timeout(time: 60, unit: 'MINUTES') {
-                container("docker") {
-                    sh  """
-                        while ! docker pull hub.pingcap.net/tiflash/tics:${params.ghprbActualCommit}; do sleep 60; done
-                        """
-                }
-            }
-        }
+    stage("Wait for ci build") {
+        echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}"
     }
 
     node("${GO_BUILD_SLAVE}") {
@@ -33,13 +23,20 @@ catchError {
                         cp -R /nfs/cache/git/src-tics.tar.gz*  ./
                         mkdir -p ${curws}/tics
                         tar -xzf src-tics.tar.gz -C ${curws}/tics --strip-components=1
-                    """
+                        """
                     }
                 }
             }
             dir("${curws}/tics") {
                 util.checkoutTiCS("${params.ghprbActualCommit}", "${params.ghprbPullId}")
             }
+            timeout(time: 60, unit: 'MINUTES') {
+                container("golang") {
+                    sh  """
+                        COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=${curws}/tics/tests/.build bash -e ${curws}/tics/release-centos7/build/fetch-ci-build.sh
+                        """
+                }
+            }
         }
         stash includes: "tics/**", name: "git-code-tics", useDefaultExcludes: false
     }
@@ -74,10 +71,9 @@ catchError {
 
 stage('Summary') {
     def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
-    def msg = "Build Result: `${currentBuild.currentResult}`" + "\n" +
+    def msg = "Result: `${currentBuild.currentResult}`" + "\n" +
             "Elapsed Time: `${duration} mins`" + "\n" +
             "${env.RUN_DISPLAY_URL}"
 
     echo "${msg}"
-
 }
diff --git a/.ci/util.groovy b/.ci/util.groovy
@@ -1,26 +1,30 @@
+def doCheckout(commit, refspec) {
+    checkout(changelog: false, poll: false, scm: [
+        $class           : "GitSCM",
+        branches         : [
+                [name: "${commit}"],
+        ],
+        userRemoteConfigs: [
+                [
+                        url          : "git@github.com:pingcap/tics.git",
+                        refspec      : refspec,
+                        credentialsId: "github-sre-bot-ssh",
+                ]
+        ],
+        extensions       : [
+                [$class: 'PruneStaleBranch'],
+                [$class: 'CleanBeforeCheckout'],
+        ],
+    ])
+}
+
 def checkoutTiCS(commit, pullId) {
     def refspec = "+refs/heads/*:refs/remotes/origin/*"
     if (pullId) {
         refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*"
     }
     try {
-        checkout(changelog: false, poll: false, scm: [
-                $class           : "GitSCM",
-                branches         : [
-                        [name: "${commit}"],
-                ],
-                userRemoteConfigs: [
-                        [
-                                url          : "git@github.com:pingcap/tics.git",
-                                refspec      : refspec,
-                                credentialsId: "github-sre-bot-ssh",
-                        ]
-                ],
-                extensions       : [
-                        [$class: 'PruneStaleBranch'],
-                        [$class: 'CleanBeforeCheckout'],
-                ],
-        ])
+        doCheckout(commit, refspec)
     } catch (info) {
         retry(2) {
             echo "checkout failed, retry.."
@@ -29,27 +33,42 @@ def checkoutTiCS(commit, pullId) {
                 echo ".git already exist or not a valid git dir. Delete dir..."
                 deleteDir()
             }
-            checkout(changelog: false, poll: false, scm: [
-                    $class           : "GitSCM",
-                    branches         : [
-                            [name: "${commit}"],
-                    ],
-                    userRemoteConfigs: [
-                            [
-                                    url          : "git@github.com:pingcap/tics.git",
-                                    refspec      : refspec,
-                                    credentialsId: "github-sre-bot-ssh",
-                            ]
-                    ],
-                    extensions       : [
-                            [$class: 'PruneStaleBranch'],
-                            [$class: 'CleanBeforeCheckout'],
-                    ],
-            ])
+            doCheckout(commit, refspec)
         }
     }
 }
 
+def checkoutTiCSFull(commit, pullId) {
+    def refspec = "+refs/heads/*:refs/remotes/origin/*"
+    if (pullId) {
+        refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*"
+    }
+    checkout(changelog: false, poll: false, scm: [
+            $class                           : "GitSCM",
+            branches                         : [
+                    [name: "${commit}"],
+            ],
+            userRemoteConfigs                : [
+                    [
+                            url          : "git@github.com:pingcap/tics.git",
+                            refspec      : refspec,
+                            credentialsId: "github-sre-bot-ssh",
+                    ]
+            ],
+            extensions                       : [
+                    [$class             : 'SubmoduleOption',
+                     disableSubmodules  : false,
+                     parentCredentials  : true,
+                     recursiveSubmodules: true,
+                     trackingSubmodules : false,
+                     reference          : ''],
+                    [$class: 'PruneStaleBranch'],
+                    [$class: 'CleanBeforeCheckout'],
+            ],
+            doGenerateSubmoduleConfigurations: false,
+    ])
+}
+
 def runClosure(label, Closure body) {
     podTemplate(name: label, label: label, instanceCap: 15, containers: [
             containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true,
@@ -70,6 +89,16 @@ def runTest(label, testPath, tidbBranch) {
     runClosure(label) {
         stage("Unstash") {
             unstash 'git-code-tics'
+            dir("tics") {
+                timeout(time: 5, unit: 'MINUTES') {
+                    container("docker") {
+                        sh """
+                        pwd
+                        DOWNLOAD_TAR=true COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=./tests/.build bash -e release-centos7/build/fetch-ci-build.sh
+                        """
+                    }
+                }
+            }
         }
         dir(testPath) {
             stage("Test") {

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -22,6 +22,7 @@ endif ()
 # Write compile_commands.json
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 
+option (USE_CCACHE "Set to OFF to disable ccache" ON)
 include (cmake/find_ccache.cmake)
 
 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@@ -148,14 +149,24 @@ if (NOT ARCH_ARM)
     set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3")
 endif ()
 
+option (DEBUG_WITHOUT_DEBUG_INFO "Set to ON to build dev target without debug info (remove flag `-g` in order to accelerate compiling speed and reduce target binary size)" OFF)
+if (DEBUG_WITHOUT_DEBUG_INFO)
+    string(REPLACE "-g" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
+    string(REPLACE "-g" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
+    message (STATUS "Build debug target without debug info")
+else ()
+    set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g3 -ggdb3")
+    set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g3 -ggdb3")
+endif ()
+
 set (CMAKE_BUILD_COLOR_MAKEFILE          ON)
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${CMAKE_CXX_FLAGS_ADD}")
-set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
 
 set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -fno-inline ${CMAKE_C_FLAGS_ADD}")
 
 if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND ARCH_FREEBSD))
     set (CMAKE_EXE_LINKER_FLAGS          "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
@@ -228,7 +239,7 @@ if (UNBUNDLED OR NOT (ARCH_LINUX OR APPLE) OR ARCH_32)
     option (NO_WERROR "Disable -Werror compiler option" ON)
 endif ()
 
-message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
+message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED}")
 
 include(GNUInstallDirs)
 

diff --git a/cmake/find_ccache.cmake b/cmake/find_ccache.cmake
@@ -1,14 +1,19 @@
 
 find_program (CCACHE_FOUND ccache)
-if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache")
-    execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
-    string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION} )
+if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache")
+    execute_process (COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
+    execute_process (COMMAND ${CCACHE_FOUND} "-p" OUTPUT_VARIABLE CCACHE_CONFIG)
+    string (REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
 
     if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-       #message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
-       set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
-       set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+        message (STATUS "Using ccache: ${CCACHE_FOUND}, version ${CCACHE_VERSION}")
+        message (STATUS "Show ccache config:")
+        message ("${CCACHE_CONFIG}")
+        set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+        set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
     else ()
-       message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
+        message (STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
     endif ()
+else ()
+    message (STATUS "Not using ccache ${CCACHE_FOUND}, USE_CCACHE=${USE_CCACHE}")
 endif ()