enhance: refine the pipeline (milvus-io#37412)

1. enhance: allow to pass conan repo url 2. enhance: add throttle for cpu-e2e and cpp-unit-test pipeline --------- Signed-off-by: Yellow Shine <sammy.huang@zilliz.com>
tedxu · Nov 6, 2024 · b3de4b0 · b3de4b0
1 parent b2c36ef
commit b3de4b0
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 7 deletions.
diff --git a/ci/jenkins/Nightly2.groovy b/ci/jenkins/Nightly2.groovy
@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@v0.62.0') _
+@Library('jenkins-shared-library@v0.67.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 

diff --git a/ci/jenkins/PR-for-go-sdk.groovy b/ci/jenkins/PR-for-go-sdk.groovy
@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@v0.62.0') _
+@Library('jenkins-shared-library@v0.67.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 
@@ -11,6 +11,13 @@ pipeline {
         buildDiscarder logRotator(artifactDaysToKeepStr: '30')
         preserveStashes(buildCount: 5)
         disableConcurrentBuilds(abortPrevious: true)
+        timeout(time: 6, unit: 'HOURS')
+        throttleJobProperty(
+            categories: ['go-sdk'],
+            throttleEnabled: true,
+            throttleOption: 'category'
+
+        )
     }
     agent {
         kubernetes {

diff --git a/ci/jenkins/PR.groovy b/ci/jenkins/PR.groovy
@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@v0.62.0') _
+@Library('jenkins-shared-library@v0.67.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 def milvus_helm_chart_version = '4.2.8'
@@ -10,6 +10,13 @@ pipeline {
         buildDiscarder logRotator(artifactDaysToKeepStr: '30')
         preserveStashes(buildCount: 5)
         disableConcurrentBuilds(abortPrevious: true)
+        timeout(time: 6, unit: 'HOURS')
+        throttleJobProperty(
+            categories: ['cpu-e2e'],
+            throttleEnabled: true,
+            throttleOption: 'category'
+
+        )
     }
     agent {
         kubernetes {

diff --git a/ci/jenkins/UT-CPP.groovy b/ci/jenkins/UT-CPP.groovy
@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@v0.63.0') _
+@Library('jenkins-shared-library@v0.67.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 def milvus_helm_chart_version = '4.2.8'
@@ -10,6 +10,13 @@ pipeline {
         buildDiscarder logRotator(artifactDaysToKeepStr: '30')
         preserveStashes(buildCount: 5)
         disableConcurrentBuilds(abortPrevious: true)
+        timeout(time: 6, unit: 'HOURS')
+        throttleJobProperty(
+            categories: ['cpp-unit-test'],
+            throttleEnabled: true,
+            throttleOption: 'category'
+
+        )
     }
     agent {
         kubernetes {
@@ -29,7 +36,7 @@ pipeline {
                 }
             }
         }
-        stage('build') {
+        stage('build & test') {
             steps {
                 container('tkn') {
                     script {
@@ -40,7 +47,9 @@ pipeline {
                                               pullRequestNumber: "$env.CHANGE_ID",
                                               make_cmd: "make clean && make USE_ASAN=ON build-cpp-with-coverage",
                                               test_entrypoint: "./scripts/run_cpp_codecov.sh",
-                                              codecov_files: "./lcov_output.info"
+                                              codecov_report_name: "cpp-unit-test",
+                                              codecov_files: "./lcov_output.info",
+                                              tekton_pipeline_timeout: '3h'
                     }
                 }
             }

diff --git a/scripts/3rdparty_build.sh b/scripts/3rdparty_build.sh
@@ -52,9 +52,14 @@ pushd ${BUILD_OUTPUT_DIR}
 export CONAN_REVISIONS_ENABLED=1
 export CXXFLAGS="-Wno-error=address -Wno-error=deprecated-declarations"
 export CFLAGS="-Wno-error=address -Wno-error=deprecated-declarations"
+
+# Determine the Conan remote URL, using the environment variable if set, otherwise defaulting
+CONAN_ARTIFACTORY_URL="${CONAN_ARTIFACTORY_URL:-https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local}"
+
 if [[ ! `conan remote list` == *default-conan-local* ]]; then
-    conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
+    conan remote add default-conan-local $CONAN_ARTIFACTORY_URL
 fi
+
 unameOut="$(uname -s)"
 case "${unameOut}" in
   Darwin*)