Merge branch 'master' of https://github.com/Tencent/bk-ci into issue5…

…154_github_branch # Conflicts: # src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt
sawyersong2 · Oct 13, 2021 · 4ba5e2f · 4ba5e2f
2 parents 38bc1a9 + 47c87a8
commit 4ba5e2f
Show file tree

Hide file tree

Showing 324 changed files with 6,126 additions and 2,411 deletions.
diff --git a/scripts/bkenv.properties b/scripts/bkenv.properties
@@ -37,6 +37,10 @@ BK_SSM_PORT=
 BK_TURBO_PRIVATE_URL=http://bk-turbo.service.consul
 # BK_TURBO_PUBLIC_URL默认为http://turbo.$BK_DOMAIN. 按需修改. turbo网关
 BK_TURBO_PUBLIC_URL=http://turbo.$BK_DOMAIN
+# BK_CI_WEWORK_HOST 默认值为：https://qyapi.weixin.qq.com, 企业微信服务端host，用于调用企业微信api接口
+BK_CI_WEWORK_HOST=https://qyapi.weixin.qq.com
+# BK_CI_WEWORK_API_URL 默认值为：https://qyapi.weixin.qq.com, 企业微信服务端host，用于调用企业微信api接口
+BK_CI_WEWORK_API_URL=
 
 ##########
 # 1-基础配置
@@ -203,6 +207,10 @@ BK_CI_REPOSITORY_SVN_WEBHOOK_URL=
 BK_CI_STORE_USER_AVATARS_URL=$BK_PAAS_PUBLIC_URL/console/static/img/getheadimg.jpg?
 # BK_CI_FQDN_CERT BKCI站点的HTTPS证书存储位置, 默认值为空表示没有开启HTTPS，用于Agent与BKCI走HTTPS通信使用
 BK_CI_FQDN_CERT=
+# BK_CI_NOTIFY_WEWORK_API_URL 默认值为：https://qyapi.weixin.qq.com, 企业微信服务端host，用于调用企业微信api接口.
+BK_CI_NOTIFY_WEWORK_API_URL=https://qyapi.weixin.qq.com
+# BK_CI_NOTIFY_WEWORK_SEND_CHANNEL 发送rtx的通道: weworkAgent 企业微信应用,blueking 蓝鲸消息
+BK_CI_NOTIFY_WEWORK_SEND_CHANNEL=weworkAgent
 ##########
 # 4-微服务依赖
 ##########

diff --git a/src/backend/ci/boot-assembly/build.gradle.kts b/src/backend/ci/boot-assembly/build.gradle.kts
@@ -47,6 +47,7 @@ dependencies {
     implementation(project(":core:log:biz-log-sample"))
     implementation(project(":core:misc:biz-misc-sample"))
     implementation(project(":core:notify:biz-notify-blueking"))
+    implementation(project(":core:notify:biz-notify-wework"))
     implementation(project(":core:openapi:biz-openapi"))
     implementation(project(":core:plugin:biz-plugin"))
     implementation(project(":core:process:plugin-load"))

diff --git a/src/backend/ci/build.gradle.kts b/src/backend/ci/build.gradle.kts
@@ -23,7 +23,7 @@ allprojects {
     group = "com.tencent.bk.devops.ci"
     // 版本
     version = (System.getProperty("ci_version") ?: "1.6.0") +
-            if (System.getProperty("snapshot") == "true") "-SNAPSHOT" else "-RELEASE"
+        if (System.getProperty("snapshot") == "true") "-SNAPSHOT" else "-RELEASE"
 
     // 版本管理
     dependencyManagement {
@@ -32,7 +32,7 @@ allprojects {
             dependency("org.mockito:mockito-all:${Versions.Mockito}")
             dependency("com.nhaarman:mockito-kotlin-kt1.1:${Versions.MockitoKt}")
             dependency("javax.ws.rs:javax.ws.rs-api:${Versions.Jaxrs}")
-            dependency("org.bouncycastle:bcprov-jdk16:${Versions.BouncyCastle}")
+            dependency("org.bouncycastle:bcprov-jdk15on:${Versions.BouncyCastle}")
             dependency("com.github.fge:json-schema-validator:${Versions.JsonSchema}")
             dependency("org.apache.commons:commons-exec:${Versions.CommonExec}")
             dependency("com.vdurmont:emoji-java:${Versions.EmojiJava}")
@@ -83,6 +83,9 @@ allprojects {
                 entry("poi")
                 entry("poi-ooxml")
             }
+            dependencySet("com.github.taptap:${Versions.PinyinPlus}") {
+                entry("pinyin-plus")
+            }
         }
     }
 

diff --git a/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt b/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt
@@ -33,4 +33,5 @@ object Versions {
     const val Elasticsearch = "7.4.0"
     const val Lucene = "8.2.0"
     const val Feign = "11.6" // TODO 修复这个BUG:https://github.com/OpenFeign/feign/pull/1347 , 后续SpringCloud升级后可以去掉
+    const val PinyinPlus = "1.0"
 }
diff --git a/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-shadow-jar.gradle.kts b/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-shadow-jar.gradle.kts
@@ -38,7 +38,7 @@ tasks {
         }
 
         manifest {
-            attributes(mapOf("WorkerAgent-Version" to project.version))
+            attributes(mapOf("Implementation-Version" to project.version))
         }
     }
 

diff --git a/.../src/main/kotlin/com/tencent/devops/artifactory/resources/UserPipelineFileResourceImpl.kt b/.../src/main/kotlin/com/tencent/devops/artifactory/resources/UserPipelineFileResourceImpl.kt
@@ -74,7 +74,10 @@ class UserPipelineFileResourceImpl @Autowired constructor(
         path: String
     ): Result<Url> {
         val urls = archiveFileService.getFileDownloadUrls(
-            fileChannelType = FileChannelTypeEnum.WEB_DOWNLOAD, filePath = path, artifactoryType = artifactoryType
+            fileChannelType = FileChannelTypeEnum.WEB_DOWNLOAD,
+            filePath = path,
+            artifactoryType = artifactoryType,
+            fullUrl = false
         )
         return Result(Url(urls.fileUrlList[0], urls.fileUrlList[0]))
     }

diff --git a/...in/kotlin/com/tencent/devops/artifactory/service/SamplePipelineBuildArtifactoryService.kt b/...in/kotlin/com/tencent/devops/artifactory/service/SamplePipelineBuildArtifactoryService.kt
@@ -99,15 +99,6 @@ class SamplePipelineBuildArtifactoryService @Autowired constructor(
         return fileInfoList
     }
 
-    override fun synArtifactoryInfo(
-        userId: String,
-        artifactList: List<FileInfo>,
-        projectId: String,
-        pipelineId: String,
-        buildId: String,
-        buildNum: Int
-    ) = Unit
-
     companion object {
         private val logger = LoggerFactory.getLogger(SamplePipelineBuildArtifactoryService::class.java)
     }

diff --git a/...ory/src/main/kotlin/com/tencent/devops/artifactory/mq/PipelineBuildArtifactoryListener.kt b/...ory/src/main/kotlin/com/tencent/devops/artifactory/mq/PipelineBuildArtifactoryListener.kt
@@ -27,90 +27,74 @@
 
 package com.tencent.devops.artifactory.mq
 
-import com.tencent.devops.artifactory.pojo.FileInfo
-import com.tencent.devops.artifactory.service.PipelineBuildArtifactoryService
-import com.tencent.devops.common.api.util.JsonUtil
-import com.tencent.devops.common.client.Client
 import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher
 import com.tencent.devops.common.event.listener.pipeline.BaseListener
 import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildFinishBroadCastEvent
-import com.tencent.devops.process.api.service.ServicePipelineRuntimeResource
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
 @Suppress("ALL")
 class PipelineBuildArtifactoryListener @Autowired constructor(
-    pipelineEventDispatcher: PipelineEventDispatcher,
-    private val pipelineBuildArtifactoryService: PipelineBuildArtifactoryService,
-    private val client: Client
+    pipelineEventDispatcher: PipelineEventDispatcher
 ) : BaseListener<PipelineBuildFinishBroadCastEvent>(pipelineEventDispatcher) {
 
     companion object {
         private val logger = LoggerFactory.getLogger(PipelineBuildArtifactoryListener::class.java)!!
     }
 
     override fun run(event: PipelineBuildFinishBroadCastEvent) {
-        logger.info("PipelineBuildArtifactoryListener.run, event: $event")
-        val projectId = event.projectId
-        val buildId = event.buildId
-        val pipelineId = event.pipelineId
-
-        val startTime = System.currentTimeMillis()
-        val artifactList: List<FileInfo> = try {
-            pipelineBuildArtifactoryService.getArtifactList(projectId, pipelineId, buildId)
-        } catch (ignored: Throwable) {
-            logger.error("[$pipelineId]|getArtifactList-$buildId exception:", ignored)
-            emptyList()
-        }
-        logCostCall(startTime, buildId)
-        logger.info("[$pipelineId]|getArtifactList-$buildId artifact: ${JsonUtil.toJson(artifactList)}")
-
-        try {
-            if (artifactList.isEmpty()) {
-                return
-            }
-
-            val result = client.get(ServicePipelineRuntimeResource::class).updateArtifactList(
-                userId = event.userId,
-                projectId = projectId,
-                pipelineId = pipelineId,
-                buildId = buildId,
-                artifactoryFileList = artifactList
-            )
-
-            logger.info("[$buildId]|update artifact result: ${result.status} ${result.message}")
-
-            if (result.isOk() && result.data != null) {
-                pipelineBuildArtifactoryService.synArtifactoryInfo(
-                    userId = event.userId,
-                    artifactList = artifactList,
-                    projectId = projectId,
-                    pipelineId = pipelineId,
-                    buildId = buildId,
-                    buildNum = result.data!!.buildNum ?: 0
-                )
-            }
-        } catch (e: Exception) {
-            logger.error("[$buildId| update artifact list fail: ${e.localizedMessage}", e)
-            // rollback
-            client.get(ServicePipelineRuntimeResource::class).updateArtifactList(
-                userId = event.userId,
-                projectId = projectId,
-                pipelineId = pipelineId,
-                buildId = buildId,
-                artifactoryFileList = emptyList()
-            )
-        }
-    }
-
-    fun logCostCall(startTime: Long, buildId: String) {
-        val cost = System.currentTimeMillis() - startTime
-        if (cost > 2000) {
-            logger.warn("$buildId - getArtifactList cost:$cost")
-        } else if (cost > 5000) {
-            logger.error("$buildId - getArtifactList cost:$cost")
-        }
+        return
+//        logger.info("PipelineBuildArtifactoryListener.run, event: $event")
+//        val projectId = event.projectId
+//        val buildId = event.buildId
+//        val pipelineId = event.pipelineId
+//
+//        val startTime = System.currentTimeMillis()
+//        val artifactList: List<FileInfo> = try {
+//            pipelineBuildArtifactoryService.getArtifactList(projectId, pipelineId, buildId)
+//        } catch (ignored: Throwable) {
+//            logger.error("[$pipelineId]|getArtifactList-$buildId exception:", ignored)
+//            emptyList()
+//        }
+//        logger.info("[$pipelineId]|getArtifactList-$buildId artifact: ${JsonUtil.toJson(artifactList)}")
+//
+//        try {
+//            if (artifactList.isEmpty()) {
+//                return
+//            }
+//
+//            val result = client.get(ServicePipelineRuntimeResource::class).updateArtifactList(
+//                userId = event.userId,
+//                projectId = projectId,
+//                pipelineId = pipelineId,
+//                buildId = buildId,
+//                artifactoryFileList = artifactList
+//            )
+//
+//            logger.info("[$buildId]|update artifact result: ${result.status} ${result.message}")
+//
+//            if (result.isOk() && result.data != null) {
+//                pipelineBuildArtifactoryService.synArtifactoryInfo(
+//                    userId = event.userId,
+//                    artifactList = artifactList,
+//                    projectId = projectId,
+//                    pipelineId = pipelineId,
+//                    buildId = buildId,
+//                    buildNum = result.data!!.buildNum ?: 0
+//                )
+//            }
+//        } catch (e: Exception) {
+//            logger.error("[$buildId| update artifact list fail: ${e.localizedMessage}", e)
+//            // rollback
+//            client.get(ServicePipelineRuntimeResource::class).updateArtifactList(
+//                userId = event.userId,
+//                projectId = projectId,
+//                pipelineId = pipelineId,
+//                buildId = buildId,
+//                artifactoryFileList = emptyList()
+//            )
+//        }
     }
 }
diff --git a/...-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/ArchiveFileService.kt b/...-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/ArchiveFileService.kt
@@ -142,16 +142,19 @@ interface ArchiveFileService {
         buildId: String,
         artifactoryType: ArtifactoryType,
         customFilePath: String?,
-        fileChannelType: FileChannelTypeEnum
+        fileChannelType: FileChannelTypeEnum,
+        fullUrl: Boolean = true
     ): GetFileDownloadUrlsResponse
 
     /**
      * 获取仓库指定路径下的文件下载路径列表
+     * [fullUrl]表示是否返回包含域名的全url地址
      */
     fun getFileDownloadUrls(
         filePath: String,
         artifactoryType: ArtifactoryType,
-        fileChannelType: FileChannelTypeEnum
+        fileChannelType: FileChannelTypeEnum,
+        fullUrl: Boolean = true
     ): GetFileDownloadUrlsResponse
 
     /**

diff --git a/...src/main/kotlin/com/tencent/devops/artifactory/service/PipelineBuildArtifactoryService.kt b/...src/main/kotlin/com/tencent/devops/artifactory/service/PipelineBuildArtifactoryService.kt
@@ -33,13 +33,4 @@ import com.tencent.devops.artifactory.pojo.FileInfo
 interface PipelineBuildArtifactoryService {
 
     fun getArtifactList(projectId: String, pipelineId: String, buildId: String): List<FileInfo>
-
-    fun synArtifactoryInfo(
-        userId: String,
-        artifactList: List<FileInfo>,
-        projectId: String,
-        pipelineId: String,
-        buildId: String,
-        buildNum: Int
-    )
 }
diff --git a/...ory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/ArchiveFileServiceImpl.kt b/...ory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/ArchiveFileServiceImpl.kt
@@ -111,11 +111,10 @@ abstract class ArchiveFileServiceImpl : ArchiveFileService {
         disposition: FormDataContentDisposition,
         fileChannelType: FileChannelTypeEnum
     ): String {
-        val path = generateDestPath(fileType, projectId, customFilePath, pipelineId, buildId)
-        val destPath = if (!path.endsWith(disposition.fileName)) {
-            path + fileSeparator + disposition.fileName
+        val destPath = if (customFilePath?.endsWith(disposition.fileName) != true) {
+            (customFilePath ?: "") + fileSeparator + disposition.fileName
         } else {
-            path
+            customFilePath
         }
         val servicePipelineResource = client.get(ServicePipelineResource::class)
         val pipelineName = servicePipelineResource.getPipelineNameByIds(
@@ -177,23 +176,36 @@ abstract class ArchiveFileServiceImpl : ArchiveFileService {
         return flag
     }
 
-    protected fun generateFileDownloadUrl(fileChannelType: FileChannelTypeEnum, destPath: String): String {
-        val urlPrefix = when (fileChannelType) {
+    protected fun generateFileDownloadUrl(
+        fileChannelType: FileChannelTypeEnum,
+        destPath: String,
+        fullUrl: Boolean = true
+    ): String {
+        val urlPrefix = StringBuilder()
+        when (fileChannelType) {
             FileChannelTypeEnum.WEB_SHOW -> {
-                HomeHostUtil.getHost(commonConfig.devopsHostGateway!!) +
-                    "/ms/artifactory/api/user/artifactories/file/download"
+                if (fullUrl) {
+                    urlPrefix.append(HomeHostUtil.getHost(commonConfig.devopsHostGateway!!))
+                }
+                urlPrefix.append("/ms/artifactory/api/user/artifactories/file/download")
             }
             FileChannelTypeEnum.WEB_DOWNLOAD -> {
-                HomeHostUtil.getHost(commonConfig.devopsHostGateway!!) +
-                    "/ms/artifactory/api/user/artifactories/file/download/local"
+                if (fullUrl) {
+                    urlPrefix.append(HomeHostUtil.getHost(commonConfig.devopsHostGateway!!))
+                }
+                urlPrefix.append("/ms/artifactory/api/user/artifactories/file/download/local")
             }
             FileChannelTypeEnum.SERVICE -> {
-                HomeHostUtil.getHost(commonConfig.devopsApiGateway!!) +
-                    "/ms/artifactory/api/service/artifactories/file/download"
+                if (fullUrl) {
+                    urlPrefix.append(HomeHostUtil.getHost(commonConfig.devopsApiGateway!!))
+                }
+                urlPrefix.append("/ms/artifactory/api/service/artifactories/file/download")
             }
             FileChannelTypeEnum.BUILD -> {
-                HomeHostUtil.getHost(commonConfig.devopsBuildGateway!!) +
-                    "/ms/artifactory/api/build/artifactories/file/download"
+                if (fullUrl) {
+                    urlPrefix.append(HomeHostUtil.getHost(commonConfig.devopsBuildGateway!!))
+                }
+                urlPrefix.append("/ms/artifactory/api/build/artifactories/file/download")
             }
         }
         val filePath = URLEncoder.encode("/$destPath", "UTF-8")

diff --git a/...c/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt b/...c/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt
@@ -235,7 +235,8 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor(
         buildId: String,
         artifactoryType: ArtifactoryType,
         customFilePath: String?,
-        fileChannelType: FileChannelTypeEnum
+        fileChannelType: FileChannelTypeEnum,
+        fullUrl: Boolean
     ): GetFileDownloadUrlsResponse {
         val filePath = generateDestPath(
             fileType = artifactoryType.toFileType(),
@@ -246,13 +247,14 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor(
         )
         return getFileDownloadUrls(filePath = "$projectId/${BkRepoUtils.getRepoName(artifactoryType)}/$filePath",
             artifactoryType = artifactoryType,
-            fileChannelType = fileChannelType)
+            fileChannelType = fileChannelType, fullUrl = fullUrl)
     }
 
     override fun getFileDownloadUrls(
         filePath: String,
         artifactoryType: ArtifactoryType,
-        fileChannelType: FileChannelTypeEnum
+        fileChannelType: FileChannelTypeEnum,
+        fullUrl: Boolean
     ): GetFileDownloadUrlsResponse {
         logger.info("getFileDownloadUrls, filePath: $filePath, artifactoryType, $artifactoryType, " +
             "fileChannelType, $fileChannelType")
@@ -272,7 +274,7 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor(
             page = 1,
             pageSize = DOWNLOAD_FILE_URL_LIMIT
         ).map {
-            generateFileDownloadUrl(fileChannelType, "${artifactoryInfo.projectId}/$repoName/${it.fullPath}")
+            generateFileDownloadUrl(fileChannelType, "${artifactoryInfo.projectId}/$repoName/${it.fullPath}", fullUrl)
         }
         if (fileUrls.isEmpty()) {
             throw ErrorCodeException(errorCode = CommonMessageCode.PARAMETER_IS_INVALID, params = arrayOf(filePath))
-Original file line number
+Diff line change
@@ Expand Up / @@ -38,7 +38,7 @@ tasks { @@
             }
             manifest {
-                attributes(mapOf("WorkerAgent-Version" to project.version))
+                attributes(mapOf("Implementation-Version" to project.version))
             }
         }
@@ Expand Down @@