feat: 流水线Job配额管理 TencentBlueKing#5154

sawyersong2 · Sep 17, 2021 · 7d1b7d5 · 7d1b7d5
1 parent c531d40
commit 7d1b7d5
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 13 deletions.
diff --git a/src/backend/ci/core/dispatch/biz-dispatch/build.gradle.kts b/src/backend/ci/core/dispatch/biz-dispatch/build.gradle.kts
@@ -40,6 +40,7 @@ dependencies {
     api(project(":core:common:common-redis"))
     api(project(":core:common:common-db"))
     api(project(":core:common:common-auth:common-auth-api"))
+    api(project(":core:common:common-kafka"))
     api("com.vmware:vijava")
     api("org.json:json")
     api("org.apache.commons:commons-exec")

diff --git a/...z-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaBusinessService.kt b/...z-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaBusinessService.kt
@@ -29,15 +29,19 @@ package com.tencent.devops.dispatch.service
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonInclude
+import com.tencent.devops.common.api.util.JsonUtil
 import com.tencent.devops.common.client.Client
+import com.tencent.devops.common.kafka.KafkaClient
 import com.tencent.devops.common.pipeline.enums.ChannelCode
 import com.tencent.devops.common.redis.RedisLock
 import com.tencent.devops.common.redis.RedisOperation
 import com.tencent.devops.common.service.Profile
 import com.tencent.devops.dispatch.dao.JobQuotaProjectRunTimeDao
 import com.tencent.devops.dispatch.dao.RunningJobsDao
+import com.tencent.devops.dispatch.pojo.JobQuotaHistory
 import com.tencent.devops.dispatch.pojo.JobQuotaStatus
 import com.tencent.devops.dispatch.pojo.enums.JobQuotaVmType
+import com.tencent.devops.model.dispatch.tables.records.TDispatchRunningJobsRecord
 import com.tencent.devops.notify.pojo.EmailNotifyMessage
 import com.tencent.devops.notify.pojo.RtxNotifyMessage
 import com.tencent.devops.process.api.service.ServicePipelineResource
@@ -48,6 +52,7 @@ import org.springframework.scheduling.annotation.Scheduled
 import org.springframework.stereotype.Service
 import java.time.Duration
 import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
 
 @Service
 @Suppress("ALL")
@@ -58,7 +63,8 @@ class JobQuotaBusinessService @Autowired constructor(
     private val jobQuotaProjectRunTimeDao: JobQuotaProjectRunTimeDao,
     private val dslContext: DSLContext,
     private val client: Client,
-    private val profile: Profile
+    private val profile: Profile,
+    private val jobQuotaInterface: JobQuotaInterface
 ) {
 /*    @Value("\${dispatch.jobQuota.systemAlertReceiver:#{null}}")
     private val systemAlertReceiver: String? = null
@@ -184,7 +190,7 @@ class JobQuotaBusinessService @Autowired constructor(
     }*/
 
     private fun jobAgentFinish(projectId: String, buildId: String, vmSeqId: String?) {
-        val redisLock = RedisLock(redisStringHashOperation, JOB_END_LOCK_KEY + projectId, 60L)
+        val redisLock = RedisLock(redisStringHashOperation, "$JOB_END_LOCK_KEY$buildId$vmSeqId", 60L)
         try {
             val lockSuccess = redisLock.tryLock()
             if (lockSuccess) {
@@ -195,7 +201,20 @@ class JobQuotaBusinessService @Autowired constructor(
                     // 保存构建时间，单位秒
                     incProjectJobRunningTime(projectId, JobQuotaVmType.parse(it.vmType), duration.toMillis() / 1000)
                     LOG.info("$projectId|$buildId|$vmSeqId|${JobQuotaVmType.parse(it.vmType)} >> Finish time: " +
-                            "increase ${duration.toHours()} hours. >>>")
+                            "increase ${duration.toMillis() / 1000} seconds. >>>")
+
+                    // 保存构建记录
+                    jobQuotaInterface.saveJobQuotaHistory(
+                        JobQuotaHistory(
+                            projectId = it.projectId,
+                            buildId = it.buildId,
+                            vmSeqId = it.vmSeqId,
+                            executeCount = 1,
+                            vmType = it.vmType,
+                            createTime = it.createdTime.format(dateTimeFormatter),
+                            agentStartTime = it.agentStartTime.format(dateTimeFormatter),
+                            agentFinishTime = LocalDateTime.now().format(dateTimeFormatter)
+                    ))
                 }
             } else {
                 LOG.info("$projectId|$buildId|$vmSeqId >> DeleteRunningJob get lock failed, not run>>>")
@@ -710,7 +729,9 @@ class JobQuotaBusinessService @Autowired constructor(
         private const val TIMEOUT_DAYS = 7L
         private const val CHECK_RUNNING_DAYS = 1L
         private const val QUOTA_PROJECT_ALL_KEY = "project_time_quota_all_key"
+
         private val LOG = LoggerFactory.getLogger(JobQuotaBusinessService::class.java)
+        private val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
     }
 
     @JsonInclude(JsonInclude.Include.NON_NULL)

diff --git a/...iz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaManagerService.kt b/...iz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaManagerService.kt
@@ -81,15 +81,9 @@ class JobQuotaManagerService @Autowired constructor(
             return JobQuotaProject(
                 projectId = projectId,
                 vmType = jobQuotaVmType,
-                runningJobMax = if (isGitCiProject(projectId)) {
-                    systemDefault.runningJobMaxGitCiProject
-                } else { systemDefault.runningJobMaxProject },
-                runningTimeJobMax = if (isGitCiProject(projectId)) {
-                    systemDefault.runningTimeJobMaxGitCi
-                } else { systemDefault.runningTimeJobMax },
-                runningTimeProjectMax = if (isGitCiProject(projectId)) {
-                    systemDefault.runningTimeJobMaxProjectGitCi
-                } else { systemDefault.runningTimeJobMaxProject },
+                runningJobMax = systemDefault.runningJobMaxProject,
+                runningTimeJobMax = systemDefault.runningTimeJobMax,
+                runningTimeProjectMax = systemDefault.runningTimeJobMaxProject,
                 createdTime = now,
                 updatedTime = now,
                 operator = ""

diff --git a/src/backend/ci/core/dispatch/boot-dispatch/build.gradle.kts b/src/backend/ci/core/dispatch/boot-dispatch/build.gradle.kts
@@ -26,7 +26,7 @@
  */
 
 dependencies {
-    api(project(":core:dispatch:biz-dispatch"))
+    api(project(":core:dispatch:biz-dispatch-sample"))
     api(project(":core:common:common-auth:common-auth-mock"))
     api(project(":core:common:common-auth:common-auth-blueking"))
     api(project(":core:common:common-auth:common-auth-v3"))

diff --git a/src/backend/ci/settings.gradle.kts b/src/backend/ci/settings.gradle.kts
@@ -176,6 +176,7 @@ include(":core:process:plugin-trigger")
 include(":core:dispatch")
 include(":core:dispatch:api-dispatch")
 include(":core:dispatch:biz-dispatch")
+include(":core:dispatch:biz-dispatch-sample")
 include(":core:dispatch:boot-dispatch")
 include(":core:dispatch:model-dispatch")