From b0c930385a5f224ddd9a897f03ea8a5690fe70bd Mon Sep 17 00:00:00 2001
From: sawyersong2 <wudiduomjimo@gmail.com>
Date: Thu, 23 Sep 2021 14:38:30 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B5=81=E6=B0=B4=E7=BA=BFJob=E9=85=8D?=
 =?UTF-8?q?=E9=A2=9D=E7=AE=A1=E7=90=86=20#5154?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../dispatch.sdk/service/JobQuotaService.kt   | 26 ++++++++++++++++++-
 .../service/JobQuotaBusinessService.kt        |  2 +-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/JobQuotaService.kt b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/JobQuotaService.kt
index 0b1e50c74ea..4d428282df6 100644
--- a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/JobQuotaService.kt
+++ b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/JobQuotaService.kt
@@ -32,8 +32,10 @@ import com.tencent.devops.common.client.Client
 import com.tencent.devops.common.dispatch.sdk.BuildFailureException
 import com.tencent.devops.common.dispatch.sdk.DispatchSdkErrorCode
 import com.tencent.devops.common.log.utils.BuildLogPrinter
+import com.tencent.devops.common.service.utils.SpringContextUtil
 import com.tencent.devops.dispatch.api.ServiceJobQuotaBusinessResource
 import com.tencent.devops.dispatch.pojo.enums.JobQuotaVmType
+import com.tencent.devops.process.engine.common.VMUtils
 import com.tencent.devops.process.pojo.mq.PipelineAgentStartupEvent
 import com.tencent.devops.process.pojo.mq.PipelineBuildLessStartupDispatchEvent
 import org.slf4j.LoggerFactory
@@ -45,6 +47,8 @@ class JobQuotaService constructor(
 ) {
     companion object {
         private val logger = LoggerFactory.getLogger(JobQuotaService::class.java)
+        private const val RETRY_TIME = 5
+        private const val RETRY_DELTA = 60 * 1000
     }
 
     @Value("\${dispatch.jobQuota.enable:false}")
@@ -67,7 +71,11 @@ class JobQuotaService constructor(
                 vmType = vmType
             )
 
-            if (checkResult != null && !checkResult) {
+            if (checkResult == null || checkResult) {
+                return
+            }
+
+            if (!checkResult && startupEvent.retryTime > RETRY_TIME) {
                 logger.error("$projectId|$vmType|$buildId|$vmSeqId|$executeCount Job quota excess.")
                 throw BuildFailureException(
                     errorType = ErrorType.USER,
@@ -75,6 +83,22 @@ class JobQuotaService constructor(
                     formatErrorMessage = "JOB配额超限",
                     errorMessage = "JOB配额超限"
                 )
+            } else {
+                logger.info("$projectId|$vmType|$buildId|$vmSeqId|$executeCount Job quota excess. delay: $RETRY_DELTA " +
+                        "and retry. retryTime: ${startupEvent.retryTime}")
+
+                buildLogPrinter.addYellowLine(
+                    buildId = buildId,
+                    message = "当前项目下正在执行的【${vmType.displayName}】JOB数量已经达到配额最大值，" +
+                            "将延迟 ${RETRY_DELTA / 1000}s 后重试，当前重试次数：${startupEvent.retryTime}",
+                    tag = VMUtils.genStartVMTaskId(containerId),
+                    jobId = containerHashId,
+                    executeCount = executeCount ?: 1
+                )
+
+                startupEvent.retryTime += 1
+                startupEvent.delayMills = RETRY_DELTA
+                SpringContextUtil.getBean(DispatchService::class.java).redispatch(startupEvent)
             }
 
             logger.info("$projectId|$vmType|$buildId|$vmSeqId|$executeCount Check job quota success.")
diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaBusinessService.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaBusinessService.kt
index 1fa77099784..66b6745fc11 100644
--- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaBusinessService.kt
+++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaBusinessService.kt
@@ -202,7 +202,7 @@ class JobQuotaBusinessService @Autowired constructor(
 
         with(jobStatus) {
             if (runningJobCount >= jobQuota) {
-                buildLogPrinter.addRedLine(
+                buildLogPrinter.addYellowLine(
                     buildId = buildId,
                     message = "当前项目下正在执行的【${vmType.displayName}】JOB数量已经达到配额最大值，" +
                             "正在执行JOB数量：$runningJobCount, 配额: $jobQuota",