Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: 流水线Job配额管理 #5154 #5333

Merged
merged 4 commits into from
Oct 15, 2021
Merged

feat: 流水线Job配额管理 #5154 #5333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
497c199
feat: 流水线Job配额管理 #5154
sawyersong2 Oct 15, 2021
44228c9
feat: 流水线Job配额管理 #5154
sawyersong2 Oct 15, 2021
867bc3f
feat: 流水线Job配额管理 #5154
sawyersong2 Oct 15, 2021
456bd03
feat: 流水线Job配额管理 #5154
sawyersong2 Oct 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion ...atch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/listener/BuildListener.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,13 @@ interface BuildListener {
logger.warn("Fail to handle the shutdown message - ($event)", t)
} finally {
val jobQuotaService = getJobQuotaService()
jobQuotaService.removeRunningJob(event.projectId, event.buildId, event.vmSeqId, event.executeCount)
jobQuotaService.removeRunningJob(
projectId = event.projectId,
pipelineId = event.pipelineId,
buildId = event.buildId,
vmSeqId = event.vmSeqId,
executeCount = event.executeCount
)
}
}

Expand Down
9 changes: 8 additions & 1 deletion ...tch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/JobQuotaService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,19 @@ class JobQuotaService constructor(
}
}

fun removeRunningJob(projectId: String, buildId: String, vmSeqId: String?, executeCount: Int?) {
fun removeRunningJob(
projectId: String,
pipelineId: String,
buildId: String,
vmSeqId: String?,
executeCount: Int?
) {
if (jobQuotaEnable) {
logger.info("Remove running job to dispatch:[$projectId|$buildId|$vmSeqId]")
try {
client.get(ServiceJobQuotaBusinessResource::class).removeRunningJob(
projectId = projectId,
pipelineId = pipelineId,
buildId = buildId,
vmSeqId = vmSeqId ?: "1",
executeCount = executeCount ?: 1
Expand Down
1 change: 1 addition & 0 deletions ...h-docker/src/main/kotlin/com/tencent/devops/dispatch/docker/listener/AgentLessListener.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ class AgentLessListener @Autowired constructor(
// 不管shutdown成功失败，都要回收配额；这里回收job，将自动累加agent执行时间
jobQuotaService.removeRunningJob(
projectId = pipelineBuildLessDockerAgentShutdownEvent.projectId,
pipelineId = pipelineBuildLessDockerAgentShutdownEvent.pipelineId,
buildId = pipelineBuildLessDockerAgentShutdownEvent.buildId,
vmSeqId = pipelineBuildLessDockerAgentShutdownEvent.vmSeqId,
executeCount = pipelineBuildLessDockerAgentShutdownEvent.executeCount
Expand Down
40 changes: 25 additions & 15 deletions ...dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildJobQuotaBusinessResource.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,19 @@

package com.tencent.devops.dispatch.api

import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BUILD_ID
import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_PIPELINE_ID
import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_PROJECT_ID
import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_VM_SEQ_ID
import com.tencent.devops.common.api.pojo.Result
import io.swagger.annotations.Api
import io.swagger.annotations.ApiOperation
import io.swagger.annotations.ApiParam
import javax.ws.rs.Consumes
import javax.ws.rs.DELETE
import javax.ws.rs.HeaderParam
import javax.ws.rs.POST
import javax.ws.rs.Path
import javax.ws.rs.PathParam
import javax.ws.rs.Produces
import javax.ws.rs.QueryParam
import javax.ws.rs.core.MediaType
Expand All @@ -48,16 +52,19 @@ interface BuildJobQuotaBusinessResource {

@ApiOperation("上报一个Agent启动")
@POST
@Path("/agent/projects/{projectId}/builds/{buildId}/vmSeqs/{vmSeqId}")
@Path("/agent/start")
fun addRunningAgent(
@ApiParam(value = "projectId", required = true)
@PathParam("projectId")
@ApiParam("项目ID", required = true)
@HeaderParam(AUTH_HEADER_DEVOPS_PROJECT_ID)
projectId: String,
@ApiParam(value = "buildId", required = true)
@PathParam("buildId")
@ApiParam(value = "流水线ID", required = true)
@HeaderParam(AUTH_HEADER_DEVOPS_PIPELINE_ID)
pipelineId: String,
@ApiParam(value = "构建ID", required = true)
@HeaderParam(AUTH_HEADER_DEVOPS_BUILD_ID)
buildId: String,
@ApiParam(value = "vmSeqId", required = true)
@PathParam("vmSeqId")
@ApiParam(value = "构建job序号", required = true)
@HeaderParam(AUTH_HEADER_DEVOPS_VM_SEQ_ID)
vmSeqId: String,
@ApiParam(value = "executeCount", required = true)
@QueryParam("executeCount")
Expand All @@ -66,16 +73,19 @@ interface BuildJobQuotaBusinessResource {

@ApiOperation("上报一个Agent结束")
@DELETE
@Path("/agent/projects/{projectId}/builds/{buildId}/vmSeqs/{vmSeqId}")
@Path("/agent/shutdown")
fun removeRunningAgent(
@ApiParam(value = "projectId", required = true)
@PathParam("projectId")
@ApiParam("项目ID", required = true)
@HeaderParam(AUTH_HEADER_DEVOPS_PROJECT_ID)
projectId: String,
@ApiParam(value = "buildId", required = true)
@PathParam("buildId")
@ApiParam(value = "流水线ID", required = true)
@HeaderParam(AUTH_HEADER_DEVOPS_PIPELINE_ID)
pipelineId: String,
@ApiParam(value = "构建ID", required = true)
@HeaderParam(AUTH_HEADER_DEVOPS_BUILD_ID)
buildId: String,
@ApiParam(value = "vmSeqId", required = true)
@PathParam("vmSeqId")
@ApiParam(value = "构建job序号", required = true)
@HeaderParam(AUTH_HEADER_DEVOPS_VM_SEQ_ID)
vmSeqId: String,
@ApiParam(value = "executeCount", required = true)
@QueryParam("executeCount")
Expand Down
5 changes: 4 additions & 1 deletion ...spatch/src/main/kotlin/com/tencent/devops/dispatch/api/ServiceJobQuotaBusinessResource.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,14 @@ interface ServiceJobQuotaBusinessResource {

@ApiOperation("上报一个JOB结束")
@DELETE
@Path("/job/projects/{projectId}/builds/{buildId}/vmSeqs/{vmSeqId}")
@Path("/job/projects/{projectId}/pipelines/{pipelineId}/builds/{buildId}/vmSeqs/{vmSeqId}")
fun removeRunningJob(
@ApiParam(value = "projectId", required = true)
@PathParam("projectId")
projectId: String,
@ApiParam(value = "pipelineId", required = true)
@PathParam("pipelineId")
pipelineId: String,
@ApiParam(value = "buildId", required = true)
@PathParam("buildId")
buildId: String,
Expand Down
6 changes: 4 additions & 2 deletions ...c/main/kotlin/com/tencent/devops/dispatch/controller/BuildJobQuotaBusinessResourceImpl.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,21 +40,23 @@ class BuildJobQuotaBusinessResourceImpl @Autowired constructor(

override fun addRunningAgent(
projectId: String,
pipelineId: String,
buildId: String,
vmSeqId: String,
executeCount: Int
): Result<Boolean> {
jobQuotaBusinessService.updateAgentStartTime(projectId, buildId, vmSeqId, executeCount)
jobQuotaBusinessService.updateAgentStartTime(projectId, pipelineId, buildId, vmSeqId, executeCount)
return Result(true)
}

override fun removeRunningAgent(
projectId: String,
pipelineId: String,
buildId: String,
vmSeqId: String,
executeCount: Int
): Result<Boolean> {
jobQuotaBusinessService.updateRunningTime(projectId, buildId, vmSeqId, executeCount)
jobQuotaBusinessService.updateRunningTime(projectId, pipelineId, buildId, vmSeqId, executeCount)
return Result(true)
}
}
3 changes: 2 additions & 1 deletion ...main/kotlin/com/tencent/devops/dispatch/controller/ServiceJobQuotaBusinessResourceImpl.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,12 @@ class ServiceJobQuotaBusinessResourceImpl @Autowired constructor(

override fun removeRunningJob(
projectId: String,
pipelineId: String,
buildId: String,
vmSeqId: String,
executeCount: Int
): Result<Boolean> {
jobQuotaBusinessService.deleteRunningJob(projectId, buildId, vmSeqId, executeCount)
jobQuotaBusinessService.deleteRunningJob(projectId, pipelineId, buildId, vmSeqId, executeCount)
return Result(true)
}
}
1 change: 1 addition & 0 deletions ...dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/pojo/JobQuotaHistory.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ package com.tencent.devops.dispatch.pojo

data class JobQuotaHistory(
val projectId: String,
val pipelineId: String,
val buildId: String,
val vmSeqId: String,
val executeCount: Int,
Expand Down
9 changes: 7 additions & 2 deletions ...z-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/JobQuotaBusinessService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,19 +101,21 @@ class JobQuotaBusinessService @Autowired constructor(
*/
fun deleteRunningJob(
projectId: String,
pipelineId: String,
buildId: String,
vmSeqId: String,
executeCount: Int
) {
LOG.info("$projectId|$buildId|$vmSeqId|$executeCount >>> update agent startTime.")
jobAgentFinish(projectId, buildId, vmSeqId, executeCount)
jobAgentFinish(projectId, pipelineId, buildId, vmSeqId, executeCount)
}

/**
* agent成功启动时更新
*/
fun updateAgentStartTime(
projectId: String,
pipelineId: String,
buildId: String,
vmSeqId: String,
executeCount: Int
Expand All @@ -127,12 +129,13 @@ class JobQuotaBusinessService @Autowired constructor(
*/
fun updateRunningTime(
projectId: String,
pipelineId: String,
buildId: String,
vmSeqId: String,
executeCount: Int
) {
LOG.info("$projectId|$buildId|$vmSeqId|$executeCount >>> update agent running time.")
jobAgentFinish(projectId, buildId, vmSeqId, executeCount)
jobAgentFinish(projectId, pipelineId, buildId, vmSeqId, executeCount)
}

private fun getProjectRunningJobStatus(projectId: String, vmType: JobQuotaVmType): JobQuotaStatus {
Expand Down Expand Up @@ -264,6 +267,7 @@ class JobQuotaBusinessService @Autowired constructor(

private fun jobAgentFinish(
projectId: String,
pipelineId: String,
buildId: String,
vmSeqId: String,
executeCount: Int
Expand Down Expand Up @@ -293,6 +297,7 @@ class JobQuotaBusinessService @Autowired constructor(
jobQuotaInterface.saveJobQuotaHistory(
JobQuotaHistory(
projectId = projectId,
pipelineId = pipelineId,
buildId = buildId,
vmSeqId = vmSeqId,
executeCount = executeCount,
Expand Down
6 changes: 2 additions & 4 deletions ...er-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/quota/QuotaResourceApi.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ class QuotaResourceApi : AbstractBuildResourceApi(), QuotaApi {
retryCount: Int
): Result<Boolean> {
try {
val path = "/ms/dispatch/api/build/quotas/running/agent/projects/$projectId/" +
"builds/$buildId/vmSeqs/$vmSeqId?executeCount=$executeCount"
val path = "/ms/dispatch/api/build/quotas/running/agent/shutdown?executeCount=$executeCount"
val request = buildDelete(path)
val errorMessage = "Quota上报agent运行结束状态失败"
val responseContent = request(
Expand All @@ -67,8 +66,7 @@ class QuotaResourceApi : AbstractBuildResourceApi(), QuotaApi {
retryCount: Int
): Result<Boolean> {
try {
val path = "/ms/dispatch/api/build/quotas/running/agent/projects/$projectId/" +
"builds/$buildId/vmSeqs/$vmSeqId?executeCount=$executeCount"
val path = "/ms/dispatch/api/build/quotas/running/agent/start?executeCount=$executeCount"
val request = buildPost(path)
val errorMessage = "Quota上报agent开始运行失败"
val responseContent = request(
Expand Down