feat:数据库表同步projectId数据 TencentBlueKing#5339

src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/pojo/process/PipelineBuildDetail.kt

@@ -0,0 +1,39 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.misc.pojo.process
+
+import io.swagger.annotations.ApiModel
+import io.swagger.annotations.ApiModelProperty
+
+@ApiModel("构建详情")
+data class PipelineBuildDetail(
+    @ApiModelProperty("构建Id", required = true)
+    val buildId: String,
+    @ApiModelProperty("流水线模型信息", required = true)
+    val modelInfo: Map<String, Any>
+)

src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/pojo/process/PipelineModel.kt

@@ -0,0 +1,41 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.misc.pojo.process
+
+import io.swagger.annotations.ApiModel
+import io.swagger.annotations.ApiModelProperty
+
+@ApiModel("构建详情")
+data class PipelineModel(
+    @ApiModelProperty("流水线Id", required = true)
+    val pipelineId: String,
+    @ApiModelProperty("版本号", required = true)
+    val version: Int,
+    @ApiModelProperty("流水线模型信息", required = true)
+    val modelInfo: Map<String, Any>
+)

src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/cron/process/PipelineBuildHistoryDataClearJob.kt

@@ -27,6 +27,7 @@
 
 package com.tencent.devops.misc.cron.process
 
+import com.tencent.devops.common.api.util.JsonUtil
 import com.tencent.devops.common.redis.RedisLock
 import com.tencent.devops.common.redis.RedisOperation
 import com.tencent.devops.misc.config.MiscBuildDataClearConfig
@@ -305,6 +306,12 @@ class PipelineBuildHistoryDataClearJob @Autowired constructor(
                 maxBuildNum = maxBuildNum.toInt()
             )
         }
+        // 同步构建时间晚于maxStartTime的记录的项目信息
+        syncBuildProjectData(
+            pipelineId = pipelineId,
+            projectId = projectId,
+            maxStartTime = projectDataClearConfig.maxStartTime
+        )
     }
 
     private fun cleanDeletePipelineData(pipelineId: String, projectId: String) {
@@ -348,6 +355,70 @@ class PipelineBuildHistoryDataClearJob @Autowired constructor(
                     qualityDataClearService.clearBuildData(buildId)
                     artifactoryDataClearService.clearBuildData(buildId)
                     processDataClearService.clearOtherBuildData(projectId, pipelineId, buildId)
+                } else {
+                    // 分库路由需要同步T_PIPELINE_BUILD_DETAIL的项目信息（同步完后再删除该逻辑）
+                    val buildDetail = processMiscService.getPipelineBuildDetailList(listOf(buildId))?.get(0)
+                    if (buildDetail != null) {
+                        processMiscService.updatePipelineBuildDetailProject(
+                            buildId = buildId,
+                            projectId = projectId,
+                            model = JsonUtil.toJson(buildDetail.modelInfo, false)
+                        )
+                    }
+                }
+            }
+            totalHandleNum += DEFAULT_PAGE_SIZE
+        }
+    }
+
+    private fun syncBuildProjectData(
+        pipelineId: String,
+        projectId: String,
+        maxStartTime: LocalDateTime? = null
+    ) {
+        // 同步项目信息
+        val pipelineResourceList = processMiscService.getPipelineResourceList(pipelineId)
+        pipelineResourceList?.forEach { pipelineResource ->
+            processMiscService.updatePipelineResourceProject(
+                pipelineId = pipelineId,
+                version = pipelineResource.version,
+                projectId = projectId,
+                model = JsonUtil.toJson(pipelineResource.modelInfo, false)
+            )
+        }
+        val pipelineResourceVersionList = processMiscService.getPipelineResourceVersionList(pipelineId)
+        pipelineResourceVersionList?.forEach { pipelineResourceVersion ->
+            processMiscService.updatePipelineResourceVersionProject(
+                pipelineId = pipelineId,
+                version = pipelineResourceVersion.version,
+                projectId = projectId,
+                model = JsonUtil.toJson(pipelineResourceVersion.modelInfo, false)
+            )
+        }
+        processMiscService.updateTemplatePipelineProject(pipelineId, projectId)
+        val totalBuildCount = processMiscService.getTotalBuildCount(
+            pipelineId = pipelineId,
+            maxStartTime = maxStartTime,
+            geTimeFlag = true
+        )
+        var totalHandleNum = 0
+        while (totalHandleNum < totalBuildCount) {
+            val pipelineHistoryBuildIdList = processMiscService.getHistoryBuildIdList(
+                pipelineId = pipelineId,
+                totalHandleNum = totalHandleNum,
+                handlePageSize = DEFAULT_PAGE_SIZE,
+                isCompletelyDelete = false,
+                maxStartTime = maxStartTime
+            )
+            if (!pipelineHistoryBuildIdList.isNullOrEmpty()) {
+                // 分库路由需要同步T_PIPELINE_BUILD_DETAIL的项目信息（同步完后再删除该逻辑）
+                val buildDetailList = processMiscService.getPipelineBuildDetailList(pipelineHistoryBuildIdList)
+                buildDetailList?.forEach { buildDetail ->
+                    processMiscService.updatePipelineBuildDetailProject(
+                        buildId = buildDetail.buildId,
+                        projectId = projectId,
+                        model = JsonUtil.toJson(buildDetail.modelInfo, false)
+                    )
                 }
             }
             totalHandleNum += DEFAULT_PAGE_SIZE

src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/process/ProcessDao.kt

@@ -27,11 +27,19 @@
 
 package com.tencent.devops.misc.dao.process
 
+import com.tencent.devops.model.process.tables.TPipelineBuildDetail
 import com.tencent.devops.model.process.tables.TPipelineBuildHisDataClear
 import com.tencent.devops.model.process.tables.TPipelineBuildHistory
 import com.tencent.devops.model.process.tables.TPipelineDataClear
 import com.tencent.devops.model.process.tables.TPipelineInfo
+import com.tencent.devops.model.process.tables.TPipelineResource
+import com.tencent.devops.model.process.tables.TPipelineResourceVersion
+import com.tencent.devops.model.process.tables.TTemplatePipeline
+import com.tencent.devops.model.process.tables.records.TPipelineBuildDetailRecord
 import com.tencent.devops.model.process.tables.records.TPipelineInfoRecord
+import com.tencent.devops.model.process.tables.records.TPipelineResourceRecord
+import com.tencent.devops.model.process.tables.records.TPipelineResourceVersionRecord
+import com.tencent.devops.model.process.tables.records.TTemplatePipelineRecord
 import org.jooq.Condition
 import org.jooq.DSLContext
 import org.jooq.Record
@@ -40,6 +48,7 @@ import org.jooq.impl.DSL
 import org.springframework.stereotype.Repository
 import java.time.LocalDateTime
 
+@Suppress("LongParameterList", "TooManyFunctions")
 @Repository
 class ProcessDao {
 
@@ -144,10 +153,11 @@ class ProcessDao {
         dslContext: DSLContext,
         pipelineId: String,
         maxBuildNum: Int? = null,
-        maxStartTime: LocalDateTime? = null
+        maxStartTime: LocalDateTime? = null,
+        geTimeFlag: Boolean? = null
     ): Long {
         with(TPipelineBuildHistory.T_PIPELINE_BUILD_HISTORY) {
-            val conditions = getQueryBuildHistoryCondition(pipelineId, maxBuildNum, maxStartTime)
+            val conditions = getQueryBuildHistoryCondition(pipelineId, maxBuildNum, maxStartTime, geTimeFlag)
             return dslContext.select(DSL.max(BUILD_NUM))
                 .from(this)
                 .where(conditions)
@@ -158,15 +168,20 @@ class ProcessDao {
     private fun TPipelineBuildHistory.getQueryBuildHistoryCondition(
         pipelineId: String,
         maxBuildNum: Int?,
-        maxStartTime: LocalDateTime?
+        maxStartTime: LocalDateTime?,
+        geTimeFlag: Boolean?
     ): MutableList<Condition> {
         val conditions = mutableListOf<Condition>()
         conditions.add(PIPELINE_ID.eq(pipelineId))
         if (maxBuildNum != null) {
             conditions.add(BUILD_NUM.le(maxBuildNum))
         }
         if (maxStartTime != null) {
-            conditions.add(START_TIME.lt(maxStartTime))
+            if (geTimeFlag != true) {
+                conditions.add(START_TIME.lt(maxStartTime))
+            } else {
+                conditions.add(START_TIME.ge(maxStartTime))
+            }
         }
         return conditions
     }
@@ -178,10 +193,11 @@ class ProcessDao {
         handlePageSize: Int,
         isCompletelyDelete: Boolean,
         maxBuildNum: Int? = null,
-        maxStartTime: LocalDateTime? = null
+        maxStartTime: LocalDateTime? = null,
+        geTimeFlag: Boolean? = null
     ): Result<out Record>? {
         with(TPipelineBuildHistory.T_PIPELINE_BUILD_HISTORY) {
-            val conditions = getQueryBuildHistoryCondition(pipelineId, maxBuildNum, maxStartTime)
+            val conditions = getQueryBuildHistoryCondition(pipelineId, maxBuildNum, maxStartTime, geTimeFlag)
             val baseStep = dslContext.select(BUILD_ID)
                 .from(this)
                 .where(conditions)
@@ -211,4 +227,100 @@ class ProcessDao {
                 .fetch()
         }
     }
+
+    fun getPipelineBuildDetailList(
+        dslContext: DSLContext,
+        buildIdList: List<String>
+    ): Result<TPipelineBuildDetailRecord>? {
+        with(TPipelineBuildDetail.T_PIPELINE_BUILD_DETAIL) {
+            return dslContext.selectFrom(this)
+                .where(BUILD_ID.`in`(buildIdList))
+                .fetch()
+        }
+    }
+
+    fun updatePipelineBuildDetailProject(
+        dslContext: DSLContext,
+        buildId: String,
+        projectId: String,
+        model: String? = null
+    ) {
+        with(TPipelineBuildDetail.T_PIPELINE_BUILD_DETAIL) {
+            val baseStep = dslContext.update(this)
+            if (!model.isNullOrBlank()) {
+                baseStep.set(MODEL, model)
+            }
+            baseStep.set(PROJECT_ID, projectId).where(BUILD_ID.eq(buildId)).execute()
+        }
+    }
+
+    fun getPipelineResourceList(
+        dslContext: DSLContext,
+        pipelineId: String
+    ): Result<TPipelineResourceRecord>? {
+        with(TPipelineResource.T_PIPELINE_RESOURCE) {
+            return dslContext.selectFrom(this)
+                .where(PIPELINE_ID.eq(pipelineId))
+                .fetch()
+        }
+    }
+
+    fun updatePipelineResourceProject(
+        dslContext: DSLContext,
+        pipelineId: String,
+        version: Int,
+        projectId: String,
+        model: String? = null
+    ) {
+        with(TPipelineResource.T_PIPELINE_RESOURCE) {
+            val baseStep = dslContext.update(this)
+            if (!model.isNullOrBlank()) {
+                baseStep.set(MODEL, model)
+            }
+            baseStep.set(PROJECT_ID, projectId)
+                .where(PIPELINE_ID.eq(pipelineId).and(VERSION.eq(version)))
+                .execute()
+        }
+    }
+
+    fun getPipelineResourceVersionList(
+        dslContext: DSLContext,
+        pipelineId: String
+    ): Result<TPipelineResourceVersionRecord>? {
+        with(TPipelineResourceVersion.T_PIPELINE_RESOURCE_VERSION) {
+            return dslContext.selectFrom(this)
+                .where(PIPELINE_ID.eq(pipelineId))
+                .fetch()
+        }
+    }
+
+    fun updatePipelineResourceVersionProject(
+        dslContext: DSLContext,
+        pipelineId: String,
+        version: Int,
+        projectId: String,
+        model: String? = null
+    ) {
+        with(TPipelineResourceVersion.T_PIPELINE_RESOURCE_VERSION) {
+            val baseStep = dslContext.update(this)
+            if (!model.isNullOrBlank()) {
+                baseStep.set(MODEL, model)
+            }
+            baseStep.set(PROJECT_ID, projectId)
+                .where(PIPELINE_ID.eq(pipelineId).and(VERSION.eq(version)))
+                .execute()
+        }
+    }
+
+    fun updateTemplatePipelineProject(
+        dslContext: DSLContext,
+        pipelineId: String,
+        projectId: String
+    ) {
+        with(TTemplatePipeline.T_TEMPLATE_PIPELINE) {
+            dslContext.update(this).set(PROJECT_ID, projectId)
+                .where(PIPELINE_ID.eq(pipelineId))
+                .execute()
+        }
+    }
 }