diff --git a/src/backend/ci/build.gradle.kts b/src/backend/ci/build.gradle.kts index a2e16f1f891..4e440972bfc 100644 --- a/src/backend/ci/build.gradle.kts +++ b/src/backend/ci/build.gradle.kts @@ -23,7 +23,7 @@ allprojects { group = "com.tencent.bk.devops.ci" // 版本 version = (System.getProperty("ci_version") ?: "1.6.0") + - if (System.getProperty("snapshot") == "true") "-SNAPSHOT" else "-RELEASE" + if (System.getProperty("snapshot") == "true") "-SNAPSHOT" else "-RELEASE" // 版本管理 dependencyManagement { @@ -83,6 +83,9 @@ allprojects { entry("poi") entry("poi-ooxml") } + dependencySet("com.hankcs:${Versions.HanLP}") { + entry("hanlp") + } } } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineResource.kt index bc48edb04de..672bf6cc456 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineResource.kt @@ -34,16 +34,16 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.process.engine.pojo.PipelineInfo -import com.tencent.devops.process.pojo.PipelineWithModel import com.tencent.devops.process.pojo.Pipeline import com.tencent.devops.process.pojo.PipelineCopy import com.tencent.devops.process.pojo.PipelineId import com.tencent.devops.process.pojo.PipelineIdInfo import com.tencent.devops.process.pojo.PipelineName +import com.tencent.devops.process.pojo.PipelineWithModel import com.tencent.devops.process.pojo.pipeline.DeployPipelineResult import com.tencent.devops.process.pojo.pipeline.SimplePipeline -import com.tencent.devops.process.pojo.setting.PipelineSetting import com.tencent.devops.process.pojo.setting.PipelineModelAndSetting +import com.tencent.devops.process.pojo.setting.PipelineSetting import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam @@ -473,4 +473,13 @@ interface ServicePipelineResource { @PathParam("pipelineId") pipelineId: String ): Result? + + @ApiOperation("刷新所有流水线名拼音") + @PUT + @Path("/batch/pipeline/pinyin") + fun batchUpdatePipelineNamePinYin( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String + ): Result } diff --git a/src/backend/ci/core/process/biz-base/build.gradle.kts b/src/backend/ci/core/process/biz-base/build.gradle.kts index 859f5e720af..c34dd819847 100644 --- a/src/backend/ci/core/process/biz-base/build.gradle.kts +++ b/src/backend/ci/core/process/biz-base/build.gradle.kts @@ -43,4 +43,5 @@ dependencies { api("mysql:mysql-connector-java") implementation("com.github.ben-manes.caffeine:caffeine") testImplementation(project(":core:common:common-test")) + api("com.hankcs:hanlp") } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt index fd8d5ab9ce2..9e39e67ed4d 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt @@ -432,7 +432,7 @@ class PipelineBuildSummaryDao { if (sortType != null) { val sortTypeField = when (sortType) { PipelineSortType.NAME -> { - t.field("PIPELINE_NAME")!!.asc() + t.field("PIPELINE_NAME_PINYIN")!!.asc() } PipelineSortType.CREATE_TIME -> { t.field("CREATE_TIME")!!.desc() @@ -473,6 +473,7 @@ class PipelineBuildSummaryDao { T_PIPELINE_INFO.MANUAL_STARTUP, T_PIPELINE_INFO.ELEMENT_SKIP, T_PIPELINE_INFO.TASK_COUNT, + T_PIPELINE_INFO.PIPELINE_NAME_PINYIN, T_PIPELINE_SETTING.DESC, T_PIPELINE_SETTING.RUN_LOCK_TYPE, T_PIPELINE_BUILD_SUMMARY.BUILD_NUM, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt index 198b996cc18..105eb7245a1 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt @@ -27,6 +27,8 @@ package com.tencent.devops.process.engine.dao +import com.hankcs.hanlp.HanLP +import com.hankcs.hanlp.dictionary.py.Pinyin import com.tencent.devops.common.api.util.timestampmilli import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.pojo.BuildNo @@ -45,7 +47,6 @@ import org.slf4j.LoggerFactory import org.springframework.stereotype.Repository import java.time.LocalDateTime -@Suppress("ALL") @Repository class PipelineInfoDao { @@ -186,13 +187,15 @@ class PipelineInfoDao { val query = dslContext.selectCount().from(this) .where(PROJECT_ID.`in`(projectIds)) - if (channelCode != null) + if (channelCode != null) { query.and(CHANNEL.eq(channelCode.name)) + } query.and(DELETE.eq(false)).fetchOne(0, Int::class.java)!! } } + @Suppress("unused") fun listAll(dslContext: DSLContext): Result { return with(T_PIPELINE_INFO) { dslContext.selectFrom(this) @@ -279,7 +282,11 @@ class PipelineInfoDao { } } - fun listDeletePipelineIdByProject(dslContext: DSLContext, projectId: String, days: Long?): Result? { + fun listDeletePipelineIdByProject( + dslContext: DSLContext, + projectId: String, + days: Long? + ): Result? { with(T_PIPELINE_INFO) { val conditions = mutableListOf() conditions.add(PROJECT_ID.eq(projectId)) @@ -295,7 +302,13 @@ class PipelineInfoDao { /** * 查找updateTime之前被删除的流水线 */ - fun listDeletePipelineBefore(dslContext: DSLContext, updateTime: LocalDateTime, offset: Int?, limit: Int?): Result? { + @Suppress("unused") + fun listDeletePipelineBefore( + dslContext: DSLContext, + updateTime: LocalDateTime, + offset: Int?, + limit: Int? + ): Result? { with(T_PIPELINE_INFO) { val baseQuery = dslContext.selectFrom(this) .where(DELETE.eq(true)) @@ -432,6 +445,7 @@ class PipelineInfoDao { } } + @Suppress("SpreadOperator") fun listOrderInfoByPipelineIds( dslContext: DSLContext, pipelineIds: List @@ -467,7 +481,7 @@ class PipelineInfoDao { projectIds: Set?, channelCodes: Set? ): Record1? { - val conditions = mutableListOf() + val conditions = mutableListOf() conditions.add(T_PIPELINE_INFO.DELETE.eq(false)) if (projectIds != null && projectIds.isNotEmpty()) { conditions.add(T_PIPELINE_INFO.PROJECT_ID.`in`(projectIds)) @@ -475,7 +489,7 @@ class PipelineInfoDao { if (channelCodes != null && channelCodes.isNotEmpty()) { conditions.add(T_PIPELINE_INFO.CHANNEL.`in`(channelCodes.map { it.name })) } - return dslContext.select(T_PIPELINE_INFO.PROJECT_ID.count()).from(T_PIPELINE_INFO) + return dslContext.select(DSL.count(T_PIPELINE_INFO.PROJECT_ID)).from(T_PIPELINE_INFO) .where(conditions).fetch().first() } @@ -566,7 +580,36 @@ class PipelineInfoDao { } } + fun batchUpdatePipelineNamePinYin(dslContext: DSLContext) { + val limit = 1000 + var offset = 0 + var fetchSize = 0 + do { + with(T_PIPELINE_INFO) { + val fetch = dslContext.select(PIPELINE_ID, PIPELINE_NAME).from(this).orderBy(CREATE_TIME) + .limit(offset, limit).fetch() + val updates = fetch.map { + dslContext.update(this).set(PIPELINE_NAME_PINYIN, nameToPinyin(it[PIPELINE_NAME])) + .where(PIPELINE_ID.eq(it[PIPELINE_ID])) + } + dslContext.batch(updates).execute() + val size = fetch.size + offset += size + fetchSize = size + } + } while (fetchSize == 1000) + } + companion object { private val logger = LoggerFactory.getLogger(PipelineInfoDao::class.java) } + + private fun nameToPinyin(pipelineName: String): String { + return HanLP.convertToPinyinList(pipelineName).asSequence().mapIndexed { index, it -> + // 不属于中文没有拼音 + if (Pinyin.none5 == it) { + pipelineName[index].toString() + } else it.pinyinWithoutTone + }.joinToString("") + } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt index 811be827aed..862470ff4e3 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt @@ -50,9 +50,9 @@ import com.tencent.devops.process.pojo.PipelineWithModel import com.tencent.devops.process.pojo.audit.Audit import com.tencent.devops.process.pojo.pipeline.DeployPipelineResult import com.tencent.devops.process.pojo.pipeline.SimplePipeline -import com.tencent.devops.process.pojo.setting.PipelineSetting import com.tencent.devops.process.pojo.pipeline.enums.PipelineRuleBusCodeEnum import com.tencent.devops.process.pojo.setting.PipelineModelAndSetting +import com.tencent.devops.process.pojo.setting.PipelineSetting import com.tencent.devops.process.service.PipelineInfoFacadeService import com.tencent.devops.process.service.PipelineListFacadeService import com.tencent.devops.process.service.pipeline.PipelineSettingFacadeService @@ -408,6 +408,11 @@ class ServicePipelineResourceImpl @Autowired constructor( return null } + override fun batchUpdatePipelineNamePinYin(userId: String): Result { + pipelineInfoFacadeService.batchUpdatePipelineNamePinYin(userId) + return Result(true) + } + private fun checkParams(userId: String, projectId: String) { checkUserId(userId) checkProjectId(projectId) diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt index dfbe360e104..6da19e2674a 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt @@ -46,12 +46,15 @@ import com.tencent.devops.common.pipeline.extend.ModelCheckPlugin import com.tencent.devops.common.pipeline.pojo.BuildFormProperty import com.tencent.devops.common.pipeline.pojo.BuildNo import com.tencent.devops.common.pipeline.pojo.element.atom.BeforeDeleteParam +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.utils.LogUtils import com.tencent.devops.common.service.utils.MessageCodeUtil import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.constant.ProcessMessageCode.ILLEGAL_PIPELINE_MODEL_JSON import com.tencent.devops.process.constant.ProcessMessageCode.USER_NEED_PIPELINE_X_PERMISSION import com.tencent.devops.process.engine.compatibility.BuildPropertyCompatibilityTools +import com.tencent.devops.process.engine.dao.PipelineInfoDao import com.tencent.devops.process.engine.dao.template.TemplateDao import com.tencent.devops.process.engine.service.PipelineRepositoryService import com.tencent.devops.process.engine.utils.PipelineUtils @@ -93,7 +96,9 @@ class PipelineInfoFacadeService @Autowired constructor( private val modelCheckPlugin: ModelCheckPlugin, private val pipelineBean: PipelineBean, private val processJmxApi: ProcessJmxApi, - private val client: Client + private val client: Client, + private val pipelineInfoDao: PipelineInfoDao, + private val redisOperation: RedisOperation ) { @Value("\${process.deletedPipelineStoreDays:30}") @@ -803,6 +808,19 @@ class PipelineInfoFacadeService @Autowired constructor( ) } + fun batchUpdatePipelineNamePinYin(userId: String) { + logger.info("$userId batchUpdatePipelineNamePinYin") + val redisLock = RedisLock(redisOperation, "process:batchUpdatePipelineNamePinYin", 5 * 60) + if (redisLock.tryLock()) { + try { + pipelineInfoDao.batchUpdatePipelineNamePinYin(dslContext) + } finally { + redisLock.unlock() + logger.info("$userId batchUpdatePipelineNamePinYin finished") + } + } + } + companion object { private val logger = LoggerFactory.getLogger(PipelineInfoFacadeService::class.java) } diff --git a/support-files/sql/1001_ci_process_ddl_mysql.sql b/support-files/sql/1001_ci_process_ddl_mysql.sql index ebf15a374f4..f260e50f889 100644 --- a/support-files/sql/1001_ci_process_ddl_mysql.sql +++ b/support-files/sql/1001_ci_process_ddl_mysql.sql @@ -294,6 +294,7 @@ CREATE TABLE IF NOT EXISTS `T_PIPELINE_INFO` ( `TASK_COUNT` int(11) DEFAULT '0', `DELETE` bit(1) DEFAULT b'0', `ID` bigint(20) NOT NULL AUTO_INCREMENT, + `PIPELINE_NAME_PINYIN` varchar(1300) DEFAULT NULL COMMENT '流水线名称拼音', PRIMARY KEY (`PIPELINE_ID`), UNIQUE KEY `T_PIPELINE_INFO_NAME_uindex` (`PROJECT_ID`,`PIPELINE_NAME`), KEY `PROJECT_ID` (`PROJECT_ID`,`PIPELINE_ID`), diff --git a/support-files/sql/2007_ci_process-update_v1.6_mysql.sql b/support-files/sql/2007_ci_process-update_v1.6_mysql.sql index 4a3d1469224..c643d5d755a 100644 --- a/support-files/sql/2007_ci_process-update_v1.6_mysql.sql +++ b/support-files/sql/2007_ci_process-update_v1.6_mysql.sql @@ -69,6 +69,15 @@ BEGIN ALTER TABLE T_PIPELINE_BUILD_STAGE ADD COLUMN `CHECK_OUT` mediumtext NULL COMMENT '准出检查配置'; END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'T_PIPELINE_INFO' + AND COLUMN_NAME = 'PIPELINE_NAME_PINYIN') THEN + ALTER TABLE T_PIPELINE_INFO + ADD COLUMN `PIPELINE_NAME_PINYIN` varchar(1300) DEFAULT NULL COMMENT '流水线名称拼音'; + END IF; + COMMIT; END DELIMITER ;