Merge pull request #11211 from yongyiduan/stream-daimatongbu

【蓝盾-评审会已评审】【PAC】feat：新建/编辑流水线支持以 Code 方式编排流水线 #8125 core部分代码回退
TencentBlueKing · Nov 13, 2024 · 426e12a · 426e12a
2 parents 190494c + cc38ac5
commit 426e12a
Show file tree

Hide file tree

Showing 24 changed files with 3,150 additions and 1,143 deletions.
diff --git a/...n-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelContainer.kt b/...n-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelContainer.kt
@@ -48,8 +48,8 @@ import com.tencent.devops.common.pipeline.pojo.element.Element
 import com.tencent.devops.common.web.utils.I18nUtil
 import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo
 import com.tencent.devops.process.yaml.creator.inner.InnerModelCreator
-import com.tencent.devops.process.yaml.pojo.StreamDispatchInfo
 import com.tencent.devops.process.yaml.transfer.TransferCacheService
+import com.tencent.devops.process.yaml.pojo.StreamDispatchInfo
 import com.tencent.devops.process.yaml.utils.ModelCreateUtil
 import com.tencent.devops.process.yaml.v2.models.IfType
 import com.tencent.devops.process.yaml.v2.models.Resources

diff --git a/...line-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreScriptBuildYaml.kt b/...line-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreScriptBuildYaml.kt
@@ -29,7 +29,6 @@ package com.tencent.devops.process.yaml.v2.models
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonInclude
-import com.fasterxml.jackson.annotation.JsonProperty
 import com.tencent.devops.process.yaml.v2.models.job.PreJob
 import com.tencent.devops.process.yaml.v2.models.on.PreTriggerOn
 import com.tencent.devops.process.yaml.v2.models.stage.PreStage
@@ -66,7 +65,6 @@ data class PreScriptBuildYaml(
     override var version: String?,
     override var name: String?,
     override var label: List<String>? = null,
-    @JsonProperty("on")
     override var triggerOn: PreTriggerOn?,
     override var variables: Map<String, Variable>? = null,
     override var stages: List<PreStage>? = null,

diff --git a/...l/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreTemplateScriptBuildYaml.kt b/...l/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/PreTemplateScriptBuildYaml.kt
@@ -29,7 +29,6 @@ package com.tencent.devops.process.yaml.v2.models
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonInclude
-import com.fasterxml.jackson.annotation.JsonProperty
 import com.tencent.devops.process.yaml.v2.models.on.PreTriggerOn
 
 /**
@@ -43,7 +42,6 @@ data class PreTemplateScriptBuildYaml(
     val version: String?,
     val name: String?,
     val label: List<String>? = null,
-    @JsonProperty("on")
     val triggerOn: PreTriggerOn?,
     val variables: Map<String, Any>?,
     val stages: List<Map<String, Any>>?,

diff --git a/...ipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/ScriptBuildYaml.kt b/...ipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/ScriptBuildYaml.kt
@@ -29,7 +29,6 @@ package com.tencent.devops.process.yaml.v2.models
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonInclude
-import com.fasterxml.jackson.annotation.JsonProperty
 import com.tencent.devops.process.yaml.v2.models.job.Job
 import com.tencent.devops.process.yaml.v2.models.on.TriggerOn
 import com.tencent.devops.process.yaml.v2.models.stage.Stage
@@ -45,7 +44,6 @@ data class ScriptBuildYaml(
     val version: String?,
     val name: String?,
     val label: List<String>?,
-    @JsonProperty("on")
     val triggerOn: TriggerOn?,
     val variables: Map<String, Variable>?,
     val stages: List<Stage>,

diff --git a/.../main/kotlin/com/tencent/devops/process/yaml/v2/models/export/ExportPreScriptBuildYaml.kt b/.../main/kotlin/com/tencent/devops/process/yaml/v2/models/export/ExportPreScriptBuildYaml.kt
@@ -29,7 +29,6 @@ package com.tencent.devops.process.yaml.v2.models.export
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonInclude
-import com.fasterxml.jackson.annotation.JsonProperty
 import com.tencent.devops.process.yaml.v2.models.Extends
 import com.tencent.devops.process.yaml.v2.models.GitNotices
 import com.tencent.devops.process.yaml.v2.models.Resources
@@ -49,7 +48,6 @@ data class ExportPreScriptBuildYaml(
     var version: String?,
     var name: String?,
     var label: List<String>? = null,
-    @JsonProperty("on")
     var triggerOn: PreTriggerOn?,
     var variables: Map<String, String>? = null,
     var stages: List<PreStage>? = null,

diff --git a/...in/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/models/NoReplaceTemplate.kt b/...in/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/models/NoReplaceTemplate.kt
@@ -29,7 +29,6 @@ package com.tencent.devops.process.yaml.v2.parsers.template.models
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonInclude
-import com.fasterxml.jackson.annotation.JsonProperty
 import com.tencent.devops.process.yaml.v2.models.Concurrency
 import com.tencent.devops.process.yaml.v2.models.Extends
 import com.tencent.devops.process.yaml.v2.models.Resources
@@ -42,7 +41,6 @@ data class NoReplaceTemplate(
     var version: String?,
     var name: String?,
     var label: List<String>? = null,
-    @JsonProperty("on")
     var triggerOn: PreTriggerOn?,
     var extends: Extends?,
     var resources: Resources?,

diff --git a/...-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt b/...-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt
@@ -50,7 +50,6 @@ import com.tencent.devops.common.api.util.JsonUtil
 import com.tencent.devops.common.api.util.UUIDUtil
 import com.tencent.devops.common.api.util.YamlUtil
 import com.tencent.devops.common.web.utils.I18nUtil
-import com.tencent.devops.process.yaml.transfer.TransferMapper
 import com.tencent.devops.process.yaml.v2.enums.StreamMrEventAction
 import com.tencent.devops.process.yaml.v2.enums.TemplateType
 import com.tencent.devops.process.yaml.v2.exception.YamlFormatException
@@ -117,9 +116,9 @@ object ScriptYmlUtils {
     @Throws(JsonProcessingException::class)
     fun formatYaml(yamlStr: String): String {
         // replace custom tag
-//        val yamlNormal = formatYamlCustom(yamlStr)
+        val yamlNormal = formatYamlCustom(yamlStr)
         // replace anchor tag
-        return TransferMapper.formatYaml(yamlStr)
+        return YamlUtil.loadYamlRetryOnAccident(yamlNormal)
     }
 
     fun parseVersion(yamlStr: String?): YmlVersion? {
@@ -224,8 +223,8 @@ object ScriptYmlUtils {
             val startString = line.trim().replace("\\s".toRegex(), "")
             if (startString.startsWith("if:") || startString.startsWith("-if:")) {
                 val ifPrefix = line.substring(0 until line.indexOfFirst { it == ':' } + 1)
-                val condition = line.removePrefix(ifPrefix).trim()
-                    .removeSurrounding("\"")
+                val condition = line.substring(line.indexOfFirst { it == '"' } + 1 until line.length).trimEnd()
+                    .removeSuffix("\"")
 
                 // 去掉花括号
                 val baldExpress = condition.replace("\${{", "").replace("}}", "").trim()

diff --git a/...yaml/src/test/kotlin/com/tencent/devops/process/yaml/parsers/template/YamlTemplateTest.kt b/...yaml/src/test/kotlin/com/tencent/devops/process/yaml/parsers/template/YamlTemplateTest.kt
@@ -28,7 +28,7 @@
 package com.tencent.devops.process.yaml.parsers.template
 
 import com.tencent.devops.common.api.util.JsonUtil
-import com.tencent.devops.process.yaml.transfer.TransferMapper
+import com.tencent.devops.common.api.util.YamlUtil
 import com.tencent.devops.process.yaml.v2.exception.YamlFormatException
 import com.tencent.devops.process.yaml.v2.models.PreScriptBuildYaml
 import com.tencent.devops.process.yaml.v2.models.PreTemplateScriptBuildYaml
@@ -313,9 +313,7 @@ variables:
         val sb = yamlContent ?: getStrFromResource(testYaml!!)
 
         val yaml = ScriptYmlUtils.formatYaml(sb)
-        val preTemplateYamlObject = TransferMapper.getObjectMapper().readValue(
-            yaml, PreTemplateScriptBuildYaml::class.java
-        )
+        val preTemplateYamlObject = YamlUtil.getObjectMapper().readValue(yaml, PreTemplateScriptBuildYaml::class.java)
         preTemplateYamlObject.resources?.pools?.forEach { pool ->
             resourceExt?.put(pool.format(), pool)
         }
@@ -328,12 +326,12 @@ variables:
             repo = null,
             resourcePoolMapExt = resourceExt,
             conf = YamlTemplateConf(useOldParametersExpression = useOldParametersExpression)
-        ).replace() as PreScriptBuildYaml
+        ).replace()
         if (!normalized) {
             return preScriptBuildYaml
         }
         val (normalOb, trans) = ScriptYmlUtils.normalizeGitCiYaml(preScriptBuildYaml, "")
-        val yamls = TransferMapper.toYaml(normalOb)
+        val yamls = YamlUtil.toYaml(normalOb)
         println("------------ Trans -----------")
         println(JsonUtil.toJson(trans))
         println("------------------------")
@@ -349,7 +347,6 @@ variables:
             "templates/${param.targetRepo!!.repository}/templates/${param.path}"
         }
         val sb = getStrFromResource(newPath)
-        println(newPath)
         return ScriptYmlUtils.formatYaml(sb)
     }
 

diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/test/resources/Sample1.yml b/src/backend/ci/core/common/common-pipeline-yaml/src/test/resources/Sample1.yml
@@ -91,12 +91,14 @@ stages:
             timeout-minutes: 480
             continue-on-error: false
             retry-times: 0
+            env:
 
           - run: |
               echo "stage1, job_id_1"
               echo "hello world" >> test.txt
 
         timeout-minutes: 480
+        env:
 
         continue-on-error: false
         strategy:
@@ -134,7 +136,7 @@ stages:
                 username: ${{ settings.CSIGHUB_USER }}
                 password: ${{ settings.CSIGHUB_PWD }}
 
-#finally:
+finally:
 #  f_job_1:
 #    name: f-job-1
 #    if: FAILURE
@@ -159,4 +161,3 @@ resources:
     - repository: epc-ieg-bk-common
       name: ieg/epc-ieg-bk-common
       ref: refs/tags/v1.0
-