Merge pull request #636 from wangyu096/github_bugfix/magic_namespace_…

…var_ref perf: 补充作业执行方案全局变量引用检测方法对魔法变量的解析逻辑 #583
TencentBlueKing · Jan 18, 2022 · d2234ad · d2234ad
2 parents 0839188 + 19e862e
commit d2234ad
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 23 deletions.
diff --git a/...-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskPlanResourceImpl.java b/...-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskPlanResourceImpl.java
@@ -48,7 +48,7 @@
 import com.tencent.bk.job.crontab.model.CronJobVO;
 import com.tencent.bk.job.manage.api.web.WebTaskPlanResource;
 import com.tencent.bk.job.manage.common.util.IamPathUtil;
-import com.tencent.bk.job.manage.manager.variable.StepVariableParser;
+import com.tencent.bk.job.manage.manager.variable.StepRefVariableParser;
 import com.tencent.bk.job.manage.model.dto.TaskPlanQueryDTO;
 import com.tencent.bk.job.manage.model.dto.task.TaskPlanInfoDTO;
 import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO;
@@ -319,7 +319,7 @@ public Response<TaskPlanVO> getPlanById(String username, Long appId, Long templa
         }
         TaskPlanInfoDTO taskPlan = planService.getTaskPlanById(appId, templateId, planId);
         if (taskPlan != null) {
-            StepVariableParser.parseStepRefVars(taskPlan.getStepList(), taskPlan.getVariableList());
+            StepRefVariableParser.parseStepRefVars(taskPlan.getStepList(), taskPlan.getVariableList());
 
             final String templateVersion = taskTemplateBasicInfo.getVersion();
             if (StringUtils.isNotEmpty(templateVersion)) {
@@ -357,7 +357,7 @@ public Response<TaskPlanVO> getDebugPlan(String username, Long appId, Long templ
         TaskPlanInfoDTO taskPlan = planService.getDebugTaskPlan(username, appId, templateId);
         TaskPlanVO taskPlanVO = null;
         if (taskPlan != null) {
-            StepVariableParser.parseStepRefVars(taskPlan.getStepList(), taskPlan.getVariableList());
+            StepRefVariableParser.parseStepRefVars(taskPlan.getStepList(), taskPlan.getVariableList());
             taskPlanVO = TaskPlanInfoDTO.toVO(taskPlan);
             taskPlanVO.setCanView(true);
             taskPlanVO.setCanEdit(true);

diff --git a/...age/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskTemplateResourceImpl.java b/...age/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskTemplateResourceImpl.java
@@ -44,7 +44,7 @@
 import com.tencent.bk.job.manage.common.consts.TemplateTypeEnum;
 import com.tencent.bk.job.manage.common.consts.task.TaskTemplateStatusEnum;
 import com.tencent.bk.job.manage.common.util.IamPathUtil;
-import com.tencent.bk.job.manage.manager.variable.StepVariableParser;
+import com.tencent.bk.job.manage.manager.variable.StepRefVariableParser;
 import com.tencent.bk.job.manage.model.dto.ResourceTagDTO;
 import com.tencent.bk.job.manage.model.dto.TagDTO;
 import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO;
@@ -230,7 +230,7 @@ public Response<TaskTemplateVO> getTemplateById(String username, Long appId, Lon
         if (templateInfo == null) {
             throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST);
         }
-        StepVariableParser.parseStepRefVars(templateInfo.getStepList(), templateInfo.getVariableList());
+        StepRefVariableParser.parseStepRefVars(templateInfo.getStepList(), templateInfo.getVariableList());
 
         TaskTemplateVO taskTemplateVO = TaskTemplateInfoDTO.toVO(templateInfo);
         taskTemplateVO.setCanView(true);

diff --git a/.../manager/variable/StepVariableParser.java → ...nager/variable/StepRefVariableParser.java b/.../manager/variable/StepVariableParser.java → ...nager/variable/StepRefVariableParser.java
@@ -24,6 +24,7 @@
 
 package com.tencent.bk.job.manage.manager.variable;
 
+import com.tencent.bk.job.common.constant.TaskVariableTypeEnum;
 import com.tencent.bk.job.common.service.VariableResolver;
 import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum;
 import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum;
@@ -35,26 +36,17 @@
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-public class StepVariableParser {
+/**
+ * 步骤引用的全局变量解析
+ */
+public class StepRefVariableParser {
 
-    public static List<String> parseShellScriptVar(String shellScriptContent) {
-        Matcher m = Pattern.compile("\\$\\{[#!]?([_a-zA-Z][0-9_a-zA-Z]*)\\S*}").matcher(shellScriptContent);
-        List<String> varNames = new ArrayList<>();
-        while (m.find()) {
-            String varName = m.group(1);
-            if (!varNames.contains(varName)) {
-                varNames.add(varName);
-            }
-        }
-        return varNames;
-    }
     /**
-     * 解析步骤和引用的全局变量
+     * 解析步骤引用的全局变量
      *
      * @param steps     步骤列表
      * @param variables 全局变量列表
@@ -86,21 +78,29 @@ private static void parseScriptStepRefVars(TaskStepDTO step,
         TaskScriptStepDTO scriptStep = step.getScriptStepInfo();
         List<String> refVarNames = new ArrayList<>();
 
+        // 解析脚本参数
         List<String> jobStandardVarNames = VariableResolver.resolveJobStandardVar(scriptStep.getScriptParam());
         if (CollectionUtils.isNotEmpty(jobStandardVarNames)) {
             refVarNames.addAll(jobStandardVarNames);
         }
+
+        // 解析 shell 脚本引用的一些特殊变量
         if (scriptStep.getLanguage() == ScriptTypeEnum.SHELL) {
+            // 从shell脚本中匹配所有符合shell变量命名的变量
             List<String> shellVarNames = VariableResolver.resolveShellScriptVar(scriptStep.getContent());
+            // 使用 job_import 方式引用的变量
             List<String> jobImportedVarNames = VariableResolver.resolveJobImportVariables(scriptStep.getContent());
             if (CollectionUtils.isNotEmpty(shellVarNames)) {
                 refVarNames.addAll(shellVarNames);
             }
             if (CollectionUtils.isNotEmpty(jobImportedVarNames)) {
                 refVarNames.addAll(jobImportedVarNames);
             }
+            // 魔法变量
+            refVarNames.addAll(resolveJobMagicNamespaceVar(jobImportedVarNames, variables));
         }
 
+        // 主机变量
         if (scriptStep.getExecuteTarget() != null
             && StringUtils.isNoneBlank(scriptStep.getExecuteTarget().getVariable())) {
             refVarNames.add(scriptStep.getExecuteTarget().getVariable());
@@ -110,6 +110,39 @@ private static void parseScriptStepRefVars(TaskStepDTO step,
             .distinct().collect(Collectors.toList()));
     }
 
+    /**
+     * 获取魔法变量解析之后的变量列表
+     *
+     * @param jobImportedVarNames 通过 job_import 方式引用的变量
+     * @param variables           全局变量
+     * @return 魔法变量解析之后的变量列表
+     */
+    private static List<String> resolveJobMagicNamespaceVar(List<String> jobImportedVarNames,
+                                                            List<TaskVariableDTO> variables) {
+        List<String> namespaceVarNames = variables.stream()
+            .filter(var -> var.getType() == TaskVariableTypeEnum.NAMESPACE)
+            .map(TaskVariableDTO::getName)
+            .collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(namespaceVarNames)) {
+            return Collections.emptyList();
+        }
+
+        List<String> resolvedNamespaceVars = new ArrayList<>();
+        jobImportedVarNames.stream()
+            .filter(jobImportedVarName -> jobImportedVarName.startsWith("JOB_NAMESPACE_"))
+            .forEach(jobImportedVarName -> {
+                if ("JOB_NAMESPACE_ALL".equals(jobImportedVarName)) {
+                    resolvedNamespaceVars.addAll(namespaceVarNames);
+                } else {
+                    String jobImportNamespaceVar = jobImportedVarName.substring("JOB_NAMESPACE_".length());
+                    if (namespaceVarNames.contains(jobImportNamespaceVar)) {
+                        resolvedNamespaceVars.add(jobImportNamespaceVar);
+                    }
+                }
+            });
+        return resolvedNamespaceVars.stream().distinct().collect(Collectors.toList());
+    }
+
 
     private static void parseFileStepRefVars(TaskStepDTO step,
                                              List<TaskVariableDTO> variables) {

diff --git a/...ager/variable/StepVariableParserTest.java → ...r/variable/StepRefVariableParserTest.java b/...ager/variable/StepVariableParserTest.java → ...r/variable/StepRefVariableParserTest.java
@@ -42,7 +42,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-class StepVariableParserTest {
+class StepRefVariableParserTest {
     @Test
     @DisplayName("ParseScriptStepRefVars")
     void parseScriptStepRefVars() {
@@ -79,7 +79,7 @@ void parseScriptStepRefVars() {
         host1.setName("host1");
         variables.add(host1);
 
-        StepVariableParser.parseStepRefVars(steps, variables);
+        StepRefVariableParser.parseStepRefVars(steps, variables);
         assertThat(step1.getRefVariables()).extracting("name")
             .containsOnly("var1", "var2", "var4", "host1");
     }
@@ -132,7 +132,7 @@ void parseFileStepRefVars() {
         host2.setName("host2");
         variables.add(host2);
 
-        StepVariableParser.parseStepRefVars(steps, variables);
+        StepRefVariableParser.parseStepRefVars(steps, variables);
         assertThat(step1.getRefVariables()).extracting("name")
             .containsOnly("var1", "var2", "var3", "host1", "host2");
     }