feature: 修复 Proxy gse_agent.conf 部分配置项缺失的问题 (fixed TencentBlueKing#1919)

apps/backend/agent/artifact_builder/base.py

@@ -589,10 +589,12 @@ def make(
             )
 
         artifact_meta_info["operator"] = operator
+        # Agent 包先导入文件源 -> 写配置文件 -> 创建包记录 -> 创建 Tag
         self.update_or_create_support_files(package_infos)
-        self.update_or_create_tag(artifact_meta_info)
+        # TODO update_or_create_record & update_or_create_package_records 似乎是一样的功能？
         self.update_or_create_record(artifact_meta_info)
         self.update_or_create_package_records(package_infos)
+        self.update_or_create_tag(artifact_meta_info)
 
     def __enter__(self) -> "BaseArtifactBuilder":
         return self

apps/backend/subscription/steps/agent_adapter/adapter.py

@@ -16,7 +16,6 @@
 
 from rest_framework import serializers
 
-from apps.backend import exceptions
 from apps.backend.agent.tools import fetch_proxies
 from apps.backend.constants import ProxyConfigFile
 from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType
@@ -27,7 +26,7 @@
 
 from . import base, legacy
 from .config_context import context_helper
-from .handlers import GseConfigHandler, get_gse_config_handler_class
+from .handlers import GseConfigHandler
 
 logger = logging.getLogger("app")
 
@@ -66,14 +65,15 @@ def __post_init__(self):
         )
         self._config_handler_cache: typing.Dict[str, GseConfigHandler] = {}
 
-    def get_config_handler(self, target_version: str, node_type: str) -> GseConfigHandler:
+    def get_config_handler(self, agent_name: str, target_version: str) -> GseConfigHandler:
+
         # 预留 AgentStepAdapter 支持多 Agent 版本的场景
-        cache_key: str = f"node_type:{node_type}:version{target_version}"
+        cache_key: str = f"agent_name:{agent_name}:version:{target_version}"
         config_handler: typing.Optional[GseConfigHandler] = self._config_handler_cache.get(cache_key)
         if config_handler:
             return config_handler
 
-        config_handler: GseConfigHandler = get_gse_config_handler_class(node_type)(target_version=target_version)
+        config_handler: GseConfigHandler = GseConfigHandler(agent_name=agent_name, target_version=target_version)
         self._config_handler_cache[cache_key] = config_handler
         return config_handler
 
@@ -104,23 +104,13 @@ def _get_config(
         install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]],
     ) -> str:
         agent_setup_info: base.AgentSetupInfo = self.setup_info
-        config_handler: GseConfigHandler = self.get_config_handler(agent_setup_info.version, node_type)
+        config_handler: GseConfigHandler = self.get_config_handler(agent_setup_info.name, agent_setup_info.version)
         config_tmpl_obj: base.AgentConfigTemplate = config_handler.get_matching_config_tmpl(
             os_type=host.os_type,
             cpu_arch=host.cpu_arch,
             config_name=filename,
         )
 
-        if not config_tmpl_obj:
-            logger.error(
-                f"{self.log_prefix} agent config template not exist: name -> {self.config['name']}, "
-                f"filename -> {filename}, version -> {self.config['version']}, "
-                f"os_type -> {host.os_type}, cpu_arch -> {host.cpu_arch}"
-            )
-            raise exceptions.AgentConfigTemplateNotExistError(
-                name=self.config["name"], filename=filename, os_type=host.os_type, cpu_arch=host.cpu_arch
-            )
-
         # 渲染配置
         ch: context_helper.ConfigContextHelper = context_helper.ConfigContextHelper(
             agent_setup_info=agent_setup_info,
@@ -132,7 +122,7 @@ def _get_config(
         )
         return ch.render(
             config_tmpl_obj.content,
-            config_handler.get_matching_template_env(host.os_type, host.cpu_arch),
+            config_handler.get_matching_template_env(host.os_type, host.cpu_arch, config_tmpl_obj.agent_name_from),
             config_handler.get_matching_template_extra_env(host),
         )
 

apps/backend/subscription/steps/agent_adapter/base.py

@@ -17,6 +17,7 @@
 class AgentConfigTemplate:
     name: str
     content: str
+    agent_name_from: str
 
 
 @dataclass

apps/backend/subscription/steps/agent_adapter/handlers.py

@@ -11,11 +11,11 @@
 
 import logging
 from collections import defaultdict
-from typing import Any, Dict, List, Optional, Type
+from typing import Any, Dict, List, Optional
 
 from apps.backend import exceptions
 from apps.node_man import constants
-from apps.node_man.constants import GsePackageCode, NodeType
+from apps.node_man.constants import GsePackageCode
 from apps.node_man.models import (
     GseConfigEnv,
     GseConfigExtraEnv,
@@ -31,10 +31,8 @@
 
 
 class GseConfigHandler:
-
-    AGENT_NAME = None
-
-    def __init__(self, target_version: str) -> None:
+    def __init__(self, agent_name: str, target_version: str) -> None:
+        self.agent_name = agent_name
         self.target_version = target_version
         self._config_extra_env_cache: Dict[int, List[GseConfigExtraEnv]] = {}
 
@@ -50,6 +48,16 @@ def fetch_config_extra_envs(self, bk_biz_id: int) -> List[GseConfigExtraEnv]:
         self._config_extra_env_cache[bk_biz_id] = config_extra_envs
         return config_extra_envs
 
+    def fetch_pre_dependencies(self) -> List[str]:
+        """
+        获取 Agent 的前置依赖
+        TODO 此处暂时写死，后续 Agent 包管理导包时从数据库获取
+        :return:
+        """
+        return {GsePackageCode.AGENT.value: [], GsePackageCode.PROXY.value: [GsePackageCode.AGENT.value]}.get(
+            self.agent_name
+        ) or []
+
     @staticmethod
     def get_os_key(os_type: str, cpu_arch: str) -> str:
         # 默认为 linux-x86_64，兼容CMDB同步过来的主机没有操作系统和CPU架构的场景
@@ -61,9 +69,21 @@ def get_os_key(os_type: str, cpu_arch: str) -> str:
     @cache.class_member_cache()
     def agent_config_templates(self) -> List[AgentConfigTemplate]:
         return [
-            AgentConfigTemplate(name="gse_agent.conf", content=config_templates.GSE_AGENT_CONFIG_TMPL),
-            AgentConfigTemplate(name="gse_data_proxy.conf", content=config_templates.GSE_DATA_PROXY_CONFIG_TMPL),
-            AgentConfigTemplate(name="gse_file_proxy.conf", content=config_templates.GSE_FILE_PROXY_CONFIG_TEMPL),
+            AgentConfigTemplate(
+                name="gse_agent.conf",
+                agent_name_from=GsePackageCode.AGENT.value,
+                content=config_templates.GSE_AGENT_CONFIG_TMPL,
+            ),
+            AgentConfigTemplate(
+                name="gse_data_proxy.conf",
+                agent_name_from=GsePackageCode.PROXY.value,
+                content=config_templates.GSE_DATA_PROXY_CONFIG_TMPL,
+            ),
+            AgentConfigTemplate(
+                name="gse_file_proxy.conf",
+                agent_name_from=GsePackageCode.PROXY.value,
+                content=config_templates.GSE_FILE_PROXY_CONFIG_TEMPL,
+            ),
         ]
 
     @property
@@ -74,10 +94,16 @@ def config_tmpl_obj_gby_os_key(self) -> Dict[str, List[AgentConfigTemplate]]:
         :return:
         """
         config_tmpl_obj_gby_os_key: Dict[str, List[AgentConfigTemplate]] = defaultdict(list)
-        config_tmpls: List[GseConfigTemplate] = GseConfigTemplate.objects.filter(version=self.target_version)
+        dependencies: List[str] = [self.agent_name]
+        dependencies.extend(self.fetch_pre_dependencies())
+        config_tmpls: List[GseConfigTemplate] = GseConfigTemplate.objects.filter(
+            agent_name__in=dependencies, version=self.target_version
+        )
         for config_tmpl in config_tmpls:
             config_tmpl_obj_gby_os_key[self.get_os_key(config_tmpl.os, config_tmpl.cpu_arch)].append(
-                AgentConfigTemplate(name=config_tmpl.name, content=config_tmpl.content)
+                AgentConfigTemplate(
+                    name=config_tmpl.name, agent_name_from=config_tmpl.agent_name, content=config_tmpl.content
+                )
             )
 
         # 为空说明该版本的配置文件都不存在，此时直接返回默认的配置模板
@@ -91,13 +117,15 @@ def config_tmpl_obj_gby_os_key(self) -> Dict[str, List[AgentConfigTemplate]]:
 
     @property
     @cache.class_member_cache()
-    def os_key__tmpl_env_map(self) -> Dict[str, Dict[str, Any]]:
-        os_key__tmpl_env_map: Dict[str, Dict[str, Any]] = {}
+    def tmpl_env_obj_obj_gby_os_key(self) -> Dict[str, List[GseConfigEnv]]:
+        dependencies: List[str] = [self.agent_name]
+        dependencies.extend(self.fetch_pre_dependencies())
+        os_key__tmpl_env_map: Dict[str, List[GseConfigEnv]] = defaultdict(list)
         config_envs: List[GseConfigEnv] = GseConfigEnv.objects.filter(
-            agent_name=self.AGENT_NAME, version=self.target_version
+            agent_name__in=dependencies, version=self.target_version
         )
         for config_env in config_envs:
-            os_key__tmpl_env_map[self.get_os_key(config_env.os, config_env.cpu_arch)] = config_env.env_value
+            os_key__tmpl_env_map[self.get_os_key(config_env.os, config_env.cpu_arch)].append(config_env)
 
         return os_key__tmpl_env_map
 
@@ -106,25 +134,52 @@ def get_matching_config_tmpl(self, os_type: str, cpu_arch: str, config_name: str
             self.get_os_key(os_type, cpu_arch), self.agent_config_templates
         )
         # 查找机型匹配的第一个配置
-        target_config_tmpl_obj: AgentConfigTemplate = next(
+        target_config_tmpl_obj: Optional[AgentConfigTemplate] = next(
             (config_tmpl_obj for config_tmpl_obj in config_tmpl_objs if config_tmpl_obj.name == config_name), None
         )
         if not target_config_tmpl_obj:
             logger.error(
                 "[GseConfigHandler(get_matching_config_tmpl)] AgentConfigTemplate not exist: "
                 "name -> %s, config_name -> %s, os_type -> %s, cpu_arch -> %s",
-                self.AGENT_NAME,
+                self.agent_name,
                 config_name,
                 os_type,
                 cpu_arch,
             )
             raise exceptions.AgentConfigTemplateNotExistError(
-                name=self.AGENT_NAME, filename=config_name, os_type=os_type, cpu_arch=cpu_arch
+                name=self.agent_name, filename=config_name, os_type=os_type, cpu_arch=cpu_arch
             )
         return target_config_tmpl_obj
 
-    def get_matching_template_env(self, os_type: str, cpu_arch: str) -> Dict[str, Any]:
-        return self.os_key__tmpl_env_map.get(self.get_os_key(os_type, cpu_arch)) or {}
+    def get_matching_template_env(self, os_type: str, cpu_arch: str, agent_name_from: str) -> Dict[str, Any]:
+        tmpl_env_objs: List[GseConfigEnv] = self.tmpl_env_obj_obj_gby_os_key.get(self.get_os_key(os_type, cpu_arch), [])
+
+        if not tmpl_env_objs:
+            return {}
+
+        # 匹配机型及模板渲染环境变量 Agent 来源
+        # 背景：2.0 Proxy 依赖 2.0 Agent 的配置模板（例如 gse_agent.conf），并且需要用相应的 Env 文件完成渲染
+        # 实现：根据配置文件的 Agent 来源（agent_name_from）进行匹配
+        # eg1：gse_agent.conf（agent_name_from=gse_agent）-> gse_config_env（agent_name=gse_agent）
+        # eg2：gse_data_proxy.conf（agent_name_from=gse_proxy）-> gse_config_env（agent_name=gse_proxy）
+        target_tmpl_env_obj: Optional[GseConfigEnv] = next(
+            (tmpl_env_obj for tmpl_env_obj in tmpl_env_objs if tmpl_env_obj.agent_name == agent_name_from), None
+        )
+
+        if not target_tmpl_env_obj:
+            logger.error(
+                "[GseConfigHandler(get_matching_template_env)] GseConfigEnv not exist: "
+                "name -> %s, os_type -> %s, cpu_arch -> %s, agent_name_from -> %s",
+                self.agent_name,
+                agent_name_from,
+                os_type,
+                cpu_arch,
+            )
+            raise exceptions.AgentConfigTemplateNotExistError(
+                f"GseConfigEnv[{self.agent_name}-{os_type}-{cpu_arch}-{agent_name_from}] not exist"
+            )
+
+        return target_tmpl_env_obj.env_value
 
     def get_matching_template_extra_env(self, host: Host) -> Dict[str, Any]:
         template_extra_env: Dict[str, Any] = {}
@@ -155,21 +210,3 @@ def get_matching_template_extra_env(self, host: Host) -> Dict[str, Any]:
                     config_extra_env.condition,
                 )
         return template_extra_env
-
-
-class AgentGseConfigHandler(GseConfigHandler):
-    AGENT_NAME = GsePackageCode.AGENT.value
-
-
-class ProxyGseConfigHandler(GseConfigHandler):
-    AGENT_NAME = GsePackageCode.PROXY.value
-
-
-GSE_CONFIG_HANDLER_PACKAGE_CODE_MAP: Dict = {
-    NodeType.AGENT.lower(): AgentGseConfigHandler,
-    NodeType.PROXY.lower(): ProxyGseConfigHandler,
-}
-
-
-def get_gse_config_handler_class(node_type: str) -> Type[GseConfigHandler]:
-    return GSE_CONFIG_HANDLER_PACKAGE_CODE_MAP[node_type]

apps/backend/tests/agent/artifact_builder/test_agent.py

@@ -14,10 +14,7 @@
 import mock
 from django.conf import settings
 
-from apps.backend.subscription.steps.agent_adapter.handlers import (
-    GseConfigHandler,
-    get_gse_config_handler_class,
-)
+from apps.backend.subscription.steps.agent_adapter.handlers import GseConfigHandler
 from apps.backend.tests.agent import template_env, utils
 from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType
 from apps.core.tag.models import Tag
@@ -63,9 +60,7 @@ def tag_checker(self, target_id: int):
         self.assertTrue(agent_target_version == utils.VERSION)
 
     def template_and_env_checker(self, version_str):
-        gse_config_handler: GseConfigHandler = get_gse_config_handler_class(
-            node_type=("proxy", "agent")[self.NAME == "gse_agent"]
-        )(version_str)
+        gse_config_handler: GseConfigHandler = GseConfigHandler(self.NAME, version_str)
 
         for package_os, cpu_arch in self.OS_CPU_CHOICES:
             filter_kwargs: dict = {
@@ -75,19 +70,36 @@ def template_and_env_checker(self, version_str):
                 "version": version_str,
             }
             self.assertDictEqual(
-                gse_config_handler.get_matching_template_env(package_os, cpu_arch),
+                gse_config_handler.get_matching_template_env(package_os, cpu_arch, self.NAME),
                 self.ARTIFACT_BUILDER_CLASS.parse_env(
                     (template_env.DEFAULT_AGENT_TEMPLATE_ENV, template_env.DEFAULT_PROXY_TEMPLATE_ENV)[
-                        self.NAME == "gse_proxy"
+                        self.NAME == constants.GsePackageCode.PROXY.value
                     ]
                 ),
             )
 
+            # gse_agent.conf 一定是 gse_agent 的
+            self.assertEqual(
+                gse_config_handler.get_matching_config_tmpl(
+                    package_os, cpu_arch, config_name="gse_agent.conf"
+                ).agent_name_from,
+                constants.GsePackageCode.AGENT.value,
+            )
             gse_config_handler.get_matching_config_tmpl(package_os, cpu_arch, config_name="gse_agent.conf")
 
-            if self.NAME == "gse_proxy":
-                gse_config_handler.get_matching_config_tmpl(package_os, cpu_arch, config_name="gse_data_proxy.conf")
-                gse_config_handler.get_matching_config_tmpl(package_os, cpu_arch, config_name="gse_file_proxy.conf")
+            if self.NAME == constants.GsePackageCode.PROXY.value:
+                self.assertEqual(
+                    gse_config_handler.get_matching_config_tmpl(
+                        package_os, cpu_arch, config_name="gse_data_proxy.conf"
+                    ).agent_name_from,
+                    self.NAME,
+                )
+                self.assertEqual(
+                    gse_config_handler.get_matching_config_tmpl(
+                        package_os, cpu_arch, config_name="gse_file_proxy.conf"
+                    ).agent_name_from,
+                    self.NAME,
+                )
 
             config_env_obj: models.GseConfigEnv = models.GseConfigEnv.objects.filter(**filter_kwargs).first()
             self.parse_env_checker(env_values=config_env_obj.env_value)
@@ -143,10 +155,10 @@ def setUp(self):
 
 
 class AutoTypeStrategyDiffTestCase(AutoTypeStrategyCrontabTestCase):
-    AUTO_TYPE_STRATEGY = {"gse_proxy": "crontab"}
+    AUTO_TYPE_STRATEGY = {constants.GsePackageCode.PROXY.value: "crontab"}
     AUTO_TYPE = constants.GseLinuxAutoType.RCLOCAL.value
 
 
 class AutoTypeStrategyNotEffectTestCase(AutoTypeStrategyCrontabTestCase):
-    AUTO_TYPE_STRATEGY = {"gse_agent": "crontab"}
+    AUTO_TYPE_STRATEGY = {constants.GsePackageCode.AGENT.value: "crontab"}
     AUTO_TYPE = constants.GseLinuxAutoType.CRONTAB.value

apps/backend/tests/agent/artifact_builder/test_proxy.py

@@ -53,10 +53,10 @@ def setUp(self):
 
 
 class AutoTypeStrategyDiffTestCase(AutoTypeStrategyCrontabTestCase):
-    AUTO_TYPE_STRATEGY = {"gse_proxy": "crontab"}
+    AUTO_TYPE_STRATEGY = {constants.GsePackageCode.PROXY.value: "crontab"}
     AUTO_TYPE = constants.GseLinuxAutoType.CRONTAB.value
 
 
 class AutoTypeStrategyNotEffectTestCase(AutoTypeStrategyCrontabTestCase):
-    AUTO_TYPE_STRATEGY = {"gse_agent": "crontab"}
+    AUTO_TYPE_STRATEGY = {constants.GsePackageCode.AGENT.value: "crontab"}
     AUTO_TYPE = constants.GseLinuxAutoType.RCLOCAL.value

apps/backend/tests/agent/utils.py

@@ -299,6 +299,14 @@ def gen_base_agent_pkg(cls):
                 agent_name=AgentBaseTestCase.NAME,
             )
 
+            models.GseConfigEnv.objects.update_or_create(
+                defaults={"env_value": cls.ARTIFACT_BUILDER_CLASS.parse_env(template_env.DEFAULT_AGENT_TEMPLATE_ENV)},
+                version=VERSION,
+                os=package_os,
+                cpu_arch=cpu_arch,
+                agent_name=AgentBaseTestCase.NAME,
+            )
+
 
 class AutoTypeStrategyMixin: