From ded7a87dd9445136ed07fdf0d391baeb788c15fa Mon Sep 17 00:00:00 2001 From: Yunchao Date: Fri, 18 Aug 2023 12:37:05 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20=20Agent=202.0=20=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=A8=A1=E6=9D=BF=E3=80=81=E9=85=8D=E7=BD=AE=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E7=AE=A1=E7=90=86=20(closed=20#1681)=20(#175?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/agent/artifact_builder/agent.py | 5 +- apps/backend/agent/artifact_builder/base.py | 125 +++++++++++++- apps/backend/agent/artifact_builder/proxy.py | 6 +- apps/backend/agent/config_parser.py | 45 +++++ apps/backend/exceptions.py | 6 + .../management/commands/init_agents.py | 8 + .../steps/agent_adapter/adapter.py | 70 ++++---- .../config_context/context_helper.py | 8 +- .../steps/agent_adapter/handlers.py | 150 ++++++++++++++++ .../enable_agent_pkg_manage/__init__.py | 10 ++ .../enable_agent_pkg_manage/test_agent.py | 61 +++++++ .../test_manage_commands.py | 50 ++++++ .../enable_agent_pkg_manage/test_proxy.py | 23 +++ .../agent/artifact_builder/test_agent.py | 11 ++ apps/backend/tests/agent/template_env.py | 161 ++++++++++++++++++ apps/backend/tests/agent/utils.py | 13 +- apps/core/tag/constants.py | 8 + apps/core/tag/targets/__init__.py | 4 +- apps/core/tag/targets/agent.py | 37 ++++ apps/core/tag/targets/base.py | 25 ++- apps/node_man/constants.py | 23 ++- .../0072_gseconfigenv_gseconfigtemplate.py | 93 ++++++++++ .../migrations/0073_auto_20230815_1550.py | 35 ++++ .../migrations/0074_merge_20230818_1214.py | 22 +++ apps/node_man/models.py | 57 +++++++ 25 files changed, 999 insertions(+), 57 deletions(-) create mode 100644 apps/backend/agent/config_parser.py create mode 100644 apps/backend/subscription/steps/agent_adapter/handlers.py create mode 100644 apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/__init__.py create mode 100644 apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_agent.py create mode 100644 apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_manage_commands.py create mode 100644 apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_proxy.py create mode 100644 apps/backend/tests/agent/template_env.py create mode 100644 apps/core/tag/targets/agent.py create mode 100644 apps/node_man/migrations/0072_gseconfigenv_gseconfigtemplate.py create mode 100644 apps/node_man/migrations/0073_auto_20230815_1550.py create mode 100644 apps/node_man/migrations/0074_merge_20230818_1214.py diff --git a/apps/backend/agent/artifact_builder/agent.py b/apps/backend/agent/artifact_builder/agent.py index 3f1f093ad..09ac1d641 100644 --- a/apps/backend/agent/artifact_builder/agent.py +++ b/apps/backend/agent/artifact_builder/agent.py @@ -22,6 +22,8 @@ class AgentArtifactBuilder(base.BaseArtifactBuilder): + ENV_FILES = constants.GsePackageEnv.AGENT.value + TEMPLATE_PATTERN = constants.GsePackageTemplatePattern.AGENT.value NAME = constants.GsePackageCode.AGENT.value PKG_DIR = constants.GsePackageDir.AGENT.value CERT_FILENAMES = [ @@ -39,5 +41,4 @@ def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir return extract_dir def _get_support_files_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]: - # Agent 包管理实现 - pass + return super()._get_support_files_info(extract_dir=extract_dir) diff --git a/apps/backend/agent/artifact_builder/base.py b/apps/backend/agent/artifact_builder/base.py index b1bda746c..389abba34 100644 --- a/apps/backend/agent/artifact_builder/base.py +++ b/apps/backend/agent/artifact_builder/base.py @@ -20,8 +20,11 @@ from django.utils.translation import ugettext_lazy as _ from apps.backend import exceptions +from apps.backend.agent.config_parser import GseConfigParser from apps.core.files import core_files_constants from apps.core.files.storage import get_storage +from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.handlers import TagHandler from apps.node_man import constants, models from apps.utils import cache, files @@ -44,12 +47,23 @@ class BaseArtifactBuilder(abc.ABC): # 制品存储的根路径 BASE_STORAGE_DIR: str = "agent" + VERSION_FILE_NAME: str = "VERSION" + + SUPPORT_FILES_DIR: str = "support-files" + ENV_FILES_DIR: str = os.path.join(SUPPORT_FILES_DIR, "env") + TEMPLATE_FILES_DIR: str = os.path.join(SUPPORT_FILES_DIR, "templates") + TEMPLATE_PATTERN: str = None + + ENV_FILES: typing.List[str] = [] + TEMPLATE_FILES: typing.List[str] = [] + def __init__( self, initial_artifact_path: str, cert_path: typing.Optional[str] = None, download_path: typing.Optional[str] = None, overwrite_version: typing.Optional[str] = None, + enable_agent_pkg_manage: bool = False, ): """ :param initial_artifact_path: 原始制品所在路径 @@ -68,6 +82,10 @@ def __init__( self.applied_tmp_dirs = set() # 文件源 self.storage = get_storage(file_overwrite=True) + self.enable_agent_pkg_manage = enable_agent_pkg_manage or models.GlobalSettings.get_config( + key=models.GlobalSettings.KeyEnum.ENABLE_AGENT_PKG_MANAGE.value, + default=False, + ) @staticmethod def download_file(file_path: str, target_path: str): @@ -145,14 +163,38 @@ def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir """ raise NotImplementedError - @abc.abstractmethod def _get_support_files_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]: """ 获取部署依赖文件合集：配置 / 环境变量等 :param extract_dir: 解压目录 :return: """ - raise NotImplementedError + env_file_infos: typing.List[typing.Dict] = [] + template_file_infos: typing.List[typing.Dict] = [] + # 获取env信息 + for env_file in os.listdir(os.path.join(extract_dir, self.ENV_FILES_DIR)): + if env_file not in self.ENV_FILES: + continue + env_file_infos.append( + { + "file_name": env_file, + "file_absolute_path": os.path.join(extract_dir, self.ENV_FILES_DIR, env_file), + } + ) + + # 获取配置文件信息 + for template in os.listdir(os.path.join(extract_dir, self.TEMPLATE_FILES_DIR)): + template_match: str = self.TEMPLATE_PATTERN.search(template) + + if template_match: + template_file_infos.append( + { + "file_name": template_match.group(), + "file_absolute_path": os.path.join(extract_dir, self.TEMPLATE_FILES_DIR, template), + } + ) + + return {"env": env_file_infos, "templates": template_file_infos} def _inject_dependencies(self, extract_dir: str): """ @@ -240,7 +282,9 @@ def _list_package_dir_infos(self, extract_dir: str) -> typing.List[typing.Dict]: :return: """ package_dir_infos: typing.List[typing.Dict] = [] + for pkg_dir_name in os.listdir(extract_dir): + # 通过正则提取出插件（plugin）目录名中的插件信息 re_match = constants.AGENT_PATH_RE.match(pkg_dir_name) if re_match is None: @@ -346,8 +390,8 @@ def _get_version(self, extract_dir: str) -> str: :param extract_dir: 解压目录 :return: """ - # 优先使用覆盖版本号 - if self.overwrite_version: + # TODO： 适配未开启Agent包管理，优先使用覆盖版本号 + if not self.enable_agent_pkg_manage and self.overwrite_version: return self.overwrite_version version_file_path: str = os.path.join(extract_dir, "VERSION") @@ -385,7 +429,71 @@ def update_or_create_record(self, artifact_meta_info: typing.Dict[str, typing.An """ pass - def update_or_create_package_records(self, package_infos: typing.List[typing.Dict]): + def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]): + """ + 创建或更新标签记录，待 Agent 包管理完善 + :param artifact_meta_info: + :return: + """ + if self.overwrite_version: + TagHandler.publish_tag_version( + name=self.overwrite_version, + target_type=TargetType.AGENT.value, + target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME], + target_version=artifact_meta_info["version"], + ) + + def parset_env(self, content) -> typing.Dict[str, typing.Any]: + config_env = GseConfigParser() + config_env.read_string(f"[Default]\n{content}") + return dict(config_env.items("Default")) + + def update_or_create_support_files(self, package_infos: typing.List[typing.Dict]): + """ + 创建或更新support_files记录 + :param package_infos: + :return: + """ + for package_info in package_infos: + support_files = package_info["artifact_meta_info"]["support_files_info"] + version = package_info["artifact_meta_info"]["version"] + package = package_info["package_dir_info"] + + for env_file in support_files["env"]: + with open(env_file["file_absolute_path"], "r") as f: + env_str = f.read() + + logger.info( + f"update_or_create_support_files: env_flie->{env_file} version->{version} raw_env->{env_str}" + ) + env_value = self.parset_env(env_str) + logger.info( + f"update_or_create_support_files: env_flie->{env_file} version->{version} env_value->{env_value}" + ) + models.GseConfigEnv.objects.update_or_create( + defaults={"env_value": env_value}, + version=version, + os=package["os"], + cpu_arch=package["cpu_arch"], + agent_name=self.NAME, + ) + + for template in support_files["templates"]: + with open(template["file_absolute_path"], "r") as f: + content = f.read() + logger.info( + f"update_or_create_support_files: template->{template} version->{version} content->{content}" + ) + models.GseConfigTemplate.objects.update_or_create( + defaults={"content": content}, + name=template["file_name"], + version=version, + os=package["os"], + cpu_arch=package["cpu_arch"], + agent_name=self.NAME, + ) + + def update_or_create_package_records(self, v): """ 创建或更新安装包记录，待 Agent 包管理完善 :param package_infos: @@ -457,8 +565,11 @@ def make( ) artifact_meta_info["operator"] = operator - self.update_or_create_record(artifact_meta_info) - self.update_or_create_package_records(package_infos) + if self.enable_agent_pkg_manage: + self.update_or_create_support_files(package_infos) + self.update_or_create_tag(artifact_meta_info) + self.update_or_create_record(artifact_meta_info) + self.update_or_create_package_records(package_infos) def __enter__(self) -> "BaseArtifactBuilder": return self diff --git a/apps/backend/agent/artifact_builder/proxy.py b/apps/backend/agent/artifact_builder/proxy.py index 09508b6e8..b2a8f7dd4 100644 --- a/apps/backend/agent/artifact_builder/proxy.py +++ b/apps/backend/agent/artifact_builder/proxy.py @@ -29,7 +29,8 @@ class ProxyArtifactBuilder(base.BaseArtifactBuilder): NAME = constants.GsePackageCode.PROXY.value PKG_DIR = constants.GsePackageDir.PROXY.value CERT_FILENAMES: typing.List[str] = constants.GseCert.list_member_values() - + ENV_FILES = constants.GsePackageEnv.PROXY.value + TEMPLATE_PATTERN = constants.GsePackageTemplatePattern.PROXY.value # 服务二进制目录 SERVER_BIN_DIR: str = "server/bin" # 所需的二进制文件 @@ -95,5 +96,4 @@ def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir return extract_dir def _get_support_files_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]: - # Agent 包管理实现 - pass + return super()._get_support_files_info(extract_dir=extract_dir) diff --git a/apps/backend/agent/config_parser.py b/apps/backend/agent/config_parser.py new file mode 100644 index 000000000..5d4967a5a --- /dev/null +++ b/apps/backend/agent/config_parser.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from configparser import ConfigParser, NoSectionError + +_UNSET = object() + + +class GseConfigParser(ConfigParser): + def optionxform(self, optionstr: str) -> str: + # 返回原始大小写 + return optionstr + + def format_value(self, value): + try: + value = int(value) + except ValueError: + try: + value = float(value) + except ValueError: + pass + return value + + def items(self, section=_UNSET, raw=False, vars=None): + d = self._defaults.copy() + try: + d.update(self._sections[section]) + except KeyError: + if section != self.default_section: + raise NoSectionError(section) + # Update with the entry specific variables + if vars: + for key, value in vars.items(): + d[self.optionxform(key)] = value + value_getter = lambda option: self._interpolation.before_get(self, section, option, d[option], d) # noqa + if raw: + value_getter = lambda option: d[option] # noqa + return [(option, self.format_value(value_getter(option))) for option in d.keys()] diff --git a/apps/backend/exceptions.py b/apps/backend/exceptions.py index 03564e25b..a2e810d9f 100644 --- a/apps/backend/exceptions.py +++ b/apps/backend/exceptions.py @@ -88,3 +88,9 @@ class NotSemanticVersionError(BackendBaseException): class PkgMetaInfoValidationError(BackendBaseException): MESSAGE = _("安装包元数据校验失败") ERROR_CODE = 14 + + +class AgentConfigTemplateEnvNotExistError(BackendBaseException): + MESSAGE = _("配置模板Env不存在") + MESSAGE_TPL = _("配置模板Env不存在[{name}-{version}-{os_type}-{cpu_arch}]不存在") + ERROR_CODE = 15 diff --git a/apps/backend/management/commands/init_agents.py b/apps/backend/management/commands/init_agents.py index eef03f396..98ca49747 100644 --- a/apps/backend/management/commands/init_agents.py +++ b/apps/backend/management/commands/init_agents.py @@ -29,6 +29,13 @@ def add_arguments(self, parser): parser.add_argument("-c", "--cert_path", help=f"证书目录，默认为 {settings.GSE_CERT_PATH}", type=str) parser.add_argument("-d", "--download_path", help=f"归档路径，默认为 {settings.DOWNLOAD_PATH}", type=str) parser.add_argument("-o", "--overwrite_version", help="版本号，用于覆盖原始制品内的版本信息", type=str) + parser.add_argument( + "-e", + "--enable_agent_pkg_manage", + help="是否开启Agent包管理模式, 开启则创建Tag, Template, Evn记录", + action="store_true", + default=False, + ) def handle(self, *args, **options): """ @@ -78,5 +85,6 @@ def handle(self, *args, **options): cert_path=options.get("cert_path"), download_path=options.get("download_path"), overwrite_version=options.get("overwrite_version"), + enable_agent_pkg_manage=options.get("enable_agent_pkg_manage"), ) as builder: builder.make() diff --git a/apps/backend/subscription/steps/agent_adapter/adapter.py b/apps/backend/subscription/steps/agent_adapter/adapter.py index d40f0c46a..b215e6597 100644 --- a/apps/backend/subscription/steps/agent_adapter/adapter.py +++ b/apps/backend/subscription/steps/agent_adapter/adapter.py @@ -18,15 +18,19 @@ from apps.backend import exceptions from apps.backend.agent.tools import fetch_proxies +from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.targets import get_target_helper from apps.node_man import constants, models from apps.utils import cache from env.constants import GseVersion -from . import base, config_templates, legacy +from . import base, legacy from .config_context import context_helper +from .handlers import GseConfigHandler, get_gse_config_handler_class logger = logging.getLogger("app") + LEGACY = "legacy" @@ -67,26 +71,6 @@ def config(self) -> OrderedDict: """ return self.validated_data(data=self.subscription_step.config, serializer=AgentStepConfigSerializer) - @property - @cache.class_member_cache() - def config_tmpl_obj_gby_os_key(self) -> typing.Dict[str, typing.List[base.AgentConfigTemplate]]: - """ - 获取按机型（os_type + cpu_arch）聚合的配置模板 - :return: - """ - agent_config_templates: typing.List[base.AgentConfigTemplate] = [ - base.AgentConfigTemplate(name="gse_agent.conf", content=config_templates.GSE_AGENT_CONFIG_TMPL), - base.AgentConfigTemplate(name="gse_data_proxy.conf", content=config_templates.GSE_DATA_PROXY_CONFIG_TMPL), - base.AgentConfigTemplate(name="gse_file_proxy.conf", content=config_templates.GSE_FILE_PROXY_CONFIG_TEMPL), - ] - return { - # 向 Agent 包管理过渡：AgentConfigTemplate 后续替换为数据模型对象 - self.get_os_key(constants.OsType.LINUX, constants.CpuType.x86): agent_config_templates, - self.get_os_key(constants.OsType.LINUX, constants.CpuType.x86_64): agent_config_templates, - self.get_os_key(constants.OsType.WINDOWS, constants.CpuType.x86): agent_config_templates, - self.get_os_key(constants.OsType.WINDOWS, constants.CpuType.x86_64): agent_config_templates, - } - def get_main_config_filename(self) -> str: return ("gse_agent.conf", "agent.conf")[self.is_legacy] @@ -99,24 +83,19 @@ def _get_config( proxies: typing.List[models.Host], install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]], ) -> str: - config_tmpl_objs: typing.List[base.AgentConfigTemplate] = self.config_tmpl_obj_gby_os_key.get( - self.get_os_key(host.os_type, host.cpu_arch), - [ - base.AgentConfigTemplate(name="gse_agent.conf", content=config_templates.GSE_AGENT_CONFIG_TMPL), - base.AgentConfigTemplate( - name="gse_data_proxy.conf", content=config_templates.GSE_DATA_PROXY_CONFIG_TMPL - ), - base.AgentConfigTemplate( - name="gse_file_proxy.conf", content=config_templates.GSE_FILE_PROXY_CONFIG_TEMPL - ), - ], + agent_setup_info: base.AgentSetupInfo = self.get_setup_info() + + gse_config_handler: GseConfigHandler = get_gse_config_handler_class(node_type)( + os_type=host.os_type, + cpu_arch=host.cpu_arch, + target_version=agent_setup_info.version, ) - # 查找机型匹配的第一个配置 - target_config_tmpl_obj: base.AgentConfigTemplate = next( - (config_tmpl_obj for config_tmpl_obj in config_tmpl_objs if config_tmpl_obj.name == filename), None + + config_tmpl_obj: base.AgentConfigTemplate = gse_config_handler.get_template_by_config_name( + config_name=filename, ) - if not target_config_tmpl_obj: + if not config_tmpl_obj: logger.error( f"{self.log_prefix} agent config template not exist: name -> {self.config['name']}, " f"filename -> {filename}, version -> {self.config['version']}, " @@ -128,14 +107,14 @@ def _get_config( # 渲染配置 ch: context_helper.ConfigContextHelper = context_helper.ConfigContextHelper( - agent_setup_info=self.get_setup_info(), + agent_setup_info=agent_setup_info, host=host, node_type=node_type, ap=ap, proxies=proxies, install_channel=install_channel, ) - return ch.render(target_config_tmpl_obj.content) + return ch.render(config_tmpl_obj.content, gse_config_handler.template_env) def get_config( self, @@ -171,11 +150,24 @@ def get_setup_info(self) -> base.AgentSetupInfo: 获取 Agent 设置信息 :return: """ + # 如果版本号匹配到标签名称，取对应标签下的真实版本号 + try: + target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( + target_id=AGENT_NAME_TARGET_ID_MAP[self.config.get("name")], + target_version=self.config.get("version"), + ) + except KeyError: + logger.info( + f"get_setup_info: get target version failed. " + f"agent_name: {self.config.get('name')} version: {self.config.get('version')}" + ) + target_version: str = self.config.get("version") + return base.AgentSetupInfo( is_legacy=self.is_legacy, agent_tools_relative_dir=("agent_tools/agent2", "")[self.is_legacy], name=self.config.get("name"), - version=self.config.get("version"), + version=target_version, ) @staticmethod diff --git a/apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py b/apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py index 901ad6780..3926efd6e 100644 --- a/apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py +++ b/apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py @@ -14,6 +14,7 @@ import re import typing from dataclasses import asdict, dataclass, field +from typing import Any, Dict from django.conf import settings @@ -221,7 +222,7 @@ def __post_init__(self): for context in contexts: self.context_dict.update(asdict(context, dict_factory=context.dict_factory)) - def render(self, content: str) -> str: + def render(self, content: str, template_env: Dict[str, Any] = {}) -> str: """ 渲染并返回配置 :param content: 配置模板内容 @@ -232,5 +233,8 @@ def _double(_matched) -> str: _env_name: str = str(_matched.group()) return "{{ " + _env_name[2:-2] + " }}" + # 先加载配置模板默认变量配置 + context_dict = template_env + context_dict.update(self.context_dict) content = re.sub(r"(__[0-9A-Z_]+__)", _double, content) - return nested_render_data(content, self.context_dict) + return nested_render_data(content, context_dict) diff --git a/apps/backend/subscription/steps/agent_adapter/handlers.py b/apps/backend/subscription/steps/agent_adapter/handlers.py new file mode 100644 index 000000000..fd64e0176 --- /dev/null +++ b/apps/backend/subscription/steps/agent_adapter/handlers.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging +from typing import Dict, List, Tuple + +from apps.backend import exceptions +from apps.node_man import constants +from apps.node_man.constants import GsePackageCode, NodeType +from apps.node_man.models import GlobalSettings, GseConfigEnv, GseConfigTemplate +from apps.utils import cache + +from . import config_templates +from .base import AgentConfigTemplate + +logger = logging.getLogger("app") + + +class GseConfigHandler: + AGENT_NAME = None + + def __init__(self, os_type: str, cpu_arch: str, target_version: str) -> None: + self.os_type = os_type + self.cpu_arch = cpu_arch + self.target_version = target_version + self.enable_agent_pkg_manage: bool = GlobalSettings.get_config( + key=GlobalSettings.KeyEnum.ENABLE_AGENT_PKG_MANAGE.value, default=False + ) + + @staticmethod + def get_os_key(os_type: str, cpu_arch: str) -> str: + # 默认为 linux-x86_64，兼容CMDB同步过来的主机没有操作系统和CPU架构的场景 + os_type = os_type or constants.OsType.LINUX + cpu_arch = cpu_arch or constants.CpuType.x86_64 + return f"{os_type.lower()}-{cpu_arch}" + + @property + @cache.class_member_cache() + def agent_config_templates(self) -> List[AgentConfigTemplate]: + return [ + AgentConfigTemplate(name="gse_agent.conf", content=config_templates.GSE_AGENT_CONFIG_TMPL), + AgentConfigTemplate(name="gse_data_proxy.conf", content=config_templates.GSE_DATA_PROXY_CONFIG_TMPL), + AgentConfigTemplate(name="gse_file_proxy.conf", content=config_templates.GSE_FILE_PROXY_CONFIG_TEMPL), + ] + + @property + @cache.class_member_cache() + def config_tmpl_obj_gby_os_key(self) -> Dict[str, List[AgentConfigTemplate]]: + """ + 获取按机型（os_type + cpu_arch）聚合的配置模板 + :return: + """ + return { + # 向 Agent 包管理过渡：AgentConfigTemplate 后续替换为数据模型对象 + self.get_os_key(constants.OsType.LINUX, constants.CpuType.x86): self.agent_config_templates, + self.get_os_key(constants.OsType.LINUX, constants.CpuType.x86_64): self.agent_config_templates, + self.get_os_key(constants.OsType.WINDOWS, constants.CpuType.x86): self.agent_config_templates, + self.get_os_key(constants.OsType.WINDOWS, constants.CpuType.x86_64): self.agent_config_templates, + } + + def _get_template_from_file(self, config_name) -> Tuple[bool, AgentConfigTemplate]: + config_tmpl_objs: List[AgentConfigTemplate] = self.config_tmpl_obj_gby_os_key.get( + self.get_os_key(self.os_type, self.cpu_arch), + self.agent_config_templates, + ) + # 查找机型匹配的第一个配置 + target_config_tmpl_obj: AgentConfigTemplate = next( + (config_tmpl_obj for config_tmpl_obj in config_tmpl_objs if config_tmpl_obj.name == config_name), None + ) + if not target_config_tmpl_obj: + return False, None + return True, target_config_tmpl_obj + + def _get_template_from_db(self, config_name) -> Tuple[bool, AgentConfigTemplate]: + try: + template = GseConfigTemplate.objects.get( + os=self.os_type.lower(), cpu_arch=self.cpu_arch, version=self.target_version, name=config_name + ) + except GseConfigTemplate.DoesNotExist: + return False, None + + return True, AgentConfigTemplate(name=config_name, content=template.content) + + def get_template_by_config_name(self, config_name: str) -> AgentConfigTemplate: + # 兼容原运行模式 + is_ok, config_tmpl_obj = [self._get_template_from_file, self._get_template_from_db][ + self.enable_agent_pkg_manage + ](config_name=config_name) + if not is_ok: + logger.error( + f"Agent config template not exist: name -> {self.AGENT_NAME}, " + f"filename -> {config_name}, version -> {self.target_version}, " + f"os_type -> {self.os_type}, cpu_arch -> {self.cpu_arch}" + ) + raise exceptions.AgentConfigTemplateNotExistError( + name=self.AGENT_NAME, filename=config_name, os_type=self.os_type, cpu_arch=self.cpu_arch + ) + + return config_tmpl_obj + + @property + def template_env(self) -> Dict: + if not self.enable_agent_pkg_manage: + # 兼容 未开启Agent包管理情况 + return {} + try: + template_env: GseConfigEnv = GseConfigEnv.objects.get( + agent_name=self.AGENT_NAME, + os=self.os_type, + cpu_arch=self.cpu_arch, + version=self.target_version, + ) + except GseConfigEnv.DoesNotExist: + logger.error( + f"Agent config template env not exist: name -> {self.AGENT_NAME}, " + f"version -> {self.target_version}, os_type -> {self.os_type}, cpu_arch -> {self.cpu_arch}" + ) + raise exceptions.AgentConfigTemplateEnvNotExistError( + name=self.AGENT_NAME, + version=self.target_version, + os_type=self.os_type, + cpu_arch=self.cpu_arch, + ) + return template_env.env_value + + +class AgentGseConfigHandler(GseConfigHandler): + AGENT_NAME = GsePackageCode.AGENT.value + + +class ProxyGseConfigHandler(GseConfigHandler): + AGENT_NAME = GsePackageCode.PROXY.value + + +GSE_CONFIG_HANDLER_PACKAGE_CODE_MAP: Dict = { + NodeType.AGENT.lower(): AgentGseConfigHandler, + NodeType.PROXY.lower(): ProxyGseConfigHandler, +} + + +def get_gse_config_handler_class(node_type: str) -> GseConfigHandler: + return GSE_CONFIG_HANDLER_PACKAGE_CODE_MAP[node_type] diff --git a/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/__init__.py b/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/__init__.py new file mode 100644 index 000000000..29ed269e0 --- /dev/null +++ b/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_agent.py b/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_agent.py new file mode 100644 index 000000000..753b30d9c --- /dev/null +++ b/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_agent.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from apps.backend.tests.agent import utils +from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.models import Tag +from apps.node_man import models + +from .. import test_agent + + +class FileSystemTestCase(test_agent.FileSystemTestCase): + @classmethod + def setUpClass(cls): + # 开启Agent包管理 + super().setUpClass() + models.GlobalSettings.update_config(models.GlobalSettings.KeyEnum.ENABLE_AGENT_PKG_MANAGE.value, True) + + def tag_checker(self, target_id: int): + + agent_target_version = Tag.objects.get( + target_id=target_id, + name=self.OVERWRITE_VERSION, + target_type=TargetType.AGENT.value, + ).target_version + + self.assertTrue(agent_target_version == utils.VERSION) + + def template_and_env_checker(self, version_str): + for package_os, cpu_arch in self.OS_CPU_CHOICES: + filter_kwargs: dict = { + "agent_name": self.NAME, + "os": package_os, + "cpu_arch": cpu_arch, + "version": version_str, + } + self.assertTrue(models.GseConfigEnv.objects.filter(**filter_kwargs).exists()) + self.assertTrue(models.GseConfigTemplate.objects.filter(**filter_kwargs).exists()) + + def test_make__overwrite_version(self): + """测试版本号覆盖""" + with self.ARTIFACT_BUILDER_CLASS( + initial_artifact_path=self.ARCHIVE_PATH, overwrite_version=self.OVERWRITE_VERSION + ) as builder: + builder.make() + + # 测试 + self.pkg_checker(version_str=utils.VERSION) + self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.template_and_env_checker(version_str=utils.VERSION) + + +class BkRepoTestCase(FileSystemTestCase, test_agent.BkRepoTestCase): + pass diff --git a/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_manage_commands.py b/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_manage_commands.py new file mode 100644 index 000000000..fa464385b --- /dev/null +++ b/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_manage_commands.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from apps.backend.tests.agent import utils +from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.models import Tag +from apps.node_man import models +from apps.node_man.constants import GsePackageCode + +from .. import test_manage_commands + + +class FileSystemImportAgentTestCase(test_manage_commands.FileSystemImportAgentTestCase): + @classmethod + def setUpClass(cls): + # 开启Agent包管理 + super().setUpClass() + models.GlobalSettings.update_config(models.GlobalSettings.KeyEnum.ENABLE_AGENT_PKG_MANAGE.value, True) + + def tag_checker(self, target_id: int): + + agent_target_version = Tag.objects.get( + target_id=target_id, + name=self.OVERWRITE_VERSION, + target_type=TargetType.AGENT.value, + ).target_version + + self.assertTrue(agent_target_version == utils.VERSION) + + def test_make__overwrite_version(self): + """测试版本号覆盖""" + with self.ARTIFACT_BUILDER_CLASS( + initial_artifact_path=self.ARCHIVE_PATH, overwrite_version=self.OVERWRITE_VERSION + ) as builder: + builder.make() + + # 测试 + self.pkg_checker(version_str=utils.VERSION) + self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[GsePackageCode.AGENT.value]) + + +class BkRepoImportAgentTestCase(FileSystemImportAgentTestCase, test_manage_commands.BkRepoImportAgentTestCase): + pass diff --git a/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_proxy.py b/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_proxy.py new file mode 100644 index 000000000..e64ad4604 --- /dev/null +++ b/apps/backend/tests/agent/artifact_builder/enable_agent_pkg_manage/test_proxy.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from .. import test_proxy +from . import test_agent + + +class FileSystemTestCase( + test_proxy.FileSystemTestCase, + test_agent.FileSystemTestCase, +): + pass + + +class BkRepoTestCase(FileSystemTestCase): + pass diff --git a/apps/backend/tests/agent/artifact_builder/test_agent.py b/apps/backend/tests/agent/artifact_builder/test_agent.py index dc2f69455..6246854ae 100644 --- a/apps/backend/tests/agent/artifact_builder/test_agent.py +++ b/apps/backend/tests/agent/artifact_builder/test_agent.py @@ -12,15 +12,26 @@ import mock from django.conf import settings +from django.db.utils import IntegrityError from apps.backend.tests.agent import utils from apps.mock_data import utils as mock_data_utils +from apps.node_man import models class FileSystemTestCase(utils.AgentBaseTestCase): OVERWRITE_VERSION = "stable" + @classmethod + def setUpClass(cls): + # 关闭Agent包管理 + try: + models.GlobalSettings.set_config(models.GlobalSettings.KeyEnum.ENABLE_AGENT_PKG_MANAGE.value, False) + except IntegrityError: + models.GlobalSettings.update_config(models.GlobalSettings.KeyEnum.ENABLE_AGENT_PKG_MANAGE.value, False) + super().setUpClass() + def pkg_checker(self, version_str: str): """ 安装包检查 diff --git a/apps/backend/tests/agent/template_env.py b/apps/backend/tests/agent/template_env.py new file mode 100644 index 000000000..06d158411 --- /dev/null +++ b/apps/backend/tests/agent/template_env.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +DEFAULT_AGENT_TEMPLATE_ENV = """ +# 安装部署根目录 +BK_GSE_HOME_DIR="/usr/local/gse/agent" + +# 集群信息(BK_GSE_RUN_MODE为'agent'表示AGENT模式，'proxy'表示PROXY模式, 其他值当前默认以AGENT模式运行) +BK_GSE_RUN_MODE="agent" +BK_GSE_CLOUD_ID=0 +BK_GSE_ZONE_ID="default" +BK_GSE_CITY_ID="default" + +# 接入点相关配置(多endpoint使用逗号分隔) +## 非Proxy模式下将根据access endpoints进行动态接入 +## Proxy模式下将基于access endpoints进行静态接入 +BK_GSE_ENABLE_STATIC_ACCESS=false +BK_GSE_ACCESS_CLUSTER_ENDPOINTS="127.0.0.1:28668" +BK_GSE_ACCESS_DATA_ENDPOINTS="127.0.0.1:28625" +BK_GSE_ACCESS_FILE_ENDPOINTS="127.0.0.1:28925" + +# AgentBase基础配置(TLS为Client侧证书用于和上级建立链接请求) +BK_GSE_AGENT_BASE_TLS_CA_FILE="" +BK_GSE_AGENT_BASE_TLS_CERT_FILE="" +BK_GSE_AGENT_BASE_TLS_KEY_FILE="" +BK_GSE_AGENT_BASE_TLS_PASSWORD_FILE="" +BK_GSE_AGENT_BASE_PROCESSOR_NUM=4 +BK_GSE_AGENT_BASE_PROCESSOR_SIZE=4096 + +# Proxy相关配置(TLS为Server侧证书用于接受处理下级链接请求) +BK_GSE_PROXY_TLS_CA_FILE="" +BK_GSE_PROXY_TLS_CERT_FILE="" +BK_GSE_PROXY_TLS_KEY_FILE="" +BK_GSE_PROXY_TLS_PASSWORD_FILE="" +BK_GSE_PROXY_BIND_IP="::" +BK_GSE_PROXY_BIND_PORT=28668 +BK_GSE_PROXY_THREAD_NUM=4 + +# Task相关配置 +BK_GSE_TASK_PROC_EVENT_DATA_ID=1100008 +BK_GSE_TASK_CONCURRENCE_COUNT=100 +BK_GSE_TASK_QUEUE_WAIT_TIMEOUT_MS=5000 +BK_GSE_TASK_EXECUTOR_QUEUE_SIZE=4096 +BK_GSE_TASK_SCHEDULE_QUEUE_SIZE=4096 +BK_GSE_TASK_HOST_CODE_PAGE_NAME="utf8" +BK_GSE_TASK_SCRIPT_FILE_CLEAN_BATCH_COUNT=100 +BK_GSE_TASK_SCRIPT_FILE_CLEAN_STARTUP_CLOCK_TIME=0 +BK_GSE_TASK_SCRIPT_FILE_EXPIRE_TIME_HOUR=72 +BK_GSE_TASK_SCRIPT_FILE_PREFIX="bk_gse_script_" + +# Data相关配置 (windows环境下需将BK_GSE_DATA_IPC设置为本地端口27000) +BK_GSE_DATA_IPC="${BK_GSE_HOME_DIR}/data/ipc.state.report" +BK_GSE_DATA_IPC_RECV_THREAD_NUM=4 +BK_GSE_DATA_ENABLE_COMPRESSION=false + +# File相关配置 +BK_GSE_FILE_MAX_TRANSFER_SPEED_MB_PER_SEC=100 +BK_GSE_FILE_MAX_TRANSFER_CONCURRENT_NUM=10 + +# 日志相关配置 +BK_GSE_LOG_PATH="${BK_GSE_HOME_DIR}/logs" +BK_GSE_LOG_LEVEL="INFO" +BK_GSE_LOG_FILESIZE_MB=200 +BK_GSE_LOG_FILENUM=10 +BK_GSE_LOG_ROTATE=0 +BK_GSE_LOG_FLUSH_INTERVAL_MS=100 + +# 自定义扩展配置 +BK_GSE_EXTRA_CONFIG_DIRECTORY="" +""" + +DEFAULT_PROXY_TEMPLATE_ENV = """ +# 安装部署根目录 +BK_GSE_HOME_DIR="/usr/local/gse/proxy" + +# 集群信息 +BK_GSE_CLOUD_ID=0 + +# 数据服务: 对接2.0 AGENT服务 +BK_GSE_DATA_AGENT_TCP_BIND_IP="::" +BK_GSE_DATA_AGENT_TCP_BIND_PORT=28625 +BK_GSE_DATA_AGENT_TCP_SERVER_THREAD_NUM=24 +BK_GSE_DATA_AGENT_TCP_SERVER_MAX_MESSAGE_SIZE=10485760 +BK_GSE_DATA_AGENT_TLS_CA_FILE="" +BK_GSE_DATA_AGENT_TLS_CERT_FILE="" +BK_GSE_DATA_AGENT_TLS_KEY_FILE="" +BK_GSE_DATA_AGENT_TLS_PASSWORD_FILE="" + +# 数据服务: 数据转发对端服务 +BK_GSE_DATA_PROXY_TLS_CA_FILE="" +BK_GSE_DATA_PROXY_TLS_CERT_FILE="" +BK_GSE_DATA_PROXY_TLS_KEY_FILE="" +BK_GSE_DATA_PROXY_TLS_PASSWORD_FILE="" +BK_GSE_DATA_PROXY_ENDPOINTS="127.0.0.1:28625" + +# 数据服务: metric(healthz)采集配置 +BK_GSE_DATA_METRIC_EXPORTER_BIND_IP="::" +BK_GSE_DATA_METRIC_EXPORTER_BIND_PORT=59402 +BK_GSE_DATA_METRIC_EXPORTER_THREAD_NUM=8 + +# 文件服务: Agent侧接口相关配置 +BK_GSE_FILE_AGENT_BIND_IP="::" +BK_GSE_FILE_AGENT_BIND_PORT=28925 +BK_GSE_FILE_AGENT_BIND_PORT_V1=58925 +BK_GSE_FILE_AGENT_ADVERTISE_IPV4="127.0.0.1" +BK_GSE_FILE_AGENT_ADVERTISE_IPV6="::1" +BK_GSE_FILE_AGENT_THREAD_NUM=24 +BK_GSE_FILE_AGENT_TLS_CA_FILE="" +BK_GSE_FILE_AGENT_TLS_CERT_FILE="" +BK_GSE_FILE_AGENT_TLS_KEY_FILE="" +BK_GSE_FILE_AGENT_TLS_PASSWORD_FILE="" + +# 文件服务: Bittorrent相关配置 +BK_GSE_FILE_BITTORRENT_BIND_IP="::" +BK_GSE_FILE_BITTORRENT_BIND_PORT=10020 +BK_GSE_FILE_BITTORRENT_TRACKER_BIND_PORT=10030 +BK_GSE_FILE_BITTORRENT_SPEED_LIMIT_MB_PER_SEC=10000 + +# 文件服务: Topology相关配置 +BK_GSE_FILE_TOPOLOGY_BIND_IP="::" +BK_GSE_FILE_TOPOLOGY_BIND_PORT=28930 +BK_GSE_FILE_TOPOLOGY_THRIFT_BIND_PORT=58930 +BK_GSE_FILE_TOPOLOGY_ADVERTISE_IP="127.0.0.1" +BK_GSE_FILE_TOPOLOGY_THREAD_NUM=4 +BK_GSE_FILE_TOPOLOGY_TLS_CA_FILE="" +BK_GSE_FILE_TOPOLOGY_TLS_PASSWORD_FILE="" +BK_GSE_FILE_TOPOLOGY_TLS_SVR_CERT_FILE="" +BK_GSE_FILE_TOPOLOGY_TLS_SVR_KEY_FILE="" +BK_GSE_FILE_TOPOLOGY_TLS_CLI_CERT_FILE="" +BK_GSE_FILE_TOPOLOGY_TLS_CLI_KEY_FILE="" + +# 文件服务: Proxy相关配置 +BK_GSE_FILE_PROXY_UPSTREAM_IP="127.0.0.1" +BK_GSE_FILE_PROXY_UPSTREAM_PORT=28930 +BK_GSE_FILE_PROXY_REPORT_IP="127.0.0.1" +BK_GSE_FILE_PROXY_REPORT_PORT=28930 + +# 文件服务: Cache文件缓存相关配置 +BK_GSE_FILE_CACHE_DIRS="./file_cache" +BK_GSE_FILE_CACHE_EXPIRED_TIME_SEC=7200 + +# 文件服务: metric(healthz)采集配置 +BK_GSE_FILE_METRIC_EXPORTER_BIND_IP="::" +BK_GSE_FILE_METRIC_EXPORTER_BIND_PORT=59404 +BK_GSE_FILE_METRIC_EXPORTER_THREAD_NUM=8 + +# 日志保存路径 +BK_GSE_LOG_PATH="${BK_GSE_HOME_DIR}/logs" +BK_GSE_LOG_LEVEL="INFO" +BK_GSE_LOG_FILESIZE_MB=200 +BK_GSE_LOG_FILENUM=10 +BK_GSE_LOG_ROTATE=0 +BK_GSE_LOG_FLUSH_INTERVAL_MS=100 +""" diff --git a/apps/backend/tests/agent/utils.py b/apps/backend/tests/agent/utils.py index 26538da23..2916e948a 100644 --- a/apps/backend/tests/agent/utils.py +++ b/apps/backend/tests/agent/utils.py @@ -31,6 +31,8 @@ from apps.utils.enum import EnhanceEnum from apps.utils.unittest.testcase import CustomAPITestCase +from . import template_env + VERSION = common_unit.plugin.PACKAGE_VERSION @@ -57,6 +59,7 @@ def get_setting_name__path_suffix_map(cls) -> typing.Dict[str, str]: class AgentBaseTestCase(CustomAPITestCase): TMP_DIR: str = None PKG_NAME: str = None + NAME: str = "gse_agent" OVERWRITE_VERSION: str = None ARCHIVE_NAME: str = f"gse_agent_ce-{VERSION}.tgz" ARCHIVE_PATH: str = None @@ -92,7 +95,6 @@ def setUpClass(cls): } cls.OVERWRITE_OBJ__KV_MAP = cls.OVERWRITE_OBJ__KV_MAP or {} cls.OVERWRITE_OBJ__KV_MAP[settings] = {**cls.OVERWRITE_OBJ__KV_MAP.get(settings, {}), **setting_name__path_map} - super().setUpClass() @classmethod @@ -179,6 +181,10 @@ def gen_base_artifact_files(cls, os_cpu_choices: typing.List[typing.Tuple[str, s with open(os.path.join(pkg_conf_tmpls_dir, "gse_agent_conf.template"), "w", encoding="utf-8") as templ_fs: templ_fs.write(config_templates.GSE_AGENT_CONFIG_TMPL) + # 写入环境变量 + with open(os.path.join(pkg_conf_env_dir, "gse_agent.env"), "w", encoding="utf-8") as templ_fs: + templ_fs.write(template_env.DEFAULT_AGENT_TEMPLATE_ENV) + return base_artifact_dir @classmethod @@ -206,6 +212,7 @@ def tearDownClass(cls): class ProxyBaseTestCase(AgentBaseTestCase): + NAME: str = "gse_proxy" ARCHIVE_NAME: str = f"gse_ce-{VERSION}.tgz" OS_CPU_CHOICES = [ (constants.OsType.LINUX.lower(), constants.CpuType.x86_64), @@ -247,6 +254,10 @@ def gen_base_artifact_files(cls, os_cpu_choices: typing.List[typing.Tuple[str, s with open(os.path.join(pkg_conf_tmpls_dir, templ_name), "w", encoding="utf-8") as version_fs: version_fs.write(config_templates.GSE_FILE_PROXY_CONFIG_TEMPL) + # 写入环境变量 + with open(os.path.join(pkg_conf_env_dir, "gse_proxy.env"), "w", encoding="utf-8") as templ_fs: + templ_fs.write(template_env.DEFAULT_PROXY_TEMPLATE_ENV) + return base_artifact_dir @classmethod diff --git a/apps/core/tag/constants.py b/apps/core/tag/constants.py index cf922484a..8b0c27614 100644 --- a/apps/core/tag/constants.py +++ b/apps/core/tag/constants.py @@ -13,6 +13,7 @@ from django.utils.translation import ugettext_lazy as _ +from apps.node_man.constants import GsePackageCode from apps.utils.enum import EnhanceEnum @@ -36,3 +37,10 @@ class TagChangeAction(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DELETE: _("删除标签"), cls.CREATE: _("新建标签"), cls.UPDATE: _("更新版本"), cls.OVERWRITE: _("同版本覆盖更新")} + + +# TODO: target_id 临时写死 +AGENT_NAME_TARGET_ID_MAP: Dict[str, int] = { + GsePackageCode.AGENT.value: 1, + GsePackageCode.PROXY.value: 2, +} diff --git a/apps/core/tag/targets/__init__.py b/apps/core/tag/targets/__init__.py index 020e12277..75de9134f 100644 --- a/apps/core/tag/targets/__init__.py +++ b/apps/core/tag/targets/__init__.py @@ -13,6 +13,7 @@ import typing from .. import constants, exceptions +from .agent import AgentTargetHelper from .base import BaseTargetHelper from .plugin import PluginTargetHelper @@ -20,7 +21,8 @@ TARGET_TYPE__HELPER_MAP: typing.Dict[typing.Any, typing.Type[BaseTargetHelper]] = { - constants.TargetType.PLUGIN.value: PluginTargetHelper + constants.TargetType.PLUGIN.value: PluginTargetHelper, + constants.TargetType.AGENT.value: AgentTargetHelper, } diff --git a/apps/core/tag/targets/agent.py b/apps/core/tag/targets/agent.py new file mode 100644 index 000000000..400acd7ac --- /dev/null +++ b/apps/core/tag/targets/agent.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +import logging + +from apps.core.tag.models import Tag + +from .. import constants +from . import base + +logger = logging.getLogger("app") + + +class AgentTargetHelper(base.BaseTargetHelper): + + MODEL = None + TARGET_TYPE = constants.TargetType.AGENT.value + + def _publish_tag_version(self): + Tag.objects.update_or_create( + defaults={"target_version": self.target_version}, + name=self.tag_name, + target_id=self.target_id, + target_type=self.TARGET_TYPE, + ) + + def _delete_tag_version(self): + return super()._delete_tag_version() diff --git a/apps/core/tag/targets/base.py b/apps/core/tag/targets/base.py index 079aad2e6..e1ae778a5 100644 --- a/apps/core/tag/targets/base.py +++ b/apps/core/tag/targets/base.py @@ -16,6 +16,8 @@ from django.db import transaction from django.db.models import Model +from apps.core.tag.models import Tag +from apps.node_man.models import GlobalSettings from apps.utils import cache from .. import constants, exceptions, models @@ -135,6 +137,11 @@ class BaseTargetHelper(abc.ABC): # 标签对应的目标版本 target_version: str = None + @classmethod + @cache.class_member_cache() + def enable_agent_pkg_manage(cls) -> bool: + return GlobalSettings.get_config(key=GlobalSettings.KeyEnum.ENABLE_AGENT_PKG_MANAGE, default=False) + @classmethod def get_target(cls, target_id: int): """ @@ -156,7 +163,9 @@ def get_tag(cls, target_id: int, target_version: str) -> models.Tag: :return: """ try: - return models.Tag.objects.get(target_id=target_id, type=cls.TARGET_TYPE, target_version=target_version) + return models.Tag.objects.get( + target_id=target_id, target_type=cls.TARGET_TYPE, target_version=target_version + ) except models.Tag.DoesNotExist: raise exceptions.TagNotExistError({"target_id": target_id, "target_type": cls.TARGET_TYPE}) @@ -181,6 +190,20 @@ def get_top_tag_or_none(cls, target_id: int) -> typing.Optional[models.Tag]: """ return models.Tag.objects.filter(target_id=target_id, to_top=True).first() + @classmethod + def get_target_version(cls, target_id: int, target_version: str) -> str: + if not cls.enable_agent_pkg_manage: + return target_version + + tag_name__obj_map: typing.Dict[str, Tag] = cls.get_tag_name__obj_map( + target_id=target_id, + ) + # 如果版本号匹配到标签名称，取对应标签下的真实版本号 + if target_version in tag_name__obj_map: + target_version: str = tag_name__obj_map[target_version].target_version + + return target_version + def __str__(self): return f"[{self.__class__.__name__}({self.tag_name}|{self.target_id}|{self.target_version})]" diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index 818cd0237..2173a6299 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -546,7 +546,7 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: VERSION_PATTERN = re.compile(r"[vV]?(\d+\.){1,5}\d+(-rc\d)?$") # 语义化版本正则，参考：https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string SEMANTIC_VERSION_PATTERN = re.compile( - r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)" + r"^v?(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)" r"(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?" r"(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" ) @@ -912,6 +912,27 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.PROXY: _("2.0 Proxy Agent 安装包代号"), cls.AGENT: _("2.0 Agent 安装包代号")} +class GsePackageEnv(EnhanceEnum): + """安装包Env文件名称""" + + PROXY = ["gse_proxy.env"] + AGENT = ["gse_agent.env"] + + +class GsePackageTemplate(EnhanceEnum): + """安装包Template文件名称""" + + PROXY = ["gse_data_proxy.conf", "gse_file_proxy.conf"] + AGENT = ["gse_agent.conf"] + + +class GsePackageTemplatePattern(EnhanceEnum): + """安装包Template Pattern""" + + PROXY = re.compile("|".join(GsePackageTemplate.PROXY.value)) + AGENT = re.compile("|".join(GsePackageTemplate.AGENT.value)) + + class GsePackageDir(EnhanceEnum): """安装包打包根路径""" diff --git a/apps/node_man/migrations/0072_gseconfigenv_gseconfigtemplate.py b/apps/node_man/migrations/0072_gseconfigenv_gseconfigtemplate.py new file mode 100644 index 000000000..db50aa1ee --- /dev/null +++ b/apps/node_man/migrations/0072_gseconfigenv_gseconfigtemplate.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0071_update_ap_gse_version_to_v2"), + ] + + operations = [ + migrations.CreateModel( + name="GseConfigEnv", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("env_value", models.TextField(verbose_name="环境变量值")), + ("agent_name", models.CharField(max_length=32, verbose_name="Agent名称")), + ("version", models.CharField(max_length=128, verbose_name="版本号")), + ( + "cpu_arch", + models.CharField( + choices=[ + ("x86", "x86"), + ("x86_64", "x86_64"), + ("powerpc", "powerpc"), + ("aarch64", "aarch64"), + ("sparc", "sparc"), + ], + db_index=True, + default="x86_64", + max_length=32, + verbose_name="CPU类型", + ), + ), + ( + "os", + models.CharField( + choices=[("windows", "windows"), ("linux", "linux"), ("aix", "aix"), ("solaris", "solaris")], + db_index=True, + default="linux", + max_length=32, + verbose_name="系统类型", + ), + ), + ], + ), + migrations.CreateModel( + name="GseConfigTemplate", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(max_length=32, verbose_name="配置文件名称")), + ("content", models.TextField(verbose_name="配置内容")), + ("agent_name", models.CharField(max_length=32, verbose_name="Agent名称")), + ("version", models.CharField(max_length=128, verbose_name="版本号")), + ( + "cpu_arch", + models.CharField( + choices=[ + ("x86", "x86"), + ("x86_64", "x86_64"), + ("powerpc", "powerpc"), + ("aarch64", "aarch64"), + ("sparc", "sparc"), + ], + db_index=True, + default="x86_64", + max_length=32, + verbose_name="CPU类型", + ), + ), + ( + "os", + models.CharField( + choices=[("windows", "windows"), ("linux", "linux"), ("aix", "aix"), ("solaris", "solaris")], + db_index=True, + default="linux", + max_length=32, + verbose_name="系统类型", + ), + ), + ], + ), + ] diff --git a/apps/node_man/migrations/0073_auto_20230815_1550.py b/apps/node_man/migrations/0073_auto_20230815_1550.py new file mode 100644 index 000000000..af9239caa --- /dev/null +++ b/apps/node_man/migrations/0073_auto_20230815_1550.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0072_gseconfigenv_gseconfigtemplate"), + ] + + operations = [ + migrations.AlterModelOptions( + name="gseconfigenv", + options={"verbose_name": "安装包环境变量", "verbose_name_plural": "安装包环境变量"}, + ), + migrations.AlterModelOptions( + name="gseconfigtemplate", + options={"verbose_name": "安装包模板表", "verbose_name_plural": "安装包模板表"}, + ), + migrations.AlterField( + model_name="gseconfigenv", + name="env_value", + field=models.JSONField(verbose_name="环境变量值"), + ), + ] diff --git a/apps/node_man/migrations/0074_merge_20230818_1214.py b/apps/node_man/migrations/0074_merge_20230818_1214.py new file mode 100644 index 000000000..71b87b74b --- /dev/null +++ b/apps/node_man/migrations/0074_merge_20230818_1214.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0072_subscription_instance_selector"), + ("node_man", "0073_auto_20230815_1550"), + ] + + operations = [] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 6b6affa6d..d388cd394 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -134,6 +134,8 @@ class KeyEnum(Enum): SETUP_PAGENT_SCRIPT_FILENAME = "SETUP_PAGENT_SCRIPT_FILENAME" # 周期删除订阅任务数据 CLEAN_SUBSCRIPTION_DATA_MAP = "CLEAN_SUBSCRIPTION_DATA_MAP" + # 是否开启Agent包管理 + ENABLE_AGENT_PKG_MANAGE = "ENABLE_AGENT_PKG_MANAGE" key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True) v_json = JSONField(_("值")) @@ -2363,3 +2365,58 @@ def __str__(self): f"<{self.__class__.__name__}({self.pk}) " f"info -> [{self.bk_event_type}|{self.bk_resource}|{self.bk_detail.get('bk_biz_id')}]>" ) + + +class GseConfigEnv(models.Model): + env_value = models.JSONField(_("环境变量值")) + agent_name = models.CharField(_("Agent名称"), max_length=32) + version = models.CharField(_("版本号"), max_length=128) + cpu_arch = models.CharField( + _("CPU类型"), max_length=32, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True + ) + os = models.CharField( + _("系统类型"), + max_length=32, + choices=constants.PLUGIN_OS_CHOICES, + default=constants.PluginOsType.linux, + db_index=True, + ) + + class Meta: + verbose_name = _("安装包环境变量") + verbose_name_plural = _("安装包环境变量") + + def __str__(self) -> str: + return f"{self.version}-{self.agent_name}-{self.os}-{self.cpu_arch}" + + +class GseConfigTemplate(models.Model): + name = models.CharField(_("配置文件名称"), max_length=32) + content = models.TextField(_("配置内容")) + agent_name = models.CharField(_("Agent名称"), max_length=32) + version = models.CharField(_("版本号"), max_length=128) + cpu_arch = models.CharField( + _("CPU类型"), max_length=32, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True + ) + os = models.CharField( + _("系统类型"), + max_length=32, + choices=constants.PLUGIN_OS_CHOICES, + default=constants.PluginOsType.linux, + db_index=True, + ) + + @property + def env_values(self) -> GseConfigEnv: + if not getattr(self, "_env_values", None): + self._env_values = GseConfigEnv.objects.get( + os=self.os, version=self.version, cpu_arch=self.cpu_arch, agent_name=self.agent_name + ) + return self._env_values + + class Meta: + verbose_name = _("安装包模板表") + verbose_name_plural = _("安装包模板表") + + def __str__(self) -> str: + return f"{self.name}-{self.version}-{self.agent_name}-{self.os}-{self.cpu_arch}"