From 4ef397c0fc6ac6f414ac59ee8fa340e1cc53e0fd Mon Sep 17 00:00:00 2001 From: wyyalt Date: Wed, 27 Dec 2023 15:30:33 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20=20Agent=202.0=20Proxy=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E7=BB=93=E6=9E=84=E5=8F=98=E6=9B=B4=E3=80=81=E5=8D=87?= =?UTF-8?q?=E7=BA=A7=E3=80=81=E9=87=8D=E8=BD=BD=E9=80=82=E9=85=8D=20(close?= =?UTF-8?q?d=20#2033)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/agent/manager.py | 5 ++- .../agent_new/render_and_push_gse_config.py | 36 +++++++++++++++++-- apps/backend/subscription/steps/agent.py | 4 ++- .../steps/agent_adapter/adapter.py | 32 +++++++++++++---- .../config_context/context_dataclass.py | 6 ++++ .../config_context/context_helper.py | 2 ++ .../steps/agent_adapter/legacy.py | 1 + 7 files changed, 75 insertions(+), 11 deletions(-) diff --git a/apps/backend/agent/manager.py b/apps/backend/agent/manager.py index 97ad0c99d..1fa8393d3 100644 --- a/apps/backend/agent/manager.py +++ b/apps/backend/agent/manager.py @@ -191,9 +191,12 @@ def start_nginx(cls): return act @classmethod - def render_and_push_gse_config(cls, name=components.RenderAndPushGseConfigComponent.name): + def render_and_push_gse_config( + cls, is_need_request_agent_version: bool = False, name=components.RenderAndPushGseConfigComponent.name + ): """渲染并下载 agent 配置""" act = AgentServiceActivity(component_code=components.RenderAndPushGseConfigComponent.code, name=name) + act.component.inputs.is_need_request_agent_version = Var(type=Var.PLAIN, value=is_need_request_agent_version) return act @classmethod diff --git a/apps/backend/components/collections/agent_new/render_and_push_gse_config.py b/apps/backend/components/collections/agent_new/render_and_push_gse_config.py index ced816bc9..98856d6fa 100644 --- a/apps/backend/components/collections/agent_new/render_and_push_gse_config.py +++ b/apps/backend/components/collections/agent_new/render_and_push_gse_config.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from typing import Any, Dict, List +from typing import Any, Dict, List, Union from apps.node_man import models from apps.utils.files import PathHandler @@ -18,18 +18,50 @@ class RenderAndPushGseConfigService(AgentPushConfigService): + def host_id_agent_version_map(self, data, common_data: AgentCommonData) -> Dict[int, str]: + # 重载2.0配置实时获取Agent Version信息 + is_need_request_agent_version: bool = data.get_one_of_inputs("is_need_request_agent_version") + if not is_need_request_agent_version or common_data.agent_step_adapter.is_legacy: + return + # 实时获取Agent Version信息 + # 构造 gse 请求参数 + hosts: List[Dict[str, Union[int, str]]] = [] + agent_host_id_map: Dict[int, str] = {} + for host_id in common_data.bk_host_ids: + host_obj: models.Host = common_data.host_id_obj_map[host_id] + host: Dict[str, Union[int, str]] = { + "ip": host_obj.inner_ip or host_obj.inner_ipv6, + "bk_cloud_id": host_obj.bk_cloud_id, + "bk_agent_id": host_obj.bk_agent_id, + } + hosts.append(hosts) + agent_host_id_map[common_data.gse_api_helper.get_agent_id(host)] = host_id + agent_id__agent_state_info_map: Dict[str, Dict] = common_data.gse_api_helper.list_agent_state(hosts) + + host_id_agent_version_map: Dict[int, str] = {} + for agent_id, agent_state_info in agent_id__agent_state_info_map.items(): + host_id_agent_version_map[agent_host_id_map[agent_id]] = agent_state_info["version"] + + return host_id_agent_version_map + def get_config_info_list(self, data, common_data: AgentCommonData, host: models.Host) -> List[Dict[str, Any]]: file_name_list: List[str] = common_data.agent_step_adapter.get_config_filename_by_node_type(host.node_type) general_node_type = self.get_general_node_type(host.node_type) host_ap: models.AccessPoint = self.get_host_ap(common_data=common_data, host=host) + host_id_agent_version_map: Dict[int, str] = self.host_id_agent_version_map(common_data=common_data) + config_file_list: List[Dict[str, Any]] = [] for file_name in file_name_list: config_file_list.append( { "file_name": file_name, "content": common_data.agent_step_adapter.get_config( - host=host, filename=file_name, node_type=general_node_type, ap=host_ap + host=host, + filename=file_name, + node_type=general_node_type, + ap=host_ap, + host_id_agent_version_map=host_id_agent_version_map, ), } ) diff --git a/apps/backend/subscription/steps/agent.py b/apps/backend/subscription/steps/agent.py index 30e7fcabf..ec6bb466b 100644 --- a/apps/backend/subscription/steps/agent.py +++ b/apps/backend/subscription/steps/agent.py @@ -316,6 +316,7 @@ class UpgradeAgent(ReinstallAgent): def _generate_activities(self, agent_manager: AgentManager): activities = [ agent_manager.push_upgrade_package(), + agent_manager.render_and_push_gse_config(), agent_manager.run_upgrade_command(), agent_manager.wait(30), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), @@ -457,6 +458,7 @@ class UpgradeProxy(ReinstallProxy): def _generate_activities(self, agent_manager: AgentManager): activities = [ agent_manager.push_upgrade_package(), + agent_manager.render_and_push_gse_config(), agent_manager.run_upgrade_command(), agent_manager.wait(30), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), @@ -533,7 +535,7 @@ def _generate_activities(self, agent_manager: AgentManager): activities = [ agent_manager.check_agent_status(), agent_manager.update_install_info(), - agent_manager.render_and_push_gse_config(), + agent_manager.render_and_push_gse_config(is_need_request_agent_version=True), agent_manager.reload_agent(skip_polling_result=True), agent_manager.wait(5), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), diff --git a/apps/backend/subscription/steps/agent_adapter/adapter.py b/apps/backend/subscription/steps/agent_adapter/adapter.py index e1e1e4dcd..8267b8533 100644 --- a/apps/backend/subscription/steps/agent_adapter/adapter.py +++ b/apps/backend/subscription/steps/agent_adapter/adapter.py @@ -57,6 +57,7 @@ class AgentStepAdapter: log_prefix: str = field(init=False) # 配置处理模块缓存 _config_handler_cache: typing.Dict[str, GseConfigHandler] = field(init=False) + _agent_setup_info: typing.Dict[str, base.AgentSetupInfo] = field(init=False) def __post_init__(self): self.is_legacy = self.gse_version == GseVersion.V1.value @@ -64,6 +65,7 @@ def __post_init__(self): f"[{self.__class__.__name__}({self.subscription_step.step_id})] | {self.subscription_step} |" ) self._config_handler_cache: typing.Dict[str, GseConfigHandler] = {} + self._agent_setup_info: typing.Dict[str, base.AgentSetupInfo] = {} def get_config_handler(self, agent_name: str, target_version: str) -> GseConfigHandler: @@ -102,8 +104,11 @@ def _get_config( ap: models.AccessPoint, proxies: typing.List[models.Host], install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]], + host_id_agent_version_map: typing.Optional[typing.Dict[int, str]], ) -> str: - agent_setup_info: base.AgentSetupInfo = self.setup_info + agent_setup_info: base.AgentSetupInfo = self.get_host_setup_info( + agent_version=host_id_agent_version_map[host.bk_host_id] + ) config_handler: GseConfigHandler = self.get_config_handler(agent_setup_info.name, agent_setup_info.version) config_tmpl_obj: base.AgentConfigTemplate = config_handler.get_matching_config_tmpl( os_type=host.os_type, @@ -134,6 +139,7 @@ def get_config( ap: typing.Optional[models.AccessPoint] = None, proxies: typing.Optional[typing.List[models.Host]] = None, install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]] = None, + host_id_agent_version_map: typing.Optional[typing.Dict[int, str]] = None, ) -> str: """ 获取配置 @@ -152,12 +158,17 @@ def get_config( func: typing.Callable[..., str] = (self._get_config, legacy.generate_gse_config)[self.is_legacy] return func( - host=host, filename=filename, node_type=node_type, ap=ap, proxies=proxies, install_channel=install_channel + host=host, + filename=filename, + node_type=node_type, + ap=ap, + proxies=proxies, + install_channel=install_channel, + host_id_agent_version_map=host_id_agent_version_map, ) @property - @cache.class_member_cache() - def setup_info(self) -> base.AgentSetupInfo: + def get_host_setup_info(self, agent_version: str) -> base.AgentSetupInfo: """ 获取 Agent 设置信息 TODO 后续如需支持多版本，该方法改造为 `get_host_setup_info`，根据维度进行缓存，参考 _config_handler_cache @@ -171,15 +182,22 @@ def setup_info(self) -> base.AgentSetupInfo: else: target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( target_id=AGENT_NAME_TARGET_ID_MAP[agent_name], - target_version=self.config.get("version"), + target_version=agent_version or self.config.get("version"), ) - return base.AgentSetupInfo( + cache_key: str = f"agent_name:{agent_name}:version:{target_version}" + agent_setup_info: typing.Optional[base.AgentSetupInfo] = self._agent_setup_info.get(cache_key) + if agent_setup_info: + return agent_setup_info + + agent_setup_info: base.AgentSetupInfo = base.AgentSetupInfo( is_legacy=self.is_legacy, agent_tools_relative_dir=("agent_tools/agent2", "")[self.is_legacy], - name=self.config.get("name"), + name=agent_name, version=target_version, ) + self._agent_setup_info[cache_key] = agent_setup_info + return agent_setup_info @staticmethod def validated_data(data, serializer) -> OrderedDict: diff --git a/apps/backend/subscription/steps/agent_adapter/config_context/context_dataclass.py b/apps/backend/subscription/steps/agent_adapter/config_context/context_dataclass.py index 522278be8..21dab5159 100644 --- a/apps/backend/subscription/steps/agent_adapter/config_context/context_dataclass.py +++ b/apps/backend/subscription/steps/agent_adapter/config_context/context_dataclass.py @@ -194,6 +194,12 @@ class DataAgentConfigContext(GseConfigContext, TlsBaseConfigContext): tcp_server_thread_num: int = 32 tcp_server_max_message_size: int = 10485760 + # 兼容Agent 2.0 >= 2.1.5 gse_data_proxy 配置文件结构变更 + bind_port: int = 0 + bind_ip: str = "::" + + thread_num: int = 32 + @dataclass class DataProxyConfigContext(GseConfigContext, TlsBaseConfigContext): diff --git a/apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py b/apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py index b273bc8e3..7e355732b 100644 --- a/apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py +++ b/apps/backend/subscription/steps/agent_adapter/config_context/context_helper.py @@ -153,6 +153,8 @@ def __post_init__(self): context_dataclass.DataMetricConfigContext(exporter_bind_port=self.ap.port_config["data_prometheus_port"]), context_dataclass.DataAgentConfigContext( tcp_bind_port=self.ap.port_config["data_port"], + # 兼容Agent 2.0 >= 2.1.5 gse_data_proxy 配置文件结构变更 + bind_port=self.ap.port_config["data_port"], tls_ca_file=proxy_tls_ca_file, tls_cert_file=proxy_tls_cert_file, tls_key_file=proxy_tls_key_file, diff --git a/apps/backend/subscription/steps/agent_adapter/legacy.py b/apps/backend/subscription/steps/agent_adapter/legacy.py index 993240247..643ee07b8 100644 --- a/apps/backend/subscription/steps/agent_adapter/legacy.py +++ b/apps/backend/subscription/steps/agent_adapter/legacy.py @@ -458,6 +458,7 @@ def generate_gse_config( ap: models.AccessPoint, proxies: List[models.Host], install_channel: Tuple[Optional[models.Host], Dict[str, List]], + host_id_agent_version_map: Optional[Dict[int, str]], ): """ 生成 GSE 相关配置