diff --git a/apps/backend/agent/manager.py b/apps/backend/agent/manager.py index 20db7578e..a38bd674a 100644 --- a/apps/backend/agent/manager.py +++ b/apps/backend/agent/manager.py @@ -138,6 +138,14 @@ def get_agent_status(cls, expect_status: str, name=components.GetAgentStatusComp act.component.inputs.expect_status = Var(type=Var.PLAIN, value=expect_status) return act + @classmethod + def get_agent_id(cls, name=components.GetAgentIDComponent.name): + """查询 Agent ID""" + if settings.GSE_VERSION == "V1": + return None + act = AgentServiceActivity(component_code=components.GetAgentIDComponent.code, name=name) + return act + @classmethod def check_agent_status(cls, name=components.CheckAgentStatusComponent.name): """查询Agent状态是否正常""" diff --git a/apps/backend/agent/tools.py b/apps/backend/agent/tools.py index 01125016c..50bec9562 100644 --- a/apps/backend/agent/tools.py +++ b/apps/backend/agent/tools.py @@ -250,7 +250,9 @@ def gen_commands( agent_config = host_ap.get_agent_config(host.os_type) # 安装操作 install_path = agent_config["setup_path"] - token = aes_cipher.encrypt(f"{host.inner_ip}|{host.bk_cloud_id}|{pipeline_id}|{time.time()}|{sub_inst_id}") + token = aes_cipher.encrypt( + f"{host.bk_host_id}|{host.inner_ip}|{host.bk_cloud_id}|{pipeline_id}|{time.time()}|{sub_inst_id}" + ) port_config = host_ap.port_config run_cmd_params = [ f"-s {pipeline_id}", @@ -309,8 +311,9 @@ def gen_commands( host_shell = format_run_cmd_by_os_type(host.os_type, host_tmp_path) run_cmd_params.extend( [ - f"-HLIP {host.login_ip or host.inner_ip}", + f"-HLIP {host.login_ip or host.inner_ip or host.inner_ipv6}", f"-HIIP {host.inner_ip}", + f"-HIIP6 {host.inner_ipv6}" if host.inner_ipv6 else "", f"-HA {identity_data.account}", f"-HP {identity_data.port}", f"-HI '{host_identity}'", @@ -323,6 +326,8 @@ def gen_commands( f"-HS '{host_shell}'", f"-p '{install_path}'", f"-I {jump_server.inner_ip}", + f"-I6 {jump_server.inner_ipv6}" if jump_server.inner_ipv6 else "", + f"-AI {host.bk_agent_id}" if host.bk_agent_id else "", f"-o {gen_nginx_download_url(jump_server.inner_ip)}", f"-HEP '{encrypted_password}'" if need_encrypted_password else "", "-R" if is_uninstall else "", @@ -354,6 +359,8 @@ def gen_commands( [ f"-i {host.bk_cloud_id}", f"-I {host.inner_ip}", + f"-I6 {host.inner_ipv6}" if host.inner_ipv6 else "", + f"-AI {host.bk_agent_id}" if host.bk_agent_id else "", "-N SERVER", f"-p {install_path}", f"-T {dest_dir}", diff --git a/apps/backend/components/collections/agent_new/components.py b/apps/backend/components/collections/agent_new/components.py index 7f7fdf711..922eaaf6e 100644 --- a/apps/backend/components/collections/agent_new/components.py +++ b/apps/backend/components/collections/agent_new/components.py @@ -19,6 +19,7 @@ from .choose_access_point import ChooseAccessPointService from .configure_policy import ConfigurePolicyService from .delegate_plugin_proc import DelegatePluginProcService +from .get_agent_id import GetAgentIDService from .get_agent_status import GetAgentStatusService from .install import InstallService from .install_plugins import InstallPluginsService @@ -83,6 +84,12 @@ class GetAgentStatusComponent(Component): bound_service = GetAgentStatusService +class GetAgentIDComponent(Component): + name = _("查询AgentID") + code = "get_agent_id" + bound_service = GetAgentIDService + + class ReloadAgentConfigComponent(Component): name = _("重载Agent配置") code = "reload_agent_config" diff --git a/apps/backend/components/collections/agent_new/get_agent_id.py b/apps/backend/components/collections/agent_new/get_agent_id.py new file mode 100644 index 000000000..90e8f332e --- /dev/null +++ b/apps/backend/components/collections/agent_new/get_agent_id.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from typing import Any, Dict, List + +from django.utils.translation import ugettext_lazy as _ + +from apps.backend.api.constants import POLLING_INTERVAL, POLLING_TIMEOUT +from apps.component.esbclient import client_v2 +from apps.node_man import models +from apps.utils import batch_request +from pipeline.core.flow import Service, StaticIntervalGenerator + +from .base import AgentBaseService, AgentCommonData + + +class GetAgentIDService(AgentBaseService): + """安装后AgentID同步到CMDB后才认为是可用的""" + + __need_schedule__ = True + interval = StaticIntervalGenerator(POLLING_INTERVAL) + + def outputs_format(self): + return super().outputs_format() + [ + Service.InputItem(name="polling_time", key="polling_time", type="int", required=True), + ] + + @staticmethod + def update_host_agent_id(cmdb_host_infos: List[Dict[str, Any]]): + need_update_hosts = [ + models.Host(bk_host_id=host_info["bk_host_id"], bk_agent_id=host_info.get("bk_agent_id")) + for host_info in cmdb_host_infos + ] + models.Host.objects.bulk_update(need_update_hosts, fields=["bk_agent_id"]) + + def _execute(self, data, parent_data, common_data: AgentCommonData): + self.log_info(sub_inst_ids=common_data.subscription_instance_ids, log_content=_("开始查询从 CMDB 查询主机的 Agent ID")) + data.outputs.polling_time = 0 + + def _schedule(self, data, parent_data, callback_data=None): + common_data: AgentCommonData = self.get_common_data(data) + list_cmdb_hosts_params = { + "fields": ["bk_host_id", "bk_agent_id"], + "host_property_filter": { + "condition": "AND", + "rules": [ + {"field": "bk_host_id", "operator": "in", "value": common_data.bk_host_ids}, + ], + }, + } + cmdb_host_infos: List[Dict[str, Any]] = batch_request.batch_request( + func=client_v2.cc.list_hosts_without_biz, params=list_cmdb_hosts_params + ) + + # CMDB 中 AgentID 为空的主机ID列表 + no_agent_id_host_ids = [ + host_info["bk_host_id"] for host_info in cmdb_host_infos if not host_info.get("bk_agent_id") + ] + if not no_agent_id_host_ids: + self.update_host_agent_id(cmdb_host_infos) + self.finish_schedule() + return + + polling_time = data.get_one_of_outputs("polling_time") + if polling_time + POLLING_INTERVAL > POLLING_TIMEOUT: + sub_inst_ids = [common_data.host_id__sub_inst_id_map[host_id] for host_id in no_agent_id_host_ids] + self.move_insts_to_failed( + sub_inst_ids=sub_inst_ids, + log_content=_("此主机在 CMDB 中未查到对应 Agent ID，请联系 GSE 及 CMDB 团队排查 Agent ID 上报处理是否异常！"), + ) + self.update_host_agent_id(cmdb_host_infos) + self.finish_schedule() + return + + data.outputs.polling_time = polling_time + POLLING_INTERVAL diff --git a/apps/backend/components/collections/agent_new/register_host.py b/apps/backend/components/collections/agent_new/register_host.py index bc17c2a0e..a07482a20 100644 --- a/apps/backend/components/collections/agent_new/register_host.py +++ b/apps/backend/components/collections/agent_new/register_host.py @@ -146,11 +146,13 @@ def add_hosts_to_biz_thread( host_key = f"{host_info['bk_cloud_id']}-{host_info['bk_host_innerip']}" register_params = { index: { - "bk_host_innerip": host_info["bk_host_innerip"], + "bk_host_innerip": host_info.get("bk_host_innerip", ""), + "bk_host_innerip_v6": host_info.get("bk_host_innerip_v6", ""), # "3" 表示API导入 "import_from": "3", "bk_cloud_id": host_info["bk_cloud_id"], "bk_host_outerip": host_info.get("bk_host_outerip", ""), + "bk_host_outerip_v6": host_info.get("bk_host_outerip_v6", ""), "bk_os_type": constants.BK_OS_TYPE[host_info["os_type"]], "bk_bak_operator": biz_info.get("bk_biz_maintainer") or host_info.get("username"), "operator": biz_info.get("bk_biz_maintainer") or host_info.get("username"), diff --git a/apps/backend/components/collections/common/remote.py b/apps/backend/components/collections/common/remote.py index 4bdc765b3..beaf40207 100644 --- a/apps/backend/components/collections/common/remote.py +++ b/apps/backend/components/collections/common/remote.py @@ -60,9 +60,9 @@ def conns_init_params(self) -> typing.Dict: :return: """ if self.host.node_type == constants.NodeType.PROXY: - ip = self.host.login_ip or self.host.outer_ip + ip = self.host.login_ip or self.host.outer_ip or self.host.outer_ipv6 else: - ip = self.host.login_ip or self.host.inner_ip + ip = self.host.login_ip or self.host.inner_ip or self.host.inner_ipv6 client_key_strings = [] if self.identity_data.auth_type == constants.AuthType.KEY: diff --git a/apps/backend/components/collections/plugin.py b/apps/backend/components/collections/plugin.py index 41720acc3..bb9b94a69 100644 --- a/apps/backend/components/collections/plugin.py +++ b/apps/backend/components/collections/plugin.py @@ -1144,7 +1144,10 @@ def _execute(self, data, parent_data, common_data: PluginCommonData): gse_op_params = { "meta": {"namespace": constants.GSE_NAMESPACE, "name": meta_name}, "op_type": op_type, + # GSE 1.0 目标对象 "hosts": [{"ip": host.inner_ip, "bk_cloud_id": host.bk_cloud_id}], + # GSE 2.0 目标对象 + "agent_id_list": [host.bk_agent_id], # 此字段是节点管理自用，仅用于标识，不会被GSE使用 "nodeman_spec": { "process_status_id": process_status.id, @@ -1240,7 +1243,10 @@ def _schedule(self, data, parent_data, callback_data=None): subscription_instance = group_id_instance_map.get(process_status.group_id) proc_name = self.get_plugin_meta_name(plugin, process_status) - gse_proc_key = f"{host.bk_cloud_id}:{host.inner_ip}:{constants.GSE_NAMESPACE}:{proc_name}" + if host.bk_agent_id: + gse_proc_key = f"{host.bk_agent_id}:{constants.GSE_NAMESPACE}:{proc_name}" + else: + gse_proc_key = f"{host.bk_cloud_id}:{host.inner_ip}:{constants.GSE_NAMESPACE}:{proc_name}" proc_operate_result = result["data"].get(gse_proc_key) if not proc_operate_result: self.move_insts_to_failed( diff --git a/apps/backend/plugin/serializers.py b/apps/backend/plugin/serializers.py index 4aa6b7257..67c54acb5 100644 --- a/apps/backend/plugin/serializers.py +++ b/apps/backend/plugin/serializers.py @@ -237,6 +237,7 @@ class PluginStartDebugSerializer(GatewaySerializer): """ class HostInfoSerializer(serializers.Serializer): + bk_host_id = serializers.IntegerField(required=False) ip = serializers.CharField(required=True) bk_cloud_id = serializers.IntegerField(required=True) bk_supplier_id = serializers.IntegerField(default=constants.DEFAULT_SUPPLIER_ID) diff --git a/apps/backend/plugin/views.py b/apps/backend/plugin/views.py index c0f1bbfdf..0b37fa460 100644 --- a/apps/backend/plugin/views.py +++ b/apps/backend/plugin/views.py @@ -51,6 +51,7 @@ from apps.exceptions import AppBaseException, ValidationError from apps.generic import APIViewSet from apps.node_man import constants, models +from apps.node_man.exceptions import HostNotExists from pipeline.engine.exceptions import InvalidOperationException from pipeline.service import task_service from pipeline.service.pipeline_engine_adapter.adapter_api import STATE_MAP @@ -499,15 +500,15 @@ def start_debug(self, request): host_info = params["host_info"] plugin_name = params["plugin_name"] plugin_version = params["version"] + query_host_params = dict( + bk_biz_id=host_info["bk_biz_id"], inner_ip=host_info["ip"], bk_cloud_id=host_info["bk_cloud_id"] + ) + if params.get("bk_host_id"): + query_host_params.update(bk_host_id=params["bk_host_id"]) - try: - host = models.Host.objects.get( - bk_biz_id=host_info["bk_biz_id"], - inner_ip=host_info["ip"], - bk_cloud_id=host_info["bk_cloud_id"], - ) - except models.Host.DoesNotExist: - raise ValidationError("host does not exist") + host = models.Host.objects.get(**query_host_params).first() + if not host: + raise HostNotExists("host does not exist") plugin_id = params.get("plugin_id") if plugin_id: diff --git a/apps/backend/subscription/commons.py b/apps/backend/subscription/commons.py index 93b289b7b..deac64f77 100644 --- a/apps/backend/subscription/commons.py +++ b/apps/backend/subscription/commons.py @@ -38,7 +38,7 @@ def get_host_object_attribute(bk_biz_id): def list_biz_hosts(bk_biz_id, condition, func, split_params=False): biz_custom_property = [] kwargs = { - "fields": constants.LIST_BIZ_HOSTS_KWARGS, + "fields": constants.CC_HOST_FIELDS, } if bk_biz_id: biz_custom_property = get_host_object_attribute(bk_biz_id) diff --git a/apps/backend/subscription/render_functions.py b/apps/backend/subscription/render_functions.py index aaa1a7551..30d30dca4 100644 --- a/apps/backend/subscription/render_functions.py +++ b/apps/backend/subscription/render_functions.py @@ -34,7 +34,7 @@ def get_host_detail_by_template(bk_obj_id, template_info_list: list, bk_biz_id: if not template_info_list: return [] - fields = constants.FIND_HOST_BY_TEMPLATE_FIELD + fields = constants.CC_HOST_FIELDS if bk_obj_id == models.Subscription.NodeType.SERVICE_TEMPLATE: # 服务模板 diff --git a/apps/backend/subscription/steps/agent.py b/apps/backend/subscription/steps/agent.py index c14967419..c04b9f82f 100644 --- a/apps/backend/subscription/steps/agent.py +++ b/apps/backend/subscription/steps/agent.py @@ -159,6 +159,7 @@ def _generate_activities(self, agent_manager: AgentManager): agent_manager.choose_ap(), agent_manager.install(), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), + agent_manager.get_agent_id(), agent_manager.install_plugins() if self.is_install_latest_plugins else None, ] @@ -180,6 +181,7 @@ def _generate_activities(self, agent_manager: AgentManager): agent_manager.choose_ap(), agent_manager.install(), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), + agent_manager.get_agent_id(), agent_manager.install_plugins() if self.is_install_latest_plugins else None, ] @@ -199,6 +201,7 @@ def _generate_activities(self, agent_manager: AgentManager): agent_manager.push_upgrade_package(), agent_manager.run_upgrade_command(), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), + agent_manager.get_agent_id(), ] return activities, None @@ -216,6 +219,7 @@ def _generate_activities(self, agent_manager: AgentManager): agent_manager.restart(skip_polling_result=True), agent_manager.wait(5), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), + agent_manager.get_agent_id(), ] return activities, None @@ -310,6 +314,7 @@ def _generate_activities(self, agent_manager: AgentManager): agent_manager.run_upgrade_command(), agent_manager.wait(30), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), + agent_manager.get_agent_id(), ] # 推送文件到proxy @@ -381,6 +386,7 @@ def _generate_activities(self, agent_manager: AgentManager): agent_manager.reload_agent(skip_polling_result=True), agent_manager.wait(5), agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING), + agent_manager.get_agent_id(), ] return activities, None diff --git a/apps/backend/subscription/tools.py b/apps/backend/subscription/tools.py index 9bd262c41..9058fb1b2 100644 --- a/apps/backend/subscription/tools.py +++ b/apps/backend/subscription/tools.py @@ -374,7 +374,7 @@ def get_host_detail_by_template(bk_obj_id, template_info_list: list, bk_biz_id: if not template_info_list: return [] - fields = constants.FIND_HOST_BY_TEMPLATE_FIELD + fields = constants.CC_HOST_FIELDS if bk_obj_id == models.Subscription.NodeType.SERVICE_TEMPLATE: # 服务模板 diff --git a/apps/backend/tests/components/collections/agent_new/test_get_agent_id.py b/apps/backend/tests/components/collections/agent_new/test_get_agent_id.py new file mode 100644 index 000000000..bebaeb321 --- /dev/null +++ b/apps/backend/tests/components/collections/agent_new/test_get_agent_id.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from collections import ChainMap +from typing import Any, Callable, Dict, List, Optional + +import mock + +from apps.backend.api.constants import POLLING_INTERVAL +from apps.backend.components.collections.agent_new import components +from apps.mock_data import api_mkd +from apps.mock_data import utils as mock_data_utils +from apps.mock_data.api_mkd.cmdb import unit +from pipeline.component_framework.test import ( + ComponentTestCase, + ExecuteAssertion, + ScheduleAssertion, +) + +from . import utils + + +class GetAgentIDTestCaseMixin: + CLIENT_V2_MOCK_PATHS: List[str] = [ + "apps.backend.components.collections.agent_new.get_agent_id.client_v2", + ] + list_hosts_without_biz_func: Optional[Callable] = None + cmdb_mock_client: Optional[api_mkd.cmdb.utils.CMDBMockClient] = None + + def init_mock_clients(self): + self.cmdb_mock_client = api_mkd.cmdb.utils.CMDBMockClient( + list_hosts_without_biz_return=mock_data_utils.MockReturn( + return_type=mock_data_utils.MockReturnType.SIDE_EFFECT.value, + return_obj=self.list_hosts_without_biz_func, + ) + ) + + @classmethod + def setup_obj_factory(cls): + """设置 obj_factory""" + cls.obj_factory.init_host_num = 10 + + def fetch_succeeded_sub_inst_ids(self) -> List[int]: + return self.common_inputs["subscription_instance_ids"] + + def structure_common_inputs(self) -> Dict[str, Any]: + common_inputs = super().structure_common_inputs() + return {**common_inputs} + + def structure_common_outputs(self, polling_time: Optional[int] = None, **extra_kw) -> Dict[str, Any]: + """ + 构造原子返回数据 + :param polling_time: 轮询时间 + :param extra_kw: 额外需要添加的数据 + :return: + """ + base_common_outputs = { + "succeeded_subscription_instance_ids": self.fetch_succeeded_sub_inst_ids(), + } + if polling_time is not None: + base_common_outputs["polling_time"] = polling_time + + return dict(ChainMap(extra_kw, base_common_outputs)) + + def component_cls(self): + return components.GetAgentIDComponent + + def setUp(self) -> None: + self.init_mock_clients() + for client_v2_mock_path in self.CLIENT_V2_MOCK_PATHS: + mock.patch(client_v2_mock_path, self.cmdb_mock_client).start() + super().setUp() + + +class GetAgentIDFailedTestCase(GetAgentIDTestCaseMixin, utils.AgentServiceBaseTestCase): + list_hosts_without_biz_func = unit.mock_list_host_without_biz_result + + @classmethod + def get_default_case_name(cls) -> str: + return "查询 Agent ID 失败" + + def cases(self): + return [ + ComponentTestCase( + name=self.get_default_case_name(), + inputs=self.common_inputs, + parent_data={}, + execute_assertion=ExecuteAssertion( + success=bool(self.fetch_succeeded_sub_inst_ids()), + outputs=self.structure_common_outputs(polling_time=0), + ), + schedule_assertion=[ + ScheduleAssertion( + success=True, + schedule_finished=False, + outputs=self.structure_common_outputs(polling_time=POLLING_INTERVAL), + ) + ], + execute_call_assertion=None, + ) + ] + + +class GetAgentIDSucceededTestCase(GetAgentIDTestCaseMixin, utils.AgentServiceBaseTestCase): + list_hosts_without_biz_func = unit.mock_list_host_without_biz_with_agent_id_result + + @classmethod + def get_default_case_name(cls) -> str: + return "查询 Agent ID 成功" + + def cases(self): + return [ + ComponentTestCase( + name=self.get_default_case_name(), + inputs=self.common_inputs, + parent_data={}, + execute_assertion=ExecuteAssertion( + success=bool(self.fetch_succeeded_sub_inst_ids()), + outputs=self.structure_common_outputs(polling_time=0), + ), + schedule_assertion=[ + ScheduleAssertion( + success=True, + schedule_finished=True, + outputs=self.structure_common_outputs(polling_time=0), + ) + ], + execute_call_assertion=None, + ) + ] + + +class GetAgentIDTimeoutTestCase(GetAgentIDTestCaseMixin, utils.AgentServiceBaseTestCase): + list_hosts_without_biz_func = unit.mock_list_host_without_biz_result + + @classmethod + def get_default_case_name(cls) -> str: + return "查询 Agent ID 超时" + + def setUp(self) -> None: + super().setUp() + mock.patch( + "apps.backend.components.collections.agent_new.get_agent_id.POLLING_TIMEOUT", + POLLING_INTERVAL - 1, + ).start() + + def cases(self): + return [ + ComponentTestCase( + name=self.get_default_case_name(), + inputs=self.common_inputs, + parent_data={}, + execute_assertion=ExecuteAssertion( + success=bool(self.fetch_succeeded_sub_inst_ids()), + outputs=self.structure_common_outputs(polling_time=0), + ), + schedule_assertion=[ + ScheduleAssertion( + success=False, + schedule_finished=True, + outputs=self.structure_common_outputs(polling_time=0, succeeded_subscription_instance_ids=[]), + ) + ], + execute_call_assertion=None, + ) + ] diff --git a/apps/backend/tests/view/base.py b/apps/backend/tests/view/base.py index d2eb8f1b0..f4dc1893f 100644 --- a/apps/backend/tests/view/base.py +++ b/apps/backend/tests/view/base.py @@ -32,6 +32,7 @@ def gen_token( self, ) -> str: return models.aes_cipher.encrypt( - f"{common_unit.host.DEFAULT_IP}|{constants.DEFAULT_CLOUD}|{self.PIPELINE_ID}|" + f"{common_unit.host.DEFAULT_HOST_ID}|{common_unit.host.DEFAULT_IP}|{constants.DEFAULT_CLOUD}|" + f"{self.PIPELINE_ID}|" f"{time.time()}|{self.SUB_INST_ID}" ) diff --git a/apps/backend/tests/view/test_get_gse_config.py b/apps/backend/tests/view/test_get_gse_config.py index d6851463d..100d4552d 100644 --- a/apps/backend/tests/view/test_get_gse_config.py +++ b/apps/backend/tests/view/test_get_gse_config.py @@ -10,7 +10,6 @@ """ import copy -import random from apps.backend.constants import REDIS_AGENT_CONF_KEY_TPL from apps.backend.utils.redis import REDIS_INST @@ -31,7 +30,6 @@ def setUpTestData(cls): super().setUpTestData() host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) - host_model_data.update(bk_host_id=random.randint(1000, 10000)) cls.host = models.Host.objects.create(**host_model_data) # 此类查询在单元测试中会有如下报错， 因此将数据预先查询缓存 # TransactionManagementError "You can't execute queries until the end of the 'atomic' block" while using signals diff --git a/apps/backend/views.py b/apps/backend/views.py index f9cf97957..f9f7720d2 100644 --- a/apps/backend/views.py +++ b/apps/backend/views.py @@ -720,19 +720,18 @@ def get_gse_config(request): decrypted_token = _decrypt_token(token) if inner_ip != decrypted_token["inner_ip"] or bk_cloud_id != decrypted_token["bk_cloud_id"]: - logger.error( - "token[{token}] 非法, 请求参数为: {data}, token解析为: {decrypted_token}".format( - token=token, data=data, decrypted_token=decrypted_token - ) + err_msg = "token[{token}] 非法, 请求参数为: {data}, token解析为: {decrypted_token}".format( + token=token, data=data, decrypted_token=decrypted_token ) - raise PermissionError("what are you doing?") + logger.error(err_msg) + raise PermissionError(err_msg) config = REDIS_INST.get(REDIS_AGENT_CONF_KEY_TPL.format(file_name=filename, sub_inst_id=decrypted_token["inst_id"])) if config: return HttpResponse(config.decode()) try: - host = Host.objects.get(bk_cloud_id=bk_cloud_id, inner_ip=inner_ip) + host = Host.objects.get(bk_host_id=decrypted_token["bk_host_id"]) if filename in ["bscp.yaml"]: config = generate_bscp_config(host=host) else: @@ -818,11 +817,12 @@ def _decrypt_token(token: str) -> dict: try: token_decrypt = aes_cipher.decrypt(token) except Exception as err: - logger.error(f"{token}解析失败") + logger.error(f"{token} 解析失败") raise err - inner_ip, bk_cloud_id, task_id, timestamp, inst_id = token_decrypt.split("|") + bk_host_id, inner_ip, bk_cloud_id, task_id, timestamp, inst_id = token_decrypt.split("|") return_value = { + "bk_host_id": bk_host_id, "inner_ip": inner_ip, "bk_cloud_id": int(bk_cloud_id), "task_id": task_id, @@ -831,8 +831,7 @@ def _decrypt_token(token: str) -> dict: } # timestamp 超过1小时，认为是非法请求 if time.time() - float(timestamp) > 3600: - logger.error(f"token[{token}] 非法, timestamp超时不符合预期, {return_value}") - raise PermissionError("what are you doing?") + raise PermissionError(f"token[{token}] 非法, timestamp超时不符合预期, {return_value}") return return_value diff --git a/apps/mock_data/api_mkd/cmdb/unit.py b/apps/mock_data/api_mkd/cmdb/unit.py index 29ed269e0..61859e63d 100644 --- a/apps/mock_data/api_mkd/cmdb/unit.py +++ b/apps/mock_data/api_mkd/cmdb/unit.py @@ -8,3 +8,58 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import copy + +from apps.mock_data.common_unit import host +from apps.node_man import constants + +CMDB_HOST_INFO = { + "bk_host_id": host.DEFAULT_HOST_ID, + "bk_cloud_id": constants.DEFAULT_CLOUD, + "bk_host_innerip": host.DEFAULT_IP, + "bk_host_outerip": "", +} +CMDB_HOST_INFO_WITH_AGENT_ID = { + "bk_host_id": host.DEFAULT_HOST_ID, + "bk_cloud_id": constants.DEFAULT_CLOUD, + "bk_host_innerip": host.DEFAULT_IP, + "bk_host_outerip": host.DEFAULT_IP, + "bk_host_innerip_v6": host.DEFAULT_IPv6, + "bk_host_outerip_v6": host.DEFAULT_IPv6, + "bk_agent_id": host.BK_AGENT_ID, +} + +LIST_HOSTS_WITHOUT_BIZ_RESULT = {"count": 1, "info": [CMDB_HOST_INFO]} + +LIST_HOSTS_WITHOUT_BIZ_WITH_AGENT_ID_RESULT = {"count": 1, "info": [CMDB_HOST_INFO_WITH_AGENT_ID]} + + +def mock_list_host_without_biz_result(*args, **kwargs): + has_agent_id = kwargs.get("has_agent_id") + if has_agent_id: + host_info_template = CMDB_HOST_INFO_WITH_AGENT_ID + else: + host_info_template = CMDB_HOST_INFO + host_infos = [] + check_by_bk_host_id = False + bk_host_ids = [] + for rule in kwargs["host_property_filter"]["rules"]: + if rule["field"] == "bk_host_id": + bk_host_ids = rule["value"] + check_by_bk_host_id = True + if check_by_bk_host_id: + for bk_host_id in bk_host_ids: + host_info = copy.deepcopy(host_info_template) + host_info["bk_host_id"] = bk_host_id + host_infos.append(host_info) + else: + host_infos = [host_info_template] + return {"count": len(host_infos), "info": host_infos} + + +def mock_list_host_without_biz_without_agent_id_result(*args, **kwargs): + return mock_list_host_without_biz_result(has_agent_id=False, *args, **kwargs) + + +def mock_list_host_without_biz_with_agent_id_result(*args, **kwargs): + return mock_list_host_without_biz_result(has_agent_id=True, *args, **kwargs) diff --git a/apps/mock_data/api_mkd/gse/unit.py b/apps/mock_data/api_mkd/gse/unit.py index 87a1e6015..e7f434f50 100644 --- a/apps/mock_data/api_mkd/gse/unit.py +++ b/apps/mock_data/api_mkd/gse/unit.py @@ -8,6 +8,142 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import copy + +from apps.mock_data.common_unit import host +from apps.node_man import constants # 操作类接口一般返回的是 task_id OP_RESULT = {"task_id": "GSETASK:S:202111161138323563236795:143"} +GSE_PROCESS_VERSION = "1.7.17" +GSE_PROCESS_NAME = "test_process" + +GET_AGENT_ALIVE_STATUS_DATA = { + f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}": { + "ip": host.DEFAULT_IP, + "bk_cloud_id": constants.DEFAULT_CLOUD, + "bk_agent_alive": constants.BkAgentStatus.ALIVE.value, + } +} + +GET_AGENT_NOT_ALIVE_STATUS_DATA = { + f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}": { + "ip": host.DEFAULT_IP, + "bk_cloud_id": constants.DEFAULT_CLOUD, + "bk_agent_alive": constants.BkAgentStatus.NOT_ALIVE.value, + } +} + +GET_AGENT_INFO_DATA = { + f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}": { + "ip": host.DEFAULT_IP, + "version": GSE_PROCESS_VERSION, + "bk_cloud_id": 0, + "parent_ip": host.DEFAULT_IP, + "parent_port": 50000, + } +} + +GET_PROC_OPERATE_RESULT = { + f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}:{constants.GSE_NAMESPACE}:sub_870_host_1_host_exp002": { + "content": '{\n "value" : [\n {\n "funcID" : "",\n' + ' "instanceID" : "",\n ' + ' "procName" : "host_exp002",\n "result" : "success",\n' + ' "setupPath" : "/usr/local/gse/external_plugins/sub_870' + '_host_1/host_exp002"\n }\n ]\n}\n', + "error_code": 0, + "error_msg": "success", + } +} + +GET_AGENT_STATE_LIST_DATA = [ + { + "bk_agent_id": f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}", + "bk_cloud_id": constants.DEFAULT_CLOUD, + "version": GSE_PROCESS_VERSION, + "run_mode": 0, + "status_code": constants.GseAgentStatusCode.RUNNING.value, + } +] + +GET_AGENT_NOT_ALIVE_STATE_LIST_DATA = [ + { + "bk_agent_id": f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}", + "bk_cloud_id": constants.DEFAULT_CLOUD, + "version": GSE_PROCESS_VERSION, + "run_mode": 0, + "status_code": constants.GseAgentStatusCode.STOPPED.value, + } +] + +GET_AGENT_INFO_LIST_DATA = [ + { + "code": 0, + "message": "OK", + "bk_agent_id": f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}", + "bk_cloud_id": constants.DEFAULT_CLOUD, + "bk_host_ip": host.DEFAULT_IP, + "bk_os_type": "linux", + "report_time": 1632830304777, + "parent_ip": host.DEFAULT_IP, + "parent_port": 0, + "version": GSE_PROCESS_VERSION, + "cpu_rate": 12.34, + "mem_rate": 56.78, + "start_time": 1632830300, + "run_mode": 0, + "status_code": constants.GseAgentStatusCode.RUNNING.value, + "status": "running", + "last_status_code": constants.GseAgentStatusCode.RUNNING.value, + "last_status": "running", + "remark": "", + } +] + +GET_PROC_STATUS_DATA = { + "proc_infos": [ + { + # 需要根据不同参数补充 hosts 或者 bk_agent_id, 详见 mock_get_proc_status + "status": constants.GseProcessStatusCode.STOPPED.value, + "version": GSE_PROCESS_VERSION, + "isauto": constants.GseProcessAutoCode.AUTO.value, + "meta": { + "name": GSE_PROCESS_NAME, + "namespace": constants.GSE_NAMESPACE, + "labels": {"proc_name": GSE_PROCESS_NAME}, + }, + } + ] +} + + +def mock_get_agent_info_list(params): + agent_info_list = copy.deepcopy(GET_AGENT_INFO_LIST_DATA) + for index, agent_info in enumerate(agent_info_list): + agent_info["bk_agent_id"] = params["agent_id_list"][index] + return agent_info_list + + +def mock_get_agent_state_list(params): + agent_state_list = copy.deepcopy(GET_AGENT_STATE_LIST_DATA) + for index, agent_state in enumerate(agent_state_list): + agent_state["bk_agent_id"] = params["agent_id_list"][index] + return agent_state_list + + +def mock_get_proc_status(params): + pro_status_data = copy.deepcopy(GET_PROC_STATUS_DATA) + hosts_param = params["hosts"][0] + for index, host_info in enumerate(params["hosts"]): + if f"{hosts_param['bk_cloud_id']}:{hosts_param['ip']}" == hosts_param["agent_id"]: + pro_status_data["proc_infos"][index]["host"] = { + "ip": hosts_param["ip"], + "bk_cloud_id": hosts_param["bk_cloud_id"], + } + else: + pro_status_data["proc_infos"][index]["bk_agent_id"] = hosts_param["agent_id"] + # 添加返回一个不存在的key来模拟额外返回的case + not_exist_proc_info = copy.deepcopy(pro_status_data["proc_infos"][0]) + not_exist_proc_info["bk_agent_id"] = "not_exist_proc_info_agent_id" + pro_status_data["proc_infos"].append(not_exist_proc_info) + return pro_status_data diff --git a/apps/mock_data/api_mkd/gse/utils.py b/apps/mock_data/api_mkd/gse/utils.py index 20ce84a22..2f3b4ae89 100644 --- a/apps/mock_data/api_mkd/gse/utils.py +++ b/apps/mock_data/api_mkd/gse/utils.py @@ -10,19 +10,50 @@ """ from ... import utils +from . import unit class GseApiMockClient(utils.BaseMockClient): + DEFAULT_OPERATE_PROC_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.RETURN_VALUE.value, return_obj=unit.OP_RESULT + ) + DEFAULT_GET_OPERATE_RESULT_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.RETURN_VALUE.value, return_obj=unit.GET_PROC_OPERATE_RESULT + ) + DEFAULT_GET_AGENT_INFO_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.RETURN_VALUE.value, return_obj=unit.GET_AGENT_INFO_DATA + ) + DEFAULT_GET_AGENT_STATUS_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.RETURN_VALUE.value, return_obj=unit.GET_AGENT_ALIVE_STATUS_DATA + ) + GET_AGENT_NOT_ALIVE_STATUS_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.RETURN_VALUE.value, return_obj=unit.GET_AGENT_NOT_ALIVE_STATUS_DATA + ) + DEFAULT_GET_AGENT_INFO_LIST_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.SIDE_EFFECT.value, return_obj=unit.mock_get_agent_info_list + ) + DEFAULT_GET_AGENT_STATE_LIST_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.SIDE_EFFECT.value, return_obj=unit.mock_get_agent_state_list + ) + GET_AGENT_NOT_ALIVE_STATE_LIST_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.RETURN_VALUE.value, return_obj=unit.GET_AGENT_NOT_ALIVE_STATE_LIST_DATA + ) + DEFAULT_GET_PROC_STATUS_RETURN = utils.MockReturn( + return_type=utils.MockReturnType.SIDE_EFFECT.value, return_obj=unit.mock_get_proc_status + ) + def __init__( self, - operate_proc_return=None, - operate_proc_multi_return=None, - get_proc_operate_result_return=None, - get_proc_status_return=None, + operate_proc_return=DEFAULT_OPERATE_PROC_RETURN, + operate_proc_multi_return=DEFAULT_OPERATE_PROC_RETURN, + get_proc_operate_result_return=DEFAULT_GET_OPERATE_RESULT_RETURN, + get_proc_status_return=DEFAULT_GET_PROC_STATUS_RETURN, sync_proc_status_return=None, update_proc_info_return=None, - get_agent_info_return=None, - get_agent_status_return=None, + get_agent_info_return=DEFAULT_GET_AGENT_INFO_RETURN, + get_agent_status_return=DEFAULT_GET_AGENT_STATUS_RETURN, + get_agent_info_list_return=DEFAULT_GET_AGENT_INFO_LIST_RETURN, + get_agent_state_list_return=DEFAULT_GET_AGENT_STATE_LIST_RETURN, ): super().__init__() self.operate_proc = self.generate_magic_mock(mock_return_obj=operate_proc_return) @@ -33,3 +64,5 @@ def __init__( self.update_proc_info = self.generate_magic_mock(mock_return_obj=update_proc_info_return) self.get_agent_info = self.generate_magic_mock(mock_return_obj=get_agent_info_return) self.get_agent_status = self.generate_magic_mock(mock_return_obj=get_agent_status_return) + self.get_agent_info_list = self.generate_magic_mock(mock_return_obj=get_agent_info_list_return) + self.get_agent_state_list = self.generate_magic_mock(mock_return_obj=get_agent_state_list_return) diff --git a/apps/mock_data/common_unit/__init__.py b/apps/mock_data/common_unit/__init__.py index 3e388431a..e648ac499 100644 --- a/apps/mock_data/common_unit/__init__.py +++ b/apps/mock_data/common_unit/__init__.py @@ -10,6 +10,6 @@ """ -from . import gse, host, job, plugin, subscription +from . import host, job, plugin, subscription -__all__ = ["host", "job", "plugin", "subscription", "gse"] +__all__ = ["host", "job", "plugin", "subscription"] diff --git a/apps/mock_data/common_unit/host.py b/apps/mock_data/common_unit/host.py index 4c6064758..3fced9a14 100644 --- a/apps/mock_data/common_unit/host.py +++ b/apps/mock_data/common_unit/host.py @@ -8,14 +8,20 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import copy + from apps.node_man import constants, models from .. import utils +BK_AGENT_ID = "xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx" + DEFAULT_HOST_ID = 1 DEFAULT_IP = "127.0.0.1" +DEFAULT_IPv6 = "ABCD:EF01:2345:6789:ABCD:EF01:2345:6789" + PROXY_INNER_IP = "1.1.1.1" AP_MODEL_DATA = { @@ -85,13 +91,22 @@ "os_type": constants.OsType.LINUX, "cpu_arch": constants.CpuType.x86_64, "node_type": constants.NodeType.AGENT, - "node_from": constants.NodeFrom.NODE_MAN, + "node_from": constants.NodeFrom.CMDB, "ap_id": constants.DEFAULT_AP_ID, "upstream_nodes": [], "is_manual": False, "extra_data": {"bt_speed_limit": None, "peer_exchange_switch_for_agent": 1}, } +HOST_MODEL_DATA_WITH_AGENT_ID = copy.deepcopy(HOST_MODEL_DATA) +HOST_MODEL_DATA_WITH_AGENT_ID.update( + **{ + "bk_agent_id": BK_AGENT_ID, + "inner_ipv6": DEFAULT_IPv6, + "outer_ipv6": DEFAULT_IPv6, + } +) + IDENTITY_MODEL_DATA = { "bk_host_id": DEFAULT_HOST_ID, "auth_type": constants.AuthType.PASSWORD, diff --git a/apps/mock_data/common_unit/gse.py b/apps/mock_data/views_mkd/__init__.py similarity index 57% rename from apps/mock_data/common_unit/gse.py rename to apps/mock_data/views_mkd/__init__.py index 4359129b0..29ed269e0 100644 --- a/apps/mock_data/common_unit/gse.py +++ b/apps/mock_data/views_mkd/__init__.py @@ -8,26 +8,3 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - - -from apps.node_man import constants - -from . import host - -GET_AGENT_STATUS_DATA = { - f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}": { - "ip": host.DEFAULT_IP, - "bk_cloud_id": constants.DEFAULT_CLOUD, - "bk_agent_alive": constants.BkAgentStatus.ALIVE.value, - } -} - -GET_AGENT_INFO_DATA = { - f"{constants.DEFAULT_CLOUD}:{host.DEFAULT_IP}": { - "ip": host.DEFAULT_IP, - "version": "V0.01R060D38", - "bk_cloud_id": 0, - "parent_ip": host.DEFAULT_IP, - "parent_port": 50000, - } -} diff --git a/apps/mock_data/views_mkd/job.py b/apps/mock_data/views_mkd/job.py new file mode 100644 index 000000000..472303a3d --- /dev/null +++ b/apps/mock_data/views_mkd/job.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +# 安装请求参数 +from django.conf import settings + +from apps.mock_data import utils +from apps.mock_data.common_unit import host +from apps.node_man import constants + +JOB_INSTALL_REQUEST_PARAMS = { + "job_type": constants.JobType.INSTALL_AGENT, + "hosts": [ + { + "bk_cloud_id": constants.DEFAULT_CLOUD, + "ap_id": constants.DEFAULT_AP_ID, + "bk_biz_id": utils.DEFAULT_BK_BIZ_ID, + "os_type": constants.OsType.LINUX, + "inner_ip": host.DEFAULT_IP, + "inner_ipv6": host.DEFAULT_IPv6, + "outer_ip": host.DEFAULT_IP, + "outer_ipv6": host.DEFAULT_IPv6, + "login_ip": host.DEFAULT_IP, + "data_ip": host.DEFAULT_IP, + "account": constants.LINUX_ACCOUNT, + "port": settings.BKAPP_DEFAULT_SSH_PORT, + "auth_type": constants.AuthType.PASSWORD, + "password": "password", + "key": "key", + } + ], + "retention": 1, +} + + +JOB_OPERATE_REQUEST_PARAMS = {"job_type": constants.JobType.REINSTALL_AGENT, "bk_host_id": [host.DEFAULT_HOST_ID]} diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index b27074d6b..d5c8292ba 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -32,6 +32,7 @@ # 此值为历史遗留，后续蓝鲸不使用此字段后可废弃 DEFAULT_SUPPLIER_ID = 0 + ######################################################################################################## # 任务超时控制 ######################################################################################################## @@ -47,7 +48,6 @@ class TimeUnit: # JOB 任务超时时间 JOB_TIMEOUT = 30 * TimeUnit.MINUTE - ######################################################################################################## # 周期任务周期 run_every，非特殊统一使用秒作为单位的 int 类型，不使用crontab格式 # 以便与削峰函数 calculate_countdown 所用的 duration 复用 @@ -77,9 +77,6 @@ class TimeUnit: # GSE命名空间 GSE_NAMESPACE = "nodeman" -CC_HOST_FIELDS = ["bk_host_id", "bk_cloud_id", "bk_host_innerip", "bk_host_outerip", "bk_os_type", "bk_os_name"] - - ######################################################################################################## # 字符串常量 ######################################################################################################## @@ -588,40 +585,29 @@ class BkappRunEnvType(Enum): } ) -LIST_BIZ_HOSTS_KWARGS = [ +CC_HOST_FIELDS = [ "bk_host_id", - "bk_os_type", - "bk_host_outerip", - "bk_host_innerip", - "bk_cpu_module", + "bk_agent_id", "bk_cloud_id", - "operator", - "bk_bak_operator", - "bk_os_bit", - "bk_os_name", - "bk_host_name", - "bk_supplier_account", -] - -FIND_HOST_BY_TEMPLATE_FIELD = ( "bk_host_innerip", - "bk_cloud_id", - "bk_host_id", - "bk_biz_id", "bk_host_outerip", + "bk_host_innerip_v6", + "bk_host_outerip_v6", "bk_host_name", - "bk_os_name", "bk_os_type", + "bk_os_name", + "bk_os_bit", + "bk_os_version", + "bk_cpu_module", "operator", "bk_bak_operator", - "bk_state_name", "bk_isp_name", + "bk_biz_id", "bk_province_name", - "bk_supplier_account", - "bk_state", - "bk_os_version", "bk_state", -) + "bk_state_name", + "bk_supplier_account", +] # 限流窗口配置，用于控制CMDB订阅触发的变更频率 # 二元组：防抖时间，防抖解除窗口大小 @@ -795,6 +781,60 @@ class GseOpType(object): } +class GseAgentStatusCode(EnhanceEnum): + NOT_FOUND = -2 + QUERY_FAILED = -1 + INITIAL = 0 + STARTING = 1 + RUNNING = 2 + LOSSY = 3 + BUSY = 4 + STOPPED = 5 + UPGRADING = 6 + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.NOT_FOUND: _("未找到"), + cls.QUERY_FAILED: _("查询失败"), + cls.INITIAL: _("初始状态"), + cls.STARTING: _("启动中"), + cls.RUNNING: _("运行中"), + cls.LOSSY: _("有损状态"), + cls.BUSY: _("繁忙状态"), + cls.STOPPED: _("已停止"), + cls.UPGRADING: _("升级中"), + } + + +class GseProcessStatusCode(EnhanceEnum): + UNREGISTER = 0 + RUNNING = 1 + STOPPED = 2 + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.UNREGISTER: _("未注册"), + cls.RUNNING: _("运行中"), + cls.STOPPED: _("已停止"), + } + + +class GseProcessAutoCode(EnhanceEnum): + """进程是否在GSE托管""" + + AUTO = True + NOT_AUTO = False + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.AUTO: _("已托管"), + cls.NOT_AUTO: _("未托管"), + } + + class CmdbObjectId: BIZ = "biz" SET = "set" diff --git a/apps/node_man/handlers/job.py b/apps/node_man/handlers/job.py index f69a4a77b..b4d52ef5e 100644 --- a/apps/node_man/handlers/job.py +++ b/apps/node_man/handlers/job.py @@ -8,6 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import copy import logging import re from typing import Any, Dict, List, Optional @@ -446,7 +447,7 @@ def install( else: # 重装、卸载等操作 # 此步骤需要校验密码、秘钥 - subscription_nodes, ip_filter_list = self.update(accept_list, ip_filter_list, is_manual) + subscription_nodes, ip_filter_list = self.update_host(accept_list, ip_filter_list, is_manual) if not subscription_nodes: raise exceptions.AllIpFiltered( data={"job_id": "", "ip_filter": self.ugettext_to_unicode(ip_filter_list)} @@ -492,32 +493,34 @@ def subscription_install(self, accept_list: list, node_type: str, cloud_info: di login_ip = host.get("login_ip", "") host_ap_id, host_node_type = self.check_ap_and_biz_scope(node_type, host, cloud_info) - - instance_info = { - "is_manual": host["is_manual"], - "ap_id": host_ap_id, - "install_channel_id": host.get("install_channel_id"), - "bk_os_type": constants.BK_OS_TYPE[host["os_type"]], - "bk_host_innerip": inner_ip, - "bk_host_outerip": outer_ip, - "login_ip": login_ip, - "username": get_request_username(), - "bk_biz_id": host["bk_biz_id"], - "bk_biz_name": biz_info.get(host["bk_biz_id"]), - "bk_cloud_id": host["bk_cloud_id"], - "bk_cloud_name": str(cloud_info.get(host["bk_cloud_id"], {}).get("bk_cloud_name")), - "bk_supplier_account": settings.DEFAULT_SUPPLIER_ACCOUNT, - "host_node_type": host_node_type, - "os_type": host["os_type"], - "auth_type": host.get("auth_type", "MANUAL"), - "account": host.get("account", "MANUAL"), - "port": host.get("port"), - "password": tools.HostTools.USE_RSA_PREFIX + rsa_util.encrypt(host.get("password", "")), - "key": tools.HostTools.USE_RSA_PREFIX + rsa_util.encrypt(host.get("key", "")), - "retention": host.get("retention", 1), - "peer_exchange_switch_for_agent": host.get("peer_exchange_switch_for_agent"), - "bt_speed_limit": host.get("bt_speed_limit"), - } + instance_info = copy.deepcopy(host) + instance_info.update( + { + "is_manual": host["is_manual"], + "ap_id": host_ap_id, + "install_channel_id": host.get("install_channel_id"), + "bk_os_type": constants.BK_OS_TYPE[host["os_type"]], + "bk_host_innerip": inner_ip, + "bk_host_outerip": outer_ip, + "login_ip": login_ip, + "username": get_request_username(), + "bk_biz_id": host["bk_biz_id"], + "bk_biz_name": biz_info.get(host["bk_biz_id"]), + "bk_cloud_id": host["bk_cloud_id"], + "bk_cloud_name": str(cloud_info.get(host["bk_cloud_id"], {}).get("bk_cloud_name")), + "bk_supplier_account": settings.DEFAULT_SUPPLIER_ACCOUNT, + "host_node_type": host_node_type, + "os_type": host["os_type"], + "auth_type": host.get("auth_type", "MANUAL"), + "account": host.get("account", "MANUAL"), + "port": host.get("port"), + "password": tools.HostTools.USE_RSA_PREFIX + rsa_util.encrypt(host.get("password", "")), + "key": tools.HostTools.USE_RSA_PREFIX + rsa_util.encrypt(host.get("key", "")), + "retention": host.get("retention", 1), + "peer_exchange_switch_for_agent": host.get("peer_exchange_switch_for_agent"), + "bt_speed_limit": host.get("bt_speed_limit"), + } + ) if host_node_type == constants.NodeType.PROXY and host.get("data_path"): # proxy增加数据文件路径 @@ -543,11 +546,12 @@ def subscription_install(self, accept_list: list, node_type: str, cloud_info: di return subscription_nodes @staticmethod - def update(accept_list: list, ip_filter_list: list, is_manual: bool = False): + def update_host(accept_list: list, ip_filter_list: list, is_manual: bool = False): """ 用于更新identity认证信息 :param accept_list: 所有需要修改的数据 :param ip_filter_list: 过滤数据 + :param is_manual: 是否手动安装 """ identity_to_create = [] @@ -583,7 +587,9 @@ def update(accept_list: list, ip_filter_list: list, is_manual: bool = False): "bk_biz_id": host["bk_biz_id"], "bk_cloud_id": host["bk_cloud_id"], "inner_ip": host["inner_ip"], + "inner_ipv6": host["inner_ipv6"], "outer_ip": host["outer_ip"], + "outer_ipv6": host["outer_ipv6"], "login_ip": host["login_ip"], "data_ip": host["data_ip"], "os_type": host["os_type"], @@ -655,6 +661,8 @@ def update(accept_list: list, ip_filter_list: list, is_manual: bool = False): "bk_cloud_id": origin_host["bk_cloud_id"], "inner_ip": origin_host["inner_ip"], "outer_ip": origin_host["outer_ip"], + "inner_ipv6": origin_host["inner_ipv6"], + "outer_ipv6": origin_host["outer_ipv6"], "login_ip": host.get("login_ip", origin_host["login_ip"]), "data_ip": origin_host["data_ip"], "os_type": host.get("os_type", origin_host["os_type"]), diff --git a/apps/node_man/migrations/0059_auto_20220315_1852.py b/apps/node_man/migrations/0059_auto_20220315_1852.py new file mode 100644 index 000000000..a198069bc --- /dev/null +++ b/apps/node_man/migrations/0059_auto_20220315_1852.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2.4 on 2022-03-15 10:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0058_replenish_plugin_template"), + ] + + operations = [ + migrations.AddField( + model_name="host", + name="bk_agent_id", + field=models.CharField(blank=True, db_index=True, max_length=64, null=True, verbose_name="AgentID"), + ), + migrations.AlterField( + model_name="host", + name="cpu_arch", + field=models.CharField( + choices=[ + ("x86", "x86"), + ("x86_64", "x86_64"), + ("powerpc", "powerpc"), + ("aarch64", "aarch64"), + ("sparc", "sparc"), + ], + db_index=True, + default="x86_64", + max_length=16, + verbose_name="CPU架构", + ), + ), + ] diff --git a/apps/node_man/migrations/0060_auto_20220315_2044.py b/apps/node_man/migrations/0060_auto_20220315_2044.py new file mode 100644 index 000000000..3eebb69a0 --- /dev/null +++ b/apps/node_man/migrations/0060_auto_20220315_2044.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.4 on 2022-03-15 12:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0059_auto_20220315_1852"), + ] + + operations = [ + migrations.AddField( + model_name="host", + name="inner_ipv6", + field=models.CharField(blank=True, default="", max_length=45, null=True, verbose_name="内网IPv6"), + ), + migrations.AddField( + model_name="host", + name="outer_ipv6", + field=models.CharField(blank=True, default="", max_length=45, null=True, verbose_name="外网IPv6"), + ), + migrations.AlterField( + model_name="host", + name="data_ip", + field=models.CharField(blank=True, default="", max_length=15, null=True, verbose_name="数据IP"), + ), + migrations.AlterField( + model_name="host", + name="inner_ip", + field=models.CharField(db_index=True, max_length=15, verbose_name="内网IP"), + ), + migrations.AlterField( + model_name="host", + name="login_ip", + field=models.CharField(blank=True, default="", max_length=15, null=True, verbose_name="登录IP"), + ), + migrations.AlterField( + model_name="host", + name="outer_ip", + field=models.CharField(blank=True, default="", max_length=15, null=True, verbose_name="外网IP"), + ), + ] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index e08ab772c..6bf98263f 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -308,19 +308,23 @@ class Meta: class Host(models.Model): bk_host_id = models.IntegerField(_("CMDB主机ID"), primary_key=True) + bk_agent_id = models.CharField(_("AgentID"), max_length=64, db_index=True, blank=True, null=True) bk_biz_id = models.IntegerField(_("业务ID"), db_index=True) bk_cloud_id = models.IntegerField(_("云区域ID"), db_index=True) - inner_ip = models.CharField(_("内网IP"), max_length=45, db_index=True) - outer_ip = models.CharField(_("外网IP"), max_length=45, blank=True, null=True, default="") - login_ip = models.CharField(_("登录IP"), max_length=45, blank=True, null=True, default="") - data_ip = models.CharField(_("数据IP"), max_length=45, blank=True, null=True, default="") + inner_ip = models.CharField(_("内网IP"), max_length=15, db_index=True) + outer_ip = models.CharField(_("外网IP"), max_length=15, blank=True, null=True, default="") + login_ip = models.CharField(_("登录IP"), max_length=15, blank=True, null=True, default="") + data_ip = models.CharField(_("数据IP"), max_length=15, blank=True, null=True, default="") + + inner_ipv6 = models.CharField(_("内网IPv6"), max_length=45, blank=True, null=True, default="") + outer_ipv6 = models.CharField(_("外网IPv6"), max_length=45, blank=True, null=True, default="") os_type = models.CharField( _("操作系统"), max_length=16, choices=constants.OS_CHOICES, default=constants.OsType.LINUX, db_index=True ) cpu_arch = models.CharField( - _("操作系统"), max_length=16, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True + _("CPU架构"), max_length=16, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True ) node_type = models.CharField(_("节点类型"), max_length=16, choices=constants.NODE_CHOICES, db_index=True) node_from = models.CharField(_("节点来源"), max_length=45, choices=constants.NODE_FROM_CHOICES, default="NODE_MAN") @@ -359,18 +363,17 @@ def get_by_host_info(cls, host_info): try: return Host.objects.get(bk_host_id=bk_host_id) except Host.DoesNotExist: - exception = _("{bk_host_id}| 主机信息不存在").format(bk_host_id=bk_host_id) + exception = _("bk_host_id={bk_host_id} 主机信息不存在").format(bk_host_id=bk_host_id) else: ip = host_info.get("bk_host_innerip") or host_info.get("ip") # 兼容IP为逗号分割的多IP情况，取第一个IP ip = ip.split(",")[0] bk_cloud_id = host_info["bk_cloud_id"] - - try: - return Host.objects.get(inner_ip=ip, bk_cloud_id=bk_cloud_id) - except Host.DoesNotExist: - exception = _("{ip}|{bk_cloud_id} 主机信息不存在").format(ip=ip, bk_cloud_id=bk_cloud_id) - + host = Host.objects.get(inner_ip=ip, bk_cloud_id=bk_cloud_id).first() + if host: + return host + else: + exception = _("{bk_cloud_id}:{ip} 主机信息不存在").format(ip=ip, bk_cloud_id=bk_cloud_id) raise HostNotExists(exception) @classmethod diff --git a/apps/node_man/periodic_tasks/resource_watch_task.py b/apps/node_man/periodic_tasks/resource_watch_task.py index dc5f093b1..cd22749e9 100644 --- a/apps/node_man/periodic_tasks/resource_watch_task.py +++ b/apps/node_man/periodic_tasks/resource_watch_task.py @@ -19,7 +19,7 @@ from django.db.utils import IntegrityError from apps.component.esbclient import client_v2 -from apps.node_man import constants as const +from apps.node_man import constants from apps.node_man.models import GlobalSettings, ResourceWatchEvent, Subscription from apps.utils.cache import format_cache_key @@ -128,7 +128,7 @@ def sync_resource_watch_host_event(): 拉取主机事件 """ kwargs = { - "bk_resource": const.ResourceType.host, + "bk_resource": constants.ResourceType.host, "bk_fields": ["bk_host_innerip", "bk_os_type", "bk_host_id", "bk_cloud_id", "bk_host_outerip"], } @@ -139,7 +139,7 @@ def sync_resource_watch_host_relation_event(): """ 拉取主机关系事件 """ - kwargs = {"bk_resource": const.ResourceType.host_relation} + kwargs = {"bk_resource": constants.ResourceType.host_relation} _resource_watch(RESOURCE_WATCH_HOST_RELATION_CURSOR_KEY, kwargs) @@ -148,7 +148,7 @@ def sync_resource_watch_process_event(): """ 拉取进程 """ - kwargs = {"bk_resource": const.ResourceType.process} + kwargs = {"bk_resource": constants.ResourceType.process} _resource_watch(RESOURCE_WATCH_PROCESS_CURSOR_KEY, kwargs) @@ -219,7 +219,7 @@ def cal_next_debounce_window(current_time): """ 计算出下一次的防抖窗口 """ - windows = const.CMDB_SUBSCRIPTION_DEBOUNCE_WINDOWS + windows = constants.CMDB_SUBSCRIPTION_DEBOUNCE_WINDOWS if current_time < windows[0][0]: # 防抖时间下限 diff --git a/apps/node_man/periodic_tasks/sync_agent_status_task.py b/apps/node_man/periodic_tasks/sync_agent_status_task.py index c553a38f3..1aa2645ed 100644 --- a/apps/node_man/periodic_tasks/sync_agent_status_task.py +++ b/apps/node_man/periodic_tasks/sync_agent_status_task.py @@ -9,17 +9,24 @@ specific language governing permissions and limitations under the License. """ from celery.task import periodic_task, task +from django.conf import settings +from django.db.models import Q -from apps.component.esbclient import client_v2 from apps.node_man import constants from apps.node_man.models import Host, ProcessStatus from apps.utils.periodic_task import calculate_countdown +from common.api import GseApi from common.log import logger @task(queue="default", ignore_result=True) -def update_or_create_host_agent_status(task_id, start, end): - hosts = Host.objects.values("bk_host_id", "bk_cloud_id", "inner_ip", "node_from")[start:end] +def update_or_create_host_agent_status(task_id, start, end, has_agent_id=False): + # GSE接口不支持同时传 {cloud_id}:{ip} + {agent_id} 混合模式，因此需要分开处理 + if has_agent_id: + host_queryset = Host.objects.exclude(bk_agent_id__isnull=True).exclude(bk_agent_id__exact="") + else: + host_queryset = Host.objects.filter(Q(bk_agent_id__isnull=True) | Q(bk_agent_id__exact="")) + hosts = host_queryset.values("bk_host_id", "bk_agent_id", "bk_cloud_id", "inner_ip", "node_from")[start:end] if not hosts: # 结束递归 return @@ -31,22 +38,41 @@ def update_or_create_host_agent_status(task_id, start, end): node_from_map = {} # 生成查询参数host弄表 + agent_id_list = [] query_host = [] for host in hosts: - # IP 为空，直接跳过 - if not host["inner_ip"]: - continue - bk_host_id_map[f"{host['bk_cloud_id']}:{host['inner_ip']}"] = host["bk_host_id"] - node_from_map[f"{host['bk_cloud_id']}:{host['inner_ip']}"] = host["node_from"] + if host["bk_agent_id"] and settings.GSE_VERSION != "V1": + agent_id = host["bk_agent_id"] + else: + agent_id = f"{host['bk_cloud_id']}:{host['inner_ip']}" + bk_host_id_map[agent_id] = host["bk_host_id"] + node_from_map[agent_id] = host["node_from"] query_host.append({"ip": host["inner_ip"], "bk_cloud_id": host["bk_cloud_id"]}) + agent_id_list.append(agent_id) # 查询agent状态和版本信息 - agent_status_data = client_v2.gse.get_agent_status({"hosts": query_host}) - agent_info_data = client_v2.gse.get_agent_info({"hosts": query_host}) + if settings.GSE_VERSION == "V1": + agent_status_data = GseApi.get_agent_status({"hosts": query_host}) + agent_info_data = GseApi.get_agent_info({"hosts": query_host}) + else: + # GSE2.0 接口，补充GSE1.0字段 + agent_status_list = GseApi.get_agent_state_list({"agent_id_list": agent_id_list}) + agent_status_data = {} + for agent_status in agent_status_list: + if agent_status.get("status_code") == constants.GseAgentStatusCode.RUNNING.value: + agent_status["bk_agent_alive"] = constants.BkAgentStatus.ALIVE.value + else: + agent_status["bk_agent_alive"] = constants.BkAgentStatus.NOT_ALIVE.value + agent_status_data[agent_status["bk_agent_id"]] = agent_status + + agent_info_list = GseApi.get_agent_info_list({"agent_id_list": agent_id_list}) + agent_info_data = {agent_info["bk_agent_id"]: agent_info for agent_info in agent_info_list} # 查询需要更新主机的ProcessStatus对象 process_status_objs = ProcessStatus.objects.filter( - name="gseagent", bk_host_id__in=bk_host_id_map.values(), source_type=ProcessStatus.SourceType.DEFAULT + name=ProcessStatus.GSE_AGENT_PROCESS_NAME, + bk_host_id__in=bk_host_id_map.values(), + source_type=ProcessStatus.SourceType.DEFAULT, ).values("bk_host_id", "id", "status") # 生成bk_host_id与ProcessStatus对象的映射 @@ -63,31 +89,31 @@ def update_or_create_host_agent_status(task_id, start, end): is_running = host_info["bk_agent_alive"] == constants.BkAgentStatus.ALIVE.value version = constants.VERSION_PATTERN.search(agent_info_data[key]["version"]) + if is_running: + status = constants.ProcStateType.RUNNING + if node_from_map[key] == constants.NodeFrom.CMDB: + need_update_node_from_host.append( + Host(bk_host_id=bk_host_id_map[key], node_from=constants.NodeFrom.NODE_MAN) + ) + else: + # 状态为0时如果节点管理为CMDB标记为未安装否则为异常 + if node_from_map[key] == constants.NodeFrom.CMDB: + # NOT_INSTALLED + status = constants.ProcStateType.NOT_INSTALLED + else: + # TERMINATED + status = constants.ProcStateType.TERMINATED + if not process_status_id: # 如果不存在ProcessStatus对象需要创建 to_be_created_status.append( ProcessStatus( bk_host_id=bk_host_id_map[key], - status=constants.PROC_STATUS_DICT[host_info["bk_agent_alive"]], + status=status, version=version.group() if version else "", ) ) else: - if is_running: - status = constants.PROC_STATUS_DICT[host_info["bk_agent_alive"]] - if node_from_map[key] == constants.NodeFrom.CMDB: - need_update_node_from_host.append( - Host(bk_host_id=bk_host_id_map[key], node_from=constants.NodeFrom.NODE_MAN) - ) - else: - # 状态为0时如果节点管理为CMDB标记为未安装否则为异常 - if node_from_map[key] == constants.NodeFrom.CMDB: - # NOT_INSTALLED - status = constants.ProcStateType.NOT_INSTALLED - else: - # TERMINATED - status = constants.ProcStateType.TERMINATED - process_objs.append( ProcessStatus( id=process_status_id, status=status, version=version.group() if (version and is_running) else "" @@ -113,14 +139,18 @@ def sync_agent_status_periodic_task(): """ task_id = sync_agent_status_periodic_task.request.id logger.info(f"{task_id} | sync_agent_status_task: Start syncing host status.") - count = Host.objects.count() - for start in range(0, count, constants.QUERY_AGENT_STATUS_HOST_LENS): - countdown = calculate_countdown( - count=count / constants.QUERY_AGENT_STATUS_HOST_LENS, - index=start / constants.QUERY_AGENT_STATUS_HOST_LENS, - duration=constants.SYNC_AGENT_STATUS_TASK_INTERVAL, - ) - logger.info(f"{task_id} | sync_agent_status_task after {countdown} seconds") - update_or_create_host_agent_status.apply_async( - (task_id, start, start + constants.QUERY_AGENT_STATUS_HOST_LENS), countdown=countdown - ) + for (has_agent_id, queryset) in [ + (True, Host.objects.exclude(bk_agent_id__isnull=True).exclude(bk_agent_id__exact="")), + (False, Host.objects.filter(Q(bk_agent_id__isnull=True) | Q(bk_agent_id__exact=""))), + ]: + count = queryset.count() + for start in range(0, count, constants.QUERY_AGENT_STATUS_HOST_LENS): + countdown = calculate_countdown( + count=count / constants.QUERY_AGENT_STATUS_HOST_LENS, + index=start / constants.QUERY_AGENT_STATUS_HOST_LENS, + duration=constants.SYNC_AGENT_STATUS_TASK_INTERVAL, + ) + logger.info(f"{task_id} | sync_agent_status_task after {countdown} seconds") + update_or_create_host_agent_status.apply_async( + (task_id, start, start + constants.QUERY_AGENT_STATUS_HOST_LENS, has_agent_id), countdown=countdown + ) diff --git a/apps/node_man/periodic_tasks/sync_cmdb_host.py b/apps/node_man/periodic_tasks/sync_cmdb_host.py index bf46f63e4..dcc41aba3 100644 --- a/apps/node_man/periodic_tasks/sync_cmdb_host.py +++ b/apps/node_man/periodic_tasks/sync_cmdb_host.py @@ -18,9 +18,7 @@ from apps.component.esbclient import client_v2 from apps.exceptions import ComponentCallError -from apps.node_man import constants -from apps.node_man import constants as const -from apps.node_man import models, tools +from apps.node_man import constants, models, tools from apps.utils.batch_request import batch_request from apps.utils.concurrent import batch_call from common.log import logger @@ -74,8 +72,8 @@ def _list_biz_hosts(biz_id: int, start: int) -> dict: biz_hosts = client_v2.cc.list_biz_hosts( { "bk_biz_id": biz_id, - "fields": const.CC_HOST_FIELDS, - "page": {"start": start, "limit": const.QUERY_CMDB_LIMIT, "sort": "bk_host_id"}, + "fields": constants.CC_HOST_FIELDS, + "page": {"start": start, "limit": constants.QUERY_CMDB_LIMIT, "sort": "bk_host_id"}, } ) # 去除内网IP为空的主机 @@ -87,8 +85,8 @@ def _list_resource_pool_hosts(start): try: result = client_v2.cc.list_resource_pool_hosts( { - "page": {"start": start, "limit": const.QUERY_CMDB_LIMIT, "sort": "bk_host_id"}, - "fields": const.CC_HOST_FIELDS, + "page": {"start": start, "limit": constants.QUERY_CMDB_LIMIT, "sort": "bk_host_id"}, + "fields": constants.CC_HOST_FIELDS, } ) return result @@ -96,32 +94,38 @@ def _list_resource_pool_hosts(start): return {"info": []} -def _bulk_update_host(hosts, fields): +def _bulk_update_host(hosts, extra_fields): + update_fields = ["bk_cloud_id", "inner_ip", "outer_ip", "inner_ipv6", "outer_ipv6", "bk_agent_id"] + extra_fields if hosts: - models.Host.objects.bulk_update(hosts, fields=fields) + models.Host.objects.bulk_update(hosts, fields=update_fields) -def _generate_host(biz_id, host, bk_host_innerip, bk_host_outerip, ap_id): +def _generate_host(biz_id, host, ap_id): os_type = tools.HostV2Tools.get_os_type(host) cpu_arch = tools.HostV2Tools.get_cpu_arch(host) host_data = models.Host( bk_host_id=host["bk_host_id"], + bk_agent_id=host.get("bk_agent_id"), bk_biz_id=biz_id, bk_cloud_id=host["bk_cloud_id"], - inner_ip=bk_host_innerip, - outer_ip=bk_host_outerip, - node_from=const.NodeFrom.CMDB, + inner_ip=host["bk_host_innerip"].split(",")[0], + outer_ip=host["bk_host_outerip"].split(",")[0], + inner_ipv6=host.get("bk_host_innerip_v6"), + outer_ipv6=host.get("bk_host_outerip_v6"), + node_from=constants.NodeFrom.CMDB, os_type=os_type, cpu_arch=cpu_arch, - node_type=const.NodeType.AGENT if host["bk_cloud_id"] == const.DEFAULT_CLOUD else const.NodeType.PAGENT, + node_type=constants.NodeType.AGENT + if host["bk_cloud_id"] == constants.DEFAULT_CLOUD + else constants.NodeType.PAGENT, ap_id=ap_id, ) identify_data = models.IdentityData( bk_host_id=host["bk_host_id"], auth_type=constants.AuthType.PASSWORD, - account=const.WINDOWS_ACCOUNT if os_type == const.OsType.WINDOWS else const.LINUX_ACCOUNT, - port=const.WINDOWS_PORT if os_type == const.OsType.WINDOWS else settings.BKAPP_DEFAULT_SSH_PORT, + account=constants.WINDOWS_ACCOUNT if os_type == constants.OsType.WINDOWS else constants.LINUX_ACCOUNT, + port=constants.WINDOWS_PORT if os_type == constants.OsType.WINDOWS else settings.BKAPP_DEFAULT_SSH_PORT, ) process_status_data = models.ProcessStatus( @@ -135,9 +139,13 @@ def _generate_host(biz_id, host, bk_host_innerip, bk_host_outerip, ap_id): def find_host_biz_relations(find_host_biz_ids): host_biz_relation = {} - for count in range(math.ceil(len(find_host_biz_ids) / const.QUERY_CMDB_LIMIT)): + for count in range(math.ceil(len(find_host_biz_ids) / constants.QUERY_CMDB_LIMIT)): cc_host_biz_relations = client_v2.cc.find_host_biz_relations( - {"bk_host_id": find_host_biz_ids[count * const.QUERY_CMDB_LIMIT : (count + 1) * const.QUERY_CMDB_LIMIT]} + { + "bk_host_id": find_host_biz_ids[ + count * constants.QUERY_CMDB_LIMIT : (count + 1) * constants.QUERY_CMDB_LIMIT + ] + } ) for _host_biz in cc_host_biz_relations: host_biz_relation[_host_biz["bk_host_id"]] = _host_biz["bk_biz_id"] @@ -150,7 +158,9 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data): # 查询节点管理已存在的主机 exist_proxy_host_ids: typing.Set[int] = set( - models.Host.objects.filter(bk_host_id__in=bk_host_ids, node_type="PROXY").values_list("bk_host_id", flat=True) + models.Host.objects.filter( + bk_host_id__in=bk_host_ids, node_type=constants.NodeType.PROXY + ).values_list("bk_host_id", flat=True) ) exist_agent_host_ids: typing.Set[int] = set( models.Host.objects.filter(bk_host_id__in=bk_host_ids) @@ -179,7 +189,7 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data): need_create_host_identity_objs: typing.List[models.IdentityData] = [] need_create_process_status_objs: typing.List[models.ProcessStatus] = [] - ap_id = const.DEFAULT_AP_ID if models.AccessPoint.objects.count() > 1 else models.AccessPoint.objects.first().id + ap_id = constants.DEFAULT_AP_ID if models.AccessPoint.objects.count() > 1 else models.AccessPoint.objects.first().id # 已存在的主机批量更新,不存在的主机批量创建 for host in cmdb_host_data: @@ -191,25 +201,16 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data): ) continue - if any(["," in host["bk_host_innerip"], "," in host["bk_host_outerip"]]): - logger.info( - f"[sync_cmdb_host] update_or_create_host: task_id -> {task_id}, bk_biz_id -> {biz_id}, " - f"bk_host_id -> {host['bk_host_id']}, inner_ip -> {host['bk_host_innerip']}, " - f"outer_ip -> {host['bk_host_innerip']} contains multiple ips, get first" - ) - bk_host_innerip = host["bk_host_innerip"].split(",")[0] - bk_host_outerip = host["bk_host_outerip"].split(",")[0] - else: - bk_host_innerip = host["bk_host_innerip"] - bk_host_outerip = host["bk_host_outerip"] - + host_params = { + "bk_host_id": host["bk_host_id"], + "bk_agent_id": host.get("bk_agent_id"), + "bk_cloud_id": host["bk_cloud_id"], + "inner_ip": host["bk_host_innerip"].split(",")[0], + "outer_ip": host["bk_host_outerip"].split(",")[0], + "inner_ipv6": host.get("bk_host_innerip_v6"), + "outer_ipv6": host.get("bk_host_outerip_v6"), + } if host["bk_host_id"] in exist_agent_host_ids: - host_params = { - "bk_host_id": host["bk_host_id"], - "bk_cloud_id": host["bk_cloud_id"], - "inner_ip": bk_host_innerip, - "outer_ip": bk_host_outerip, - } os_type = tools.HostV2Tools.get_os_type(host) @@ -226,13 +227,7 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data): else: need_update_hosts_without_biz_os.append(models.Host(**host_params)) elif host["bk_host_id"] in exist_proxy_host_ids: - host_params = { - "bk_host_id": host["bk_host_id"], - "bk_cloud_id": host["bk_cloud_id"], - "inner_ip": bk_host_innerip, - "outer_ip": bk_host_outerip, - "os_type": const.OsType.LINUX, - } + host_params["os_type"] = constants.OsType.LINUX if biz_id: host_params["bk_biz_id"] = biz_id need_update_hosts.append(models.Host(**host_params)) @@ -243,9 +238,7 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data): if not biz_id: need_create_host_without_biz.append(host) else: - host_data, identify_data, process_status_data = _generate_host( - biz_id, host, bk_host_innerip, bk_host_outerip, ap_id - ) + host_data, identify_data, process_status_data = _generate_host(biz_id, host, ap_id) need_create_hosts.append(host_data) if identify_data.bk_host_id not in host_ids_in_exist_identity_data: need_create_host_identity_objs.append(identify_data) @@ -266,8 +259,6 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data): host_data, identify_data, process_status_data = _generate_host( host_biz_relation[need_create_host["bk_host_id"]], need_create_host, - need_create_host["bk_host_innerip"].split(",")[0], - need_create_host["bk_host_outerip"].split(",")[0], ap_id, ) need_create_hosts.append(host_data) @@ -279,10 +270,10 @@ def update_or_create_host_base(biz_id, task_id, cmdb_host_data): need_create_process_status_objs.append(process_status_data) with transaction.atomic(): - _bulk_update_host(need_update_hosts, ["bk_biz_id", "bk_cloud_id", "inner_ip", "outer_ip", "os_type"]) - _bulk_update_host(need_update_hosts_without_biz, ["bk_cloud_id", "inner_ip", "outer_ip", "os_type"]) - _bulk_update_host(need_update_hosts_without_os, ["bk_biz_id", "bk_cloud_id", "inner_ip", "outer_ip"]) - _bulk_update_host(need_update_hosts_without_biz_os, ["bk_cloud_id", "inner_ip", "outer_ip"]) + _bulk_update_host(need_update_hosts, ["bk_biz_id", "os_type"]) + _bulk_update_host(need_update_hosts_without_biz, ["os_type"]) + _bulk_update_host(need_update_hosts_without_os, ["bk_biz_id"]) + _bulk_update_host(need_update_hosts_without_biz_os, []) if need_create_hosts: models.Host.objects.bulk_create(need_create_hosts, batch_size=500) @@ -348,14 +339,14 @@ def _update_or_create_host(biz_id, start=0, task_id=None): logger.info( f"[sync_cmdb_host] update_or_create_host: task_id -> {task_id}, bk_biz_id -> {biz_id}, " - f"host_count -> {host_count}, range -> {start}-{start + const.QUERY_CMDB_LIMIT}" + f"host_count -> {host_count}, range -> {start}-{start + constants.QUERY_CMDB_LIMIT}" ) bk_host_ids, _ = update_or_create_host_base(biz_id, task_id, host_data) # 递归 - if host_count > start + const.QUERY_CMDB_LIMIT: - bk_host_ids += _update_or_create_host(biz_id, start + const.QUERY_CMDB_LIMIT, task_id=task_id) + if host_count > start + constants.QUERY_CMDB_LIMIT: + bk_host_ids += _update_or_create_host(biz_id, start + constants.QUERY_CMDB_LIMIT, task_id=task_id) return bk_host_ids diff --git a/apps/node_man/periodic_tasks/sync_proc_status_task.py b/apps/node_man/periodic_tasks/sync_proc_status_task.py index 5d60f7df0..ff8adc585 100644 --- a/apps/node_man/periodic_tasks/sync_proc_status_task.py +++ b/apps/node_man/periodic_tasks/sync_proc_status_task.py @@ -12,13 +12,14 @@ import typing from celery.task import periodic_task, task +from django.db.models import Q -from apps.component.esbclient import client_v2 from apps.core.concurrent import controller from apps.node_man import constants, tools from apps.node_man.models import Host, ProcessStatus from apps.utils import concurrent from apps.utils.periodic_task import calculate_countdown +from common.api import GseApi from common.log import logger @@ -42,10 +43,11 @@ def query_proc_status(proc_name, host_list): 5000: 2s-4s (2.3s 2.2s 4.1s 4.3s) """ kwargs = { - "meta": {"namespace": "nodeman", "name": proc_name, "labels": {"proc_name": proc_name}}, + "meta": {"namespace": constants.GSE_NAMESPACE, "name": proc_name, "labels": {"proc_name": proc_name}}, "hosts": host_list, + "agent_id_list": [host["agent_id"] for host in host_list], } - data = client_v2.gse.get_proc_status(kwargs) + data = GseApi.get_proc_status(kwargs) return data.get("proc_infos") or [] @@ -60,7 +62,11 @@ def proc_statues2host_key__readable_proc_status_map( """ host_key__readable_proc_status_map: typing.Dict[str, typing.Dict[str, typing.Any]] = {} for proc_status in proc_statuses: - host_key__readable_proc_status_map[f'{proc_status["host"]["ip"]}:{proc_status["host"]["bk_cloud_id"]}'] = { + if "bk_agent_id" in proc_status: + host_key = proc_status["bk_agent_id"] + else: + host_key = f"{proc_status['host']['bk_cloud_id']}:{proc_status['host']['ip']}" + host_key__readable_proc_status_map[host_key] = { "version": get_version(proc_status["version"]), "status": constants.PLUGIN_STATUS_DICT[proc_status["status"]], "is_auto": constants.AutoStateType.AUTO if proc_status["isauto"] else constants.AutoStateType.UNAUTO, @@ -74,17 +80,21 @@ def update_or_create_proc_status(task_id, hosts, sync_proc_list, start): host_ip_cloud_list = [] bk_host_id_map = {} for host in hosts: - host_ip_cloud_list.append({"ip": host.inner_ip, "bk_cloud_id": host.bk_cloud_id}) - bk_host_id_map[f"{host.inner_ip}:{host.bk_cloud_id}"] = host.bk_host_id + if host.bk_agent_id: + agent_id = host.bk_agent_id + else: + agent_id = f"{host.bk_cloud_id}:{host.inner_ip}" + host_ip_cloud_list.append({"ip": host.inner_ip, "bk_cloud_id": host.bk_cloud_id, "agent_id": agent_id}) + bk_host_id_map[agent_id] = host.bk_host_id for proc_name in sync_proc_list: past = time.time() - proc_statues = query_proc_status(proc_name=proc_name, host_list=host_ip_cloud_list) + proc_statuses = query_proc_status(proc_name=proc_name, host_list=host_ip_cloud_list) logger.info(f"this get_proc_status cost {time.time() - past}s") host_key__readable_proc_status_map: typing.Dict[ str, typing.Dict[str, typing.Any] - ] = proc_statues2host_key__readable_proc_status_map(proc_statues) + ] = proc_statues2host_key__readable_proc_status_map(proc_statuses) process_status_objs = ProcessStatus.objects.filter( name=proc_name, @@ -154,22 +164,26 @@ def update_or_create_proc_status(task_id, hosts, sync_proc_list, start): def sync_proc_status_periodic_task(): sync_proc_list = tools.PluginV2Tools.fetch_head_plugins() task_id = sync_proc_status_periodic_task.request.id - hosts = Host.objects.all() - count = hosts.count() - logger.info(f"{task_id} | sync host proc status... host_count={count}.") - - for start in range(0, count, constants.QUERY_PROC_STATUS_HOST_LENS): - countdown = calculate_countdown( - count=count / constants.QUERY_PROC_STATUS_HOST_LENS, - index=start / constants.QUERY_PROC_STATUS_HOST_LENS, - duration=constants.SYNC_PROC_STATUS_TASK_INTERVAL, - ) - logger.info(f"{task_id} | sync host proc status after {countdown} seconds") - - # (task_id, hosts[start: start + constants.QUERY_PROC_STATUS_HOST_LENS], sync_proc_list, start) - update_or_create_proc_status.apply_async( - (task_id, hosts[start : start + constants.QUERY_PROC_STATUS_HOST_LENS], sync_proc_list, start), - countdown=countdown, - ) - - logger.info(f"{task_id} | sync host proc status complate.") + host_queryset_list = [ + Host.objects.exclude(bk_agent_id__isnull=True).exclude(bk_agent_id__exact=""), + Host.objects.filter(Q(bk_agent_id__isnull=True) | Q(bk_agent_id__exact="")), + ] + for host_queryset in host_queryset_list: + count = host_queryset.count() + logger.info(f"{task_id} | sync host proc status... host_count={count}.") + + for start in range(0, count, constants.QUERY_PROC_STATUS_HOST_LENS): + countdown = calculate_countdown( + count=count / constants.QUERY_PROC_STATUS_HOST_LENS, + index=start / constants.QUERY_PROC_STATUS_HOST_LENS, + duration=constants.SYNC_PROC_STATUS_TASK_INTERVAL, + ) + logger.info(f"{task_id} | sync host proc status after {countdown} seconds") + + # (task_id, hosts[start: start + constants.QUERY_PROC_STATUS_HOST_LENS], sync_proc_list, start) + update_or_create_proc_status.apply_async( + (task_id, host_queryset[start : start + constants.QUERY_PROC_STATUS_HOST_LENS], sync_proc_list, start), + countdown=countdown, + ) + + logger.info(f"{task_id} | sync host proc status complete.") diff --git a/apps/node_man/serializers/job.py b/apps/node_man/serializers/job.py index 9d6cb55de..9f15ac549 100644 --- a/apps/node_man/serializers/job.py +++ b/apps/node_man/serializers/job.py @@ -52,10 +52,12 @@ class HostSerializer(serializers.Serializer): bk_host_id = serializers.IntegerField(label=_("主机ID"), required=False) ap_id = serializers.IntegerField(label=_("接入点ID"), required=False) install_channel_id = serializers.IntegerField(label=_("安装通道ID"), required=False, allow_null=True) - inner_ip = serializers.IPAddressField(label=_("内网IP")) - outer_ip = serializers.IPAddressField(label=_("外网IP"), required=False) - login_ip = serializers.IPAddressField(label=_("登录IP"), required=False) - data_ip = serializers.IPAddressField(label=_("数据IP"), required=False) + inner_ip = serializers.IPAddressField(label=_("内网IP"), required=False, allow_blank=True) + outer_ip = serializers.IPAddressField(label=_("外网IP"), required=False, allow_blank=True) + login_ip = serializers.IPAddressField(label=_("登录IP"), required=False, allow_blank=True) + data_ip = serializers.IPAddressField(label=_("数据IP"), required=False, allow_blank=True) + inner_ipv6 = serializers.IPAddressField(label=_("内网IPv6"), required=False, allow_blank=True) + outer_ipv6 = serializers.IPAddressField(label=_("外网IPv6"), required=False, allow_blank=True) os_type = serializers.ChoiceField(label=_("操作系统"), choices=list(constants.OS_TUPLE)) auth_type = serializers.ChoiceField(label=_("认证类型"), choices=list(constants.AUTH_TUPLE), required=False) account = serializers.CharField(label=_("账户"), required=False, allow_blank=True) @@ -72,6 +74,12 @@ def validate(self, attrs): # 获取任务类型，如果是除安装以外的操作，则密码和秘钥可以为空 job_type = self.root.initial_data.get("job_type", "") op_type = job_type.split("_")[0] + node_type = job_type.split("_")[1] + + if not (attrs.get("inner_ip") or attrs.get("inner_ipv6")): + raise ValidationError(_("请求参数 inner_ip 和 inner_ipv6 不能同时为空")) + if node_type == constants.NodeType.PROXY and not (attrs.get("outer_ip") or attrs.get("outer_ipv6")): + raise ValidationError(_("Proxy 操作的请求参数 outer_ip 和 outer_ipv6 不能同时为空")) op_not_need_identity = [ constants.OpType.REINSTALL, @@ -82,13 +90,13 @@ def validate(self, attrs): ] if op_type in op_not_need_identity and not attrs.get("bk_host_id"): - raise ValidationError(_("{op_type} 操作必须填写bk_host_id.").format(op_type=op_type)) + raise ValidationError(_("{op_type} 操作必须传入 bk_host_id 参数").format(op_type=op_type)) if ( not attrs.get("is_manual") and not attrs.get("auth_type") and job_type not in [constants.JobType.RELOAD_AGENT, constants.JobType.RELOAD_PROXY] ): - raise ValidationError(_("{op_type} 操作必须填写认证类型.").format(op_type=op_type)) + raise ValidationError(_("{op_type} 操作必须填写认证类型").format(op_type=op_type)) # identity校验 if op_type not in op_not_need_identity and not attrs.get("is_manual"): @@ -101,7 +109,7 @@ def validate(self, attrs): # 直连区域必须填写Ap_id if attrs["bk_cloud_id"] == int(constants.DEFAULT_CLOUD) and attrs.get("ap_id") is None: - raise ValidationError(_("直连区域必须填写Ap_id.")) + raise ValidationError(_("直连区域必须填写Ap_id")) # 去除空值 if attrs.get("key") == "": @@ -121,7 +129,6 @@ class InstallSerializer(serializers.Serializer): # 以下非参数传值 op_type = serializers.CharField(label=_("操作类型"), required=False) node_type = serializers.CharField(label=_("节点类型"), required=False) - tcoa_ticket = serializers.CharField(label=_("OA Ticket"), required=False) def validate(self, attrs): # 取得节点类型 @@ -131,7 +138,7 @@ def validate(self, attrs): # 替换PROXY必须填写replace_host_id if attrs["job_type"] == constants.JobType.REPLACE_PROXY and not attrs.get("replace_host_id"): - raise ValidationError(_("替换PROXY必须填写replace_host_id.")) + raise ValidationError(_("替换PROXY必须填写replace_host_id")) expected_bk_host_ids_gby_bk_biz_id: typing.Dict[int, typing.List[int]] = defaultdict(list) rsa_util = tools.HostTools.get_rsa_util() @@ -180,12 +187,12 @@ def validate(self, attrs): attrs["node_type"] = attrs["job_type"].split("_")[1] if attrs["op_type"] == "INSTALL": - raise ValidationError(_("该接口不可用于安装.")) + raise ValidationError(_("该接口不可用于安装")) if attrs["op_type"] == "REPLACE": - raise ValidationError(_("该接口不可用于替换代理.")) + raise ValidationError(_("该接口不可用于替换代理")) if attrs.get("exclude_hosts") is not None and attrs.get("bk_host_id") is not None: - raise ValidationError(_("跨页全选模式下不允许传bk_host_id参数.")) + raise ValidationError(_("跨页全选模式下不允许传bk_host_id参数")) if attrs.get("exclude_hosts") is None and attrs.get("bk_host_id") is None: raise ValidationError(_("必须选择一种模式(【是否跨页全选】)")) diff --git a/apps/node_man/tests/test_handlers/__init__.py b/apps/node_man/tests/test_handlers/__init__.py new file mode 100644 index 000000000..29ed269e0 --- /dev/null +++ b/apps/node_man/tests/test_handlers/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/apps/node_man/tests/test_ap.py b/apps/node_man/tests/test_handlers/test_ap.py similarity index 100% rename from apps/node_man/tests/test_ap.py rename to apps/node_man/tests/test_handlers/test_ap.py diff --git a/apps/node_man/tests/test_cloud.py b/apps/node_man/tests/test_handlers/test_cloud.py similarity index 100% rename from apps/node_man/tests/test_cloud.py rename to apps/node_man/tests/test_handlers/test_cloud.py diff --git a/apps/node_man/tests/test_cmdb.py b/apps/node_man/tests/test_handlers/test_cmdb.py similarity index 100% rename from apps/node_man/tests/test_cmdb.py rename to apps/node_man/tests/test_handlers/test_cmdb.py diff --git a/apps/node_man/tests/test_debug.py b/apps/node_man/tests/test_handlers/test_debug.py similarity index 100% rename from apps/node_man/tests/test_debug.py rename to apps/node_man/tests/test_handlers/test_debug.py diff --git a/apps/node_man/tests/test_healthz.py b/apps/node_man/tests/test_handlers/test_healthz.py similarity index 100% rename from apps/node_man/tests/test_healthz.py rename to apps/node_man/tests/test_handlers/test_healthz.py diff --git a/apps/node_man/tests/test_host.py b/apps/node_man/tests/test_handlers/test_host.py similarity index 100% rename from apps/node_man/tests/test_host.py rename to apps/node_man/tests/test_handlers/test_host.py diff --git a/apps/node_man/tests/test_host_v2.py b/apps/node_man/tests/test_handlers/test_host_v2.py similarity index 100% rename from apps/node_man/tests/test_host_v2.py rename to apps/node_man/tests/test_handlers/test_host_v2.py diff --git a/apps/node_man/tests/test_iam.py b/apps/node_man/tests/test_handlers/test_iam.py similarity index 98% rename from apps/node_man/tests/test_iam.py rename to apps/node_man/tests/test_handlers/test_iam.py index c09140e89..799beffe5 100644 --- a/apps/node_man/tests/test_iam.py +++ b/apps/node_man/tests/test_handlers/test_iam.py @@ -15,7 +15,7 @@ from apps.node_man.handlers.iam import IamHandler from apps.utils.unittest.testcase import CustomBaseTestCase -from .utils import MockIAM, create_cloud_area +from ..utils import MockIAM, create_cloud_area class TestIAM(CustomBaseTestCase): diff --git a/apps/node_man/tests/test_install_channel.py b/apps/node_man/tests/test_handlers/test_install_channel.py similarity index 100% rename from apps/node_man/tests/test_install_channel.py rename to apps/node_man/tests/test_handlers/test_install_channel.py diff --git a/apps/node_man/tests/test_job.py b/apps/node_man/tests/test_handlers/test_job.py similarity index 99% rename from apps/node_man/tests/test_job.py rename to apps/node_man/tests/test_handlers/test_job.py index fdc8f31d7..fdce427bb 100644 --- a/apps/node_man/tests/test_job.py +++ b/apps/node_man/tests/test_handlers/test_job.py @@ -160,7 +160,7 @@ def test_host_update(self): host_to_create, process_to_create, identity_to_create = create_host(number) # 创建完毕，进行修改 accept_list = gen_update_accept_list(host_to_create, identity_to_create) - host_ids, _ = JobHandler().update(accept_list, []) + host_ids, _ = JobHandler().update_host(accept_list, []) self.assertEqual(len(host_ids), number) @patch("apps.node_man.handlers.cmdb.client_v2", MockClient) diff --git a/apps/node_man/tests/test_meta.py b/apps/node_man/tests/test_handlers/test_meta.py similarity index 100% rename from apps/node_man/tests/test_meta.py rename to apps/node_man/tests/test_handlers/test_meta.py diff --git a/apps/node_man/tests/test_permission.py b/apps/node_man/tests/test_handlers/test_permission.py similarity index 100% rename from apps/node_man/tests/test_permission.py rename to apps/node_man/tests/test_handlers/test_permission.py diff --git a/apps/node_man/tests/test_plugin.py b/apps/node_man/tests/test_handlers/test_plugin.py similarity index 100% rename from apps/node_man/tests/test_plugin.py rename to apps/node_man/tests/test_handlers/test_plugin.py diff --git a/apps/node_man/tests/test_pluginv2.py b/apps/node_man/tests/test_handlers/test_pluginv2.py similarity index 100% rename from apps/node_man/tests/test_pluginv2.py rename to apps/node_man/tests/test_handlers/test_pluginv2.py diff --git a/apps/node_man/tests/test_policy.py b/apps/node_man/tests/test_handlers/test_policy.py similarity index 100% rename from apps/node_man/tests/test_policy.py rename to apps/node_man/tests/test_handlers/test_policy.py diff --git a/apps/node_man/tests/test_tjj.py b/apps/node_man/tests/test_handlers/test_tjj.py similarity index 100% rename from apps/node_man/tests/test_tjj.py rename to apps/node_man/tests/test_handlers/test_tjj.py diff --git a/apps/node_man/tests/test_validator.py b/apps/node_man/tests/test_handlers/test_validator.py similarity index 100% rename from apps/node_man/tests/test_validator.py rename to apps/node_man/tests/test_handlers/test_validator.py diff --git a/apps/node_man/tests/test_pericdic_tasks/mock_data.py b/apps/node_man/tests/test_pericdic_tasks/mock_data.py index c9f170811..0ec2a3c30 100644 --- a/apps/node_man/tests/test_pericdic_tasks/mock_data.py +++ b/apps/node_man/tests/test_pericdic_tasks/mock_data.py @@ -21,6 +21,7 @@ MOCK_BK_CLOUD_ID = 99999 MOCK_IP = "127.0.0.1" MOCK_HOST_ID = 233333 +MOCK_AGENT_ID = "35103d95b447465a944384968d6222cc35103d95b447465a944384968d6222cc" # 主机数量(用于批量构造) MOCK_HOST_NUM = 10 # 其他各种ID diff --git a/apps/node_man/tests/test_pericdic_tasks/test_sync_agent_status_task.py b/apps/node_man/tests/test_pericdic_tasks/test_sync_agent_status_task.py index 4bb9b4e8c..939b01850 100644 --- a/apps/node_man/tests/test_pericdic_tasks/test_sync_agent_status_task.py +++ b/apps/node_man/tests/test_pericdic_tasks/test_sync_agent_status_task.py @@ -8,38 +8,80 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - from unittest.mock import patch +from django.test import override_settings + +from apps.mock_data.api_mkd.gse.unit import GSE_PROCESS_VERSION +from apps.mock_data.api_mkd.gse.utils import GseApiMockClient +from apps.mock_data.common_unit.host import ( + HOST_MODEL_DATA, + HOST_MODEL_DATA_WITH_AGENT_ID, +) from apps.node_man import constants from apps.node_man.models import Host, ProcessStatus from apps.node_man.periodic_tasks.sync_agent_status_task import ( + sync_agent_status_periodic_task, update_or_create_host_agent_status, ) from apps.utils.unittest.testcase import CustomBaseTestCase -from .mock_data import MOCK_GET_AGENT_STATUS, MOCK_HOST -from .utils import MockClient, modify_constant_data - class TestSyncAgentStatus(CustomBaseTestCase): - @patch("apps.node_man.periodic_tasks.sync_agent_status_task.client_v2", MockClient) - def test_update_or_create_host_agent_status(self): - host, _ = Host.objects.get_or_create(**MOCK_HOST) + def test_sync_agent_status_periodic_task(self): + Host.objects.create(**HOST_MODEL_DATA) + sync_agent_status_periodic_task() + + def test_update_or_create_host_agent_status_no_host(self): + update_or_create_host_agent_status(None, 0, 1) + self.assertEqual(ProcessStatus.objects.count(), 0) + + @patch("apps.node_man.periodic_tasks.sync_agent_status_task.GseApi", GseApiMockClient()) + def test_update_or_create_host_agent_status_alive(self): + host = Host.objects.create(**HOST_MODEL_DATA) + # 测试创建ProcessStatus对象 + update_or_create_host_agent_status(None, 0, 1) + process_status = ProcessStatus.objects.get(bk_host_id=host.bk_host_id) + self.assertEqual(process_status.status, constants.ProcStateType.RUNNING) + self.assertEqual(process_status.version, GSE_PROCESS_VERSION) + host = Host.objects.get(bk_host_id=host.bk_host_id) + self.assertEqual(host.node_from, constants.NodeFrom.NODE_MAN) + @patch( + "apps.node_man.periodic_tasks.sync_agent_status_task.GseApi", + GseApiMockClient(get_agent_status_return=GseApiMockClient.GET_AGENT_NOT_ALIVE_STATUS_RETURN), + ) + def test_update_or_create_host_agent_status_not_alive(self): + host = Host.objects.create(**HOST_MODEL_DATA) # 测试创建ProcessStatus对象 update_or_create_host_agent_status(None, 0, 1) - self.assertEqual(ProcessStatus.objects.count(), 1) + process_status = ProcessStatus.objects.get(bk_host_id=host.bk_host_id) + self.assertEqual(process_status.status, constants.ProcStateType.NOT_INSTALLED) + + # Agent异常且已在节点管理录入过的，标记状态为异常 + host.node_from = constants.NodeFrom.NODE_MAN + host.save() + update_or_create_host_agent_status(None, 0, 1) + process_status = ProcessStatus.objects.get(bk_host_id=host.bk_host_id) + self.assertEqual(process_status.status, constants.ProcStateType.TERMINATED) - # agent状态为alive时 更新主机信息node_from为NODE_MAN + @override_settings(GSE_VERSION="V2") + @patch("apps.node_man.periodic_tasks.sync_agent_status_task.GseApi", GseApiMockClient()) + def test_update_or_create_host_agent_status_alive_gse_v2(self): + host = Host.objects.create(**HOST_MODEL_DATA_WITH_AGENT_ID) + # 测试创建ProcessStatus对象 + update_or_create_host_agent_status(None, 0, 1, has_agent_id=True) + process_status = ProcessStatus.objects.get(bk_host_id=host.bk_host_id) + self.assertEqual(process_status.status, constants.ProcStateType.RUNNING) + + @override_settings(GSE_VERSION="V2") + @patch( + "apps.node_man.periodic_tasks.sync_agent_status_task.GseApi", + GseApiMockClient(get_agent_state_list_return=GseApiMockClient.GET_AGENT_NOT_ALIVE_STATE_LIST_RETURN), + ) + def test_update_or_create_host_agent_status_not_alive_gse_v2(self): + host = Host.objects.create(**HOST_MODEL_DATA) + # 测试创建ProcessStatus对象 update_or_create_host_agent_status(None, 0, 1) - self.assertEqual(Host.objects.get(bk_host_id=host.bk_host_id).node_from, constants.NodeFrom.NODE_MAN) - - # agent状态信息为not alive时 更新进程信息为NOT_INSTALLED/TERMINATED - with modify_constant_data( - [(MOCK_GET_AGENT_STATUS[f"{host.bk_cloud_id}:{host.inner_ip}"], {"bk_agent_alive": 0})] - ): - update_or_create_host_agent_status(None, 0, 1) - self.assertEqual( - ProcessStatus.objects.get(bk_host_id=host.bk_host_id).status, constants.PROC_STATUS_DICT[2] - ) + process_status = ProcessStatus.objects.get(bk_host_id=host.bk_host_id) + self.assertEqual(process_status.status, constants.ProcStateType.NOT_INSTALLED) diff --git a/apps/node_man/tests/test_pericdic_tasks/test_sync_proc_status_task.py b/apps/node_man/tests/test_pericdic_tasks/test_sync_proc_status_task.py index bda8b4962..5fcde4ee4 100644 --- a/apps/node_man/tests/test_pericdic_tasks/test_sync_proc_status_task.py +++ b/apps/node_man/tests/test_pericdic_tasks/test_sync_proc_status_task.py @@ -9,25 +9,25 @@ specific language governing permissions and limitations under the License. """ -import contextlib import importlib import mock +from apps.mock_data.api_mkd.gse.unit import GSE_PROCESS_NAME +from apps.mock_data.api_mkd.gse.utils import GseApiMockClient +from apps.mock_data.common_unit.host import ( + HOST_MODEL_DATA, + HOST_MODEL_DATA_WITH_AGENT_ID, +) from apps.node_man import constants from apps.node_man.models import Host, ProcessStatus from apps.node_man.periodic_tasks import sync_proc_status_task -from apps.node_man.tests.test_pericdic_tasks.mock_data import ( - MOCK_HOST, - MOCK_PROC_NAME, - MOCK_PROC_STATUS, -) -from apps.node_man.tests.test_pericdic_tasks.utils import MockClient from apps.utils import concurrent from apps.utils.unittest.testcase import CustomBaseTestCase class TestSyncProcStatus(CustomBaseTestCase): + @classmethod def setUpClass(cls): mock.patch("apps.utils.concurrent.batch_call", concurrent.batch_call_serial).start() @@ -37,35 +37,32 @@ def setUp(self) -> None: importlib.reload(sync_proc_status_task) super().setUp() - @contextlib.contextmanager - def init_db(self): - # 创造一个虚拟主机和虚拟进程信息 - mock_host, _ = Host.objects.get_or_create(**MOCK_HOST) - mock_proc, _ = ProcessStatus.objects.get_or_create(**MOCK_PROC_STATUS) + def test_sync_proc_status_periodic_task(self): + Host.objects.create(**HOST_MODEL_DATA) + sync_proc_status_task.sync_proc_status_periodic_task() - yield mock_host - - # 清除相关数据 - Host.objects.get(bk_host_id=MOCK_HOST["bk_host_id"]).delete() - ProcessStatus.objects.get(bk_host_id=MOCK_HOST["bk_host_id"], name=MOCK_PROC_STATUS["name"]).delete() - - @mock.patch("apps.node_man.periodic_tasks.sync_proc_status_task.client_v2", MockClient) + @mock.patch("apps.node_man.periodic_tasks.sync_proc_status_task.GseApi", GseApiMockClient()) def test_update_or_create_proc_status(self, *args, **kwargs): - with self.init_db() as mock_host: - # 测试update_proc_status - # result = query_proc_status("test_proc", [{"ip": "127.0.0.1", "bk_cloud_id": 0}]) - sync_proc_status_task.update_or_create_proc_status(None, [mock_host], [MOCK_PROC_NAME], 0) - mock_proc = ProcessStatus.objects.get(bk_host_id=MOCK_HOST["bk_host_id"], name=MOCK_PROC_NAME) - self.assertEqual( - [mock_proc.status, mock_proc.bk_host_id, mock_proc.name], - [constants.ProcStateType.RUNNING, MOCK_HOST["bk_host_id"], MOCK_PROC_NAME], - ) + host = Host.objects.create(**HOST_MODEL_DATA) + # 测试新建proc_status + sync_proc_status_task.update_or_create_proc_status(None, [host], [GSE_PROCESS_NAME], 0) + mock_proc = ProcessStatus.objects.get(bk_host_id=host.bk_host_id, name=GSE_PROCESS_NAME) + self.assertEqual( + [mock_proc.status, mock_proc.bk_host_id, mock_proc.name], + [constants.ProcStateType.TERMINATED, host.bk_host_id, GSE_PROCESS_NAME], + ) + + # 测试update_proc_status + mock_proc = ProcessStatus.objects.get(bk_host_id=host.bk_host_id, name=GSE_PROCESS_NAME) + mock_proc.status = constants.ProcStateType.MANUAL_STOP + mock_proc.save() + sync_proc_status_task.update_or_create_proc_status(None, [host], [GSE_PROCESS_NAME], 0) + self.assertEqual( + [mock_proc.status, mock_proc.bk_host_id, mock_proc.name], + [constants.ProcStateType.MANUAL_STOP, host.bk_host_id, GSE_PROCESS_NAME], + ) - # 测试create_proc_status 先删掉存在的proc_status - ProcessStatus.objects.get(bk_host_id=MOCK_HOST["bk_host_id"], name=MOCK_PROC_NAME).delete() - sync_proc_status_task.update_or_create_proc_status(None, [mock_host], [MOCK_PROC_NAME], 0) - mock_proc = ProcessStatus.objects.get(bk_host_id=MOCK_HOST["bk_host_id"], name=MOCK_PROC_NAME) - self.assertEqual( - [mock_proc.status, mock_proc.bk_host_id, mock_proc.name], - [constants.ProcStateType.RUNNING, MOCK_HOST["bk_host_id"], MOCK_PROC_NAME], - ) + @mock.patch("apps.node_man.periodic_tasks.sync_proc_status_task.GseApi", GseApiMockClient()) + def test_update_or_create_proc_status_with_agent_id(self, *args, **kwargs): + host = Host.objects.create(**HOST_MODEL_DATA_WITH_AGENT_ID) + sync_proc_status_task.update_or_create_proc_status(None, [host], [GSE_PROCESS_NAME], 0) diff --git a/apps/node_man/tests/test_pericdic_tasks/utils.py b/apps/node_man/tests/test_pericdic_tasks/utils.py index 056d20d78..21758503f 100644 --- a/apps/node_man/tests/test_pericdic_tasks/utils.py +++ b/apps/node_man/tests/test_pericdic_tasks/utils.py @@ -47,6 +47,7 @@ def list_resource_pool_hosts(cls, *args, **kwargs): for index in range(count): MOCK_HOST_TMP["bk_host_id"] = index MOCK_HOST_TMP["bk_cloud_id"] = index + MOCK_HOST_TMP["bk_agent_id"] = index MOCK_HOST_TMP["bk_host_innerip"] = MOCK_HOST_TMP["inner_ip"] MOCK_HOST_TMP["bk_host_outerip"] = MOCK_HOST_TMP["outer_ip"] host_list.append(copy.deepcopy(MOCK_HOST_TMP)) diff --git a/apps/node_man/tests/test_views/__init__.py b/apps/node_man/tests/test_views/__init__.py new file mode 100644 index 000000000..29ed269e0 --- /dev/null +++ b/apps/node_man/tests/test_views/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/apps/node_man/tests/test_views.py b/apps/node_man/tests/test_views/test_index_views.py similarity index 100% rename from apps/node_man/tests/test_views.py rename to apps/node_man/tests/test_views/test_index_views.py diff --git a/apps/node_man/tests/test_views/test_job_views.py b/apps/node_man/tests/test_views/test_job_views.py new file mode 100644 index 000000000..5636893bf --- /dev/null +++ b/apps/node_man/tests/test_views/test_job_views.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import abc +import copy +from unittest.mock import patch + +from apps.exceptions import ValidationError +from apps.mock_data.views_mkd import job +from apps.node_man import constants +from apps.node_man.tests.utils import Subscription +from apps.utils.unittest.testcase import CustomAPITestCase, MockSuperUserMixin + + +class JobViewsTestCase(MockSuperUserMixin, CustomAPITestCase): + @patch("apps.node_man.handlers.job.JobHandler.create_subscription", Subscription.create_subscription) + def test_install(self): + data = copy.deepcopy(job.JOB_INSTALL_REQUEST_PARAMS) + result = self.client.post(path="/api/job/install/", data=data) + self.assertEqual(result["result"], True) + + @patch("apps.node_man.handlers.job.JobHandler.create_subscription", Subscription.create_subscription) + def test_operate(self): + data = copy.deepcopy(job.JOB_OPERATE_REQUEST_PARAMS) + result = self.client.post(path="/api/job/operate/", data=data) + self.assertEqual(result["result"], True) + + +class TestJobValidationError(JobViewsTestCase, metaclass=abc.ABCMeta): + ERROR_MSG_KEYWORD = "" + + @staticmethod + def generate_install_job_request_params(): + data = copy.deepcopy(job.JOB_INSTALL_REQUEST_PARAMS) + return data + + @patch("apps.node_man.handlers.job.JobHandler.create_subscription", Subscription.create_subscription) + def test_install_raise_validation_error(self): + data = self.generate_install_job_request_params() + result = self.client.post(path="/api/job/install/", data=data) + if self.ERROR_MSG_KEYWORD: + self.assertEqual(result["code"], ValidationError().code) + self.assertIn(self.ERROR_MSG_KEYWORD, result["message"]) + + +class TestInnerIpNotEmptyAtTheSameTimeError(TestJobValidationError): + ERROR_MSG_KEYWORD = "请求参数 inner_ip 和 inner_ipv6 不能同时为空" + + @staticmethod + def generate_install_job_request_params(): + data = copy.deepcopy(job.JOB_INSTALL_REQUEST_PARAMS) + data["hosts"][0]["inner_ip"] = "" + data["hosts"][0]["inner_ipv6"] = "" + return data + + +class TestOuterIpNotEmptyAtTheSameTimeError(TestJobValidationError): + ERROR_MSG_KEYWORD = "Proxy 操作的请求参数 outer_ip 和 outer_ipv6 不能同时为空" + + @staticmethod + def generate_install_job_request_params(): + data = copy.deepcopy(job.JOB_INSTALL_REQUEST_PARAMS) + data["job_type"] = constants.JobType.INSTALL_PROXY + data["hosts"][0]["outer_ip"] = "" + data["hosts"][0]["outer_ipv6"] = "" + return data + + +class TestBkHostIdNeededError(TestJobValidationError): + ERROR_MSG_KEYWORD = "操作必须传入 bk_host_id 参数" + + @staticmethod + def generate_install_job_request_params(): + data = copy.deepcopy(job.JOB_INSTALL_REQUEST_PARAMS) + data["job_type"] = constants.JobType.REINSTALL_AGENT + data["hosts"][0].pop("bk_host_id", "") + return data diff --git a/apps/node_man/views/job.py b/apps/node_man/views/job.py index 57b525d3d..90e9072e0 100644 --- a/apps/node_man/views/job.py +++ b/apps/node_man/views/job.py @@ -137,8 +137,7 @@ def install(self, request): @apiParam {String} hosts.auth_type 认证类型 @apiParam {String} [hosts.password] 密码 @apiParam {String} [hosts.key] 密钥 - @apiParam {Number} [hosts.retention] 密码保留天数 - @apiParam {Number} [replace_host_id] 要替换的ProxyID，替换proxy时使用 + @apiParam {Number} [hosts.retention] 密码保留天数，默认只保留1天 @apiParamExample {Json} 安装请求参数 { "job_type": "INSTALL_AGENT", @@ -160,7 +159,6 @@ def install(self, request): } ], "retention": 1, - "replace_host_id": 1 } @apiSuccessExample {json} 成功返回: { diff --git a/apps/node_man/views/password.py b/apps/node_man/views/password.py index f2b96e143..e882348ac 100644 --- a/apps/node_man/views/password.py +++ b/apps/node_man/views/password.py @@ -51,9 +51,7 @@ def fetch_pwd(self, request, *args, **kwargs): } """ - result = password.TjjPasswordHandler().fetch_pwd( - request.user.username, request.data.get("hosts", []), ticket=request.COOKIES.get("TCOA_TICKET") - ) + result = password.TjjPasswordHandler().fetch_pwd() # ticket过期返回401 if not result["result"] and ("ticket is expired" in result["message"] or "OATicket" in result["message"]): diff --git a/apps/utils/unittest/testcase.py b/apps/utils/unittest/testcase.py index f96f906a9..6da1633b7 100644 --- a/apps/utils/unittest/testcase.py +++ b/apps/utils/unittest/testcase.py @@ -13,7 +13,8 @@ from typing import Any, Dict, List, Optional, Union from unittest import mock -from django.contrib.auth.models import User +from django.conf import settings +from django.contrib.auth import get_user_model from django.test import Client, RequestFactory, TestCase from apps.utils.unittest import base @@ -138,17 +139,15 @@ def setUpTestData(cls): super().setUpTestData() # 创建用于测试的超管 - cls.superuser = User.objects.create_user( - username=cls.SUPERUSER_NAME, - password=cls.SUPERUSER_PASSWORD, - is_superuser=True, - is_staff=True, - is_active=True, - ) + user_model = get_user_model() + user = user_model.objects.first() + user.set_password(cls.SUPERUSER_PASSWORD) + user.save() def setUp(self) -> None: super().setUp() - self.client.login(username=self.SUPERUSER_NAME, password=self.SUPERUSER_PASSWORD) + settings.AUTHENTICATION_BACKENDS = ["django.contrib.auth.backends.ModelBackend"] + self.client.login(username=self.SUPERUSER_NAME, password=self.SUPERUSER_NAME) def tearDown(self) -> None: super().tearDown() diff --git a/common/api/modules/gse.py b/common/api/modules/gse.py index db425fe19..918eba169 100644 --- a/common/api/modules/gse.py +++ b/common/api/modules/gse.py @@ -67,3 +67,15 @@ def __init__(self): module=self.MODULE, description="获取Agent状态", ) + self.get_agent_info_list = DataAPI( + method="POST", + url=GSE_APIGATEWAY_ROOT_V2 + "get_agent_info_list/", + module=self.MODULE, + description="获取Agent版本信息", + ) + self.get_agent_state_list = DataAPI( + method="POST", + url=GSE_APIGATEWAY_ROOT_V2 + "get_agent_state_list/", + module=self.MODULE, + description="GSE2.0 获取Agent状态", + ) diff --git a/config/default.py b/config/default.py index 7b5162b2c..fce67bc43 100644 --- a/config/default.py +++ b/config/default.py @@ -136,6 +136,8 @@ def redirect_func(request): BLUEKING_BIZ_ID = 9991001 # 作业平台版本 JOB_VERSION = os.getenv("JOB_VERSION") or "V3" +# 管控平台平台版本 +GSE_VERSION = os.getenv("GSE_VERSION") or "V1" # 资源池业务ID BK_CMDB_RESOURCE_POOL_BIZ_ID = int(os.getenv("BK_CMDB_RESOURCE_POOL_BIZ_ID", 1)) or 1 BK_CMDB_RESOURCE_POOL_BIZ_NAME = "资源池"