feature: 支持bk_agent_id (close #562)

apps/backend/agent/manager.py

@@ -138,6 +138,14 @@ def get_agent_status(cls, expect_status: str, name=components.GetAgentStatusComp
         act.component.inputs.expect_status = Var(type=Var.PLAIN, value=expect_status)
         return act
 
+    @classmethod
+    def get_agent_id(cls, name=components.GetAgentIDComponent.name):
+        """查询 Agent ID"""
+        if settings.GSE_VERSION == "V1":
+            return None
+        act = AgentServiceActivity(component_code=components.GetAgentIDComponent.code, name=name)
+        return act
+
     @classmethod
     def check_agent_status(cls, name=components.CheckAgentStatusComponent.name):
         """查询Agent状态是否正常"""

apps/backend/agent/tools.py

@@ -250,7 +250,9 @@ def gen_commands(
     agent_config = host_ap.get_agent_config(host.os_type)
     # 安装操作
     install_path = agent_config["setup_path"]
-    token = aes_cipher.encrypt(f"{host.inner_ip}|{host.bk_cloud_id}|{pipeline_id}|{time.time()}|{sub_inst_id}")
+    token = aes_cipher.encrypt(
+        f"{host.bk_host_id}|{host.inner_ip}|{host.bk_cloud_id}|{pipeline_id}|{time.time()}|{sub_inst_id}"
+    )
     port_config = host_ap.port_config
     run_cmd_params = [
         f"-s {pipeline_id}",
@@ -309,8 +311,9 @@ def gen_commands(
         host_shell = format_run_cmd_by_os_type(host.os_type, host_tmp_path)
         run_cmd_params.extend(
             [
-                f"-HLIP {host.login_ip or host.inner_ip}",
+                f"-HLIP {host.login_ip or host.inner_ip or host.inner_ipv6}",
                 f"-HIIP {host.inner_ip}",
+                f"-HIIP6 {host.inner_ipv6}" if host.inner_ipv6 else "",
                 f"-HA {identity_data.account}",
                 f"-HP {identity_data.port}",
                 f"-HI '{host_identity}'",
@@ -323,6 +326,8 @@ def gen_commands(
                 f"-HS '{host_shell}'",
                 f"-p '{install_path}'",
                 f"-I {jump_server.inner_ip}",
+                f"-I6 {jump_server.inner_ipv6}" if jump_server.inner_ipv6 else "",
+                f"-AI {host.bk_agent_id}" if host.bk_agent_id else "",
                 f"-o {gen_nginx_download_url(jump_server.inner_ip)}",
                 f"-HEP '{encrypted_password}'" if need_encrypted_password else "",
                 "-R" if is_uninstall else "",
@@ -354,6 +359,8 @@ def gen_commands(
             [
                 f"-i {host.bk_cloud_id}",
                 f"-I {host.inner_ip}",
+                f"-I6 {host.inner_ipv6}" if host.inner_ipv6 else "",
+                f"-AI {host.bk_agent_id}" if host.bk_agent_id else "",
                 "-N SERVER",
                 f"-p {install_path}",
                 f"-T {dest_dir}",

apps/backend/components/collections/agent_new/components.py

@@ -19,6 +19,7 @@
 from .choose_access_point import ChooseAccessPointService
 from .configure_policy import ConfigurePolicyService
 from .delegate_plugin_proc import DelegatePluginProcService
+from .get_agent_id import GetAgentIDService
 from .get_agent_status import GetAgentStatusService
 from .install import InstallService
 from .install_plugins import InstallPluginsService
@@ -83,6 +84,12 @@ class GetAgentStatusComponent(Component):
     bound_service = GetAgentStatusService
 
 
+class GetAgentIDComponent(Component):
+    name = _("查询AgentID")
+    code = "get_agent_id"
+    bound_service = GetAgentIDService
+
+
 class ReloadAgentConfigComponent(Component):
     name = _("重载Agent配置")
     code = "reload_agent_config"

apps/backend/components/collections/agent_new/get_agent_id.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from typing import Any, Dict, List
+
+from django.utils.translation import ugettext_lazy as _
+
+from apps.backend.api.constants import POLLING_INTERVAL, POLLING_TIMEOUT
+from apps.component.esbclient import client_v2
+from apps.node_man import models
+from apps.utils import batch_request
+from pipeline.core.flow import Service, StaticIntervalGenerator
+
+from .base import AgentBaseService, AgentCommonData
+
+
+class GetAgentIDService(AgentBaseService):
+    """安装后AgentID同步到CMDB后才认为是可用的"""
+
+    __need_schedule__ = True
+    interval = StaticIntervalGenerator(POLLING_INTERVAL)
+
+    def outputs_format(self):
+        return super().outputs_format() + [
+            Service.InputItem(name="polling_time", key="polling_time", type="int", required=True),
+        ]
+
+    @staticmethod
+    def update_host_agent_id(cmdb_host_infos: List[Dict[str, Any]]):
+        need_update_hosts = [
+            models.Host(bk_host_id=host_info["bk_host_id"], bk_agent_id=host_info.get("bk_agent_id"))
+            for host_info in cmdb_host_infos
+        ]
+        models.Host.objects.bulk_update(need_update_hosts, fields=["bk_agent_id"])
+
+    def _execute(self, data, parent_data, common_data: AgentCommonData):
+        self.log_info(sub_inst_ids=common_data.subscription_instance_ids, log_content=_("开始查询从 CMDB 查询主机的 Agent ID"))
+        data.outputs.polling_time = 0
+
+    def _schedule(self, data, parent_data, callback_data=None):
+        common_data: AgentCommonData = self.get_common_data(data)
+        list_cmdb_hosts_params = {
+            "fields": ["bk_host_id", "bk_agent_id"],
+            "host_property_filter": {
+                "condition": "AND",
+                "rules": [
+                    {"field": "bk_host_id", "operator": "in", "value": common_data.bk_host_ids},
+                ],
+            },
+        }
+        cmdb_host_infos: List[Dict[str, Any]] = batch_request.batch_request(
+            func=client_v2.cc.list_hosts_without_biz, params=list_cmdb_hosts_params
+        )
+
+        # CMDB 中 AgentID 为空的主机ID列表
+        no_agent_id_host_ids = [
+            host_info["bk_host_id"] for host_info in cmdb_host_infos if not host_info.get("bk_agent_id")
+        ]
+        if not no_agent_id_host_ids:
+            self.update_host_agent_id(cmdb_host_infos)
+            self.finish_schedule()
+            return
+
+        polling_time = data.get_one_of_outputs("polling_time")
+        if polling_time + POLLING_INTERVAL > POLLING_TIMEOUT:
+            sub_inst_ids = [common_data.host_id__sub_inst_id_map[host_id] for host_id in no_agent_id_host_ids]
+            self.move_insts_to_failed(
+                sub_inst_ids=sub_inst_ids,
+                log_content=_("此主机在 CMDB 中未查到对应 Agent ID，请联系 GSE 及 CMDB 团队排查 Agent ID 上报处理是否异常！"),
+            )
+            self.update_host_agent_id(cmdb_host_infos)
+            self.finish_schedule()
+            return
+
+        data.outputs.polling_time = polling_time + POLLING_INTERVAL

apps/backend/components/collections/agent_new/register_host.py

@@ -146,11 +146,13 @@ def add_hosts_to_biz_thread(
             host_key = f"{host_info['bk_cloud_id']}-{host_info['bk_host_innerip']}"
             register_params = {
                 index: {
-                    "bk_host_innerip": host_info["bk_host_innerip"],
+                    "bk_host_innerip": host_info.get("bk_host_innerip", ""),
+                    "bk_host_innerip_v6": host_info.get("bk_host_innerip_v6", ""),
                     # "3" 表示API导入
                     "import_from": "3",
                     "bk_cloud_id": host_info["bk_cloud_id"],
                     "bk_host_outerip": host_info.get("bk_host_outerip", ""),
+                    "bk_host_outerip_v6": host_info.get("bk_host_outerip_v6", ""),
                     "bk_os_type": constants.BK_OS_TYPE[host_info["os_type"]],
                     "bk_bak_operator": biz_info.get("bk_biz_maintainer") or host_info.get("username"),
                     "operator": biz_info.get("bk_biz_maintainer") or host_info.get("username"),

apps/backend/components/collections/common/remote.py

@@ -60,9 +60,9 @@ def conns_init_params(self) -> typing.Dict:
         :return:
         """
         if self.host.node_type == constants.NodeType.PROXY:
-            ip = self.host.login_ip or self.host.outer_ip
+            ip = self.host.login_ip or self.host.outer_ip or self.host.outer_ipv6
         else:
-            ip = self.host.login_ip or self.host.inner_ip
+            ip = self.host.login_ip or self.host.inner_ip or self.host.inner_ipv6
 
         client_key_strings = []
         if self.identity_data.auth_type == constants.AuthType.KEY:

apps/backend/components/collections/plugin.py

@@ -1144,7 +1144,10 @@ def _execute(self, data, parent_data, common_data: PluginCommonData):
             gse_op_params = {
                 "meta": {"namespace": constants.GSE_NAMESPACE, "name": meta_name},
                 "op_type": op_type,
+                # GSE 1.0 目标对象
                 "hosts": [{"ip": host.inner_ip, "bk_cloud_id": host.bk_cloud_id}],
+                # GSE 2.0 目标对象
+                "agent_id_list": [host.bk_agent_id],
                 # 此字段是节点管理自用，仅用于标识，不会被GSE使用
                 "nodeman_spec": {
                     "process_status_id": process_status.id,
@@ -1240,7 +1243,10 @@ def _schedule(self, data, parent_data, callback_data=None):
             subscription_instance = group_id_instance_map.get(process_status.group_id)
 
             proc_name = self.get_plugin_meta_name(plugin, process_status)
-            gse_proc_key = f"{host.bk_cloud_id}:{host.inner_ip}:{constants.GSE_NAMESPACE}:{proc_name}"
+            if host.bk_agent_id:
+                gse_proc_key = f"{host.bk_agent_id}:{constants.GSE_NAMESPACE}:{proc_name}"
+            else:
+                gse_proc_key = f"{host.bk_cloud_id}:{host.inner_ip}:{constants.GSE_NAMESPACE}:{proc_name}"
             proc_operate_result = result["data"].get(gse_proc_key)
             if not proc_operate_result:
                 self.move_insts_to_failed(

apps/backend/plugin/serializers.py

@@ -237,6 +237,7 @@ class PluginStartDebugSerializer(GatewaySerializer):
     """
 
     class HostInfoSerializer(serializers.Serializer):
+        bk_host_id = serializers.IntegerField(required=False)
         ip = serializers.CharField(required=True)
         bk_cloud_id = serializers.IntegerField(required=True)
         bk_supplier_id = serializers.IntegerField(default=constants.DEFAULT_SUPPLIER_ID)

apps/backend/plugin/views.py

@@ -51,6 +51,7 @@
 from apps.exceptions import AppBaseException, ValidationError
 from apps.generic import APIViewSet
 from apps.node_man import constants, models
+from apps.node_man.exceptions import HostNotExists
 from pipeline.engine.exceptions import InvalidOperationException
 from pipeline.service import task_service
 from pipeline.service.pipeline_engine_adapter.adapter_api import STATE_MAP
@@ -499,15 +500,15 @@ def start_debug(self, request):
         host_info = params["host_info"]
         plugin_name = params["plugin_name"]
         plugin_version = params["version"]
+        query_host_params = dict(
+            bk_biz_id=host_info["bk_biz_id"], inner_ip=host_info["ip"], bk_cloud_id=host_info["bk_cloud_id"]
+        )
+        if params.get("bk_host_id"):
+            query_host_params.update(bk_host_id=params["bk_host_id"])
 
-        try:
-            host = models.Host.objects.get(
-                bk_biz_id=host_info["bk_biz_id"],
-                inner_ip=host_info["ip"],
-                bk_cloud_id=host_info["bk_cloud_id"],
-            )
-        except models.Host.DoesNotExist:
-            raise ValidationError("host does not exist")
+        host = models.Host.objects.get(**query_host_params).first()
+        if not host:
+            raise HostNotExists("host does not exist")
 
         plugin_id = params.get("plugin_id")
         if plugin_id:

apps/backend/subscription/commons.py

@@ -38,7 +38,7 @@ def get_host_object_attribute(bk_biz_id):
 def list_biz_hosts(bk_biz_id, condition, func, split_params=False):
     biz_custom_property = []
     kwargs = {
-        "fields": constants.LIST_BIZ_HOSTS_KWARGS,
+        "fields": constants.CC_HOST_FIELDS,
     }
     if bk_biz_id:
         biz_custom_property = get_host_object_attribute(bk_biz_id)

apps/backend/subscription/render_functions.py

@@ -34,7 +34,7 @@ def get_host_detail_by_template(bk_obj_id, template_info_list: list, bk_biz_id:
     if not template_info_list:
         return []
 
-    fields = constants.FIND_HOST_BY_TEMPLATE_FIELD
+    fields = constants.CC_HOST_FIELDS
 
     if bk_obj_id == models.Subscription.NodeType.SERVICE_TEMPLATE:
         # 服务模板

apps/backend/subscription/steps/agent.py

@@ -159,6 +159,7 @@ def _generate_activities(self, agent_manager: AgentManager):
             agent_manager.choose_ap(),
             agent_manager.install(),
             agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
+            agent_manager.get_agent_id(),
             agent_manager.install_plugins() if self.is_install_latest_plugins else None,
         ]
 
@@ -180,6 +181,7 @@ def _generate_activities(self, agent_manager: AgentManager):
             agent_manager.choose_ap(),
             agent_manager.install(),
             agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
+            agent_manager.get_agent_id(),
             agent_manager.install_plugins() if self.is_install_latest_plugins else None,
         ]
 
@@ -199,6 +201,7 @@ def _generate_activities(self, agent_manager: AgentManager):
             agent_manager.push_upgrade_package(),
             agent_manager.run_upgrade_command(),
             agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
+            agent_manager.get_agent_id(),
         ]
         return activities, None
 
@@ -216,6 +219,7 @@ def _generate_activities(self, agent_manager: AgentManager):
             agent_manager.restart(skip_polling_result=True),
             agent_manager.wait(5),
             agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
+            agent_manager.get_agent_id(),
         ]
 
         return activities, None
@@ -310,6 +314,7 @@ def _generate_activities(self, agent_manager: AgentManager):
             agent_manager.run_upgrade_command(),
             agent_manager.wait(30),
             agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
+            agent_manager.get_agent_id(),
         ]
 
         # 推送文件到proxy
@@ -381,6 +386,7 @@ def _generate_activities(self, agent_manager: AgentManager):
             agent_manager.reload_agent(skip_polling_result=True),
             agent_manager.wait(5),
             agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
+            agent_manager.get_agent_id(),
         ]
         return activities, None
 

apps/backend/subscription/tools.py

@@ -374,7 +374,7 @@ def get_host_detail_by_template(bk_obj_id, template_info_list: list, bk_biz_id:
     if not template_info_list:
         return []
 
-    fields = constants.FIND_HOST_BY_TEMPLATE_FIELD
+    fields = constants.CC_HOST_FIELDS
 
     if bk_obj_id == models.Subscription.NodeType.SERVICE_TEMPLATE:
         # 服务模板