Skip to content

Commit

Permalink
feature: 支持bk_agent_id (close #562)
Browse files Browse the repository at this point in the history
  • Loading branch information
zhangzhw8 authored and ZhuoZhuoCrayon committed Jul 18, 2022
1 parent 8a1fb15 commit 4430fe5
Show file tree
Hide file tree
Showing 64 changed files with 1,216 additions and 340 deletions.
8 changes: 8 additions & 0 deletions apps/backend/agent/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,14 @@ def get_agent_status(cls, expect_status: str, name=components.GetAgentStatusComp
act.component.inputs.expect_status = Var(type=Var.PLAIN, value=expect_status)
return act

@classmethod
def get_agent_id(cls, name=components.GetAgentIDComponent.name):
"""查询 Agent ID"""
if settings.GSE_VERSION == "V1":
return None
act = AgentServiceActivity(component_code=components.GetAgentIDComponent.code, name=name)
return act

@classmethod
def check_agent_status(cls, name=components.CheckAgentStatusComponent.name):
"""查询Agent状态是否正常"""
Expand Down
11 changes: 9 additions & 2 deletions apps/backend/agent/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,9 @@ def gen_commands(
agent_config = host_ap.get_agent_config(host.os_type)
# 安装操作
install_path = agent_config["setup_path"]
token = aes_cipher.encrypt(f"{host.inner_ip}|{host.bk_cloud_id}|{pipeline_id}|{time.time()}|{sub_inst_id}")
token = aes_cipher.encrypt(
f"{host.bk_host_id}|{host.inner_ip}|{host.bk_cloud_id}|{pipeline_id}|{time.time()}|{sub_inst_id}"
)
port_config = host_ap.port_config
run_cmd_params = [
f"-s {pipeline_id}",
Expand Down Expand Up @@ -309,8 +311,9 @@ def gen_commands(
host_shell = format_run_cmd_by_os_type(host.os_type, host_tmp_path)
run_cmd_params.extend(
[
f"-HLIP {host.login_ip or host.inner_ip}",
f"-HLIP {host.login_ip or host.inner_ip or host.inner_ipv6}",
f"-HIIP {host.inner_ip}",
f"-HIIP6 {host.inner_ipv6}" if host.inner_ipv6 else "",
f"-HA {identity_data.account}",
f"-HP {identity_data.port}",
f"-HI '{host_identity}'",
Expand All @@ -323,6 +326,8 @@ def gen_commands(
f"-HS '{host_shell}'",
f"-p '{install_path}'",
f"-I {jump_server.inner_ip}",
f"-I6 {jump_server.inner_ipv6}" if jump_server.inner_ipv6 else "",
f"-AI {host.bk_agent_id}" if host.bk_agent_id else "",
f"-o {gen_nginx_download_url(jump_server.inner_ip)}",
f"-HEP '{encrypted_password}'" if need_encrypted_password else "",
"-R" if is_uninstall else "",
Expand Down Expand Up @@ -354,6 +359,8 @@ def gen_commands(
[
f"-i {host.bk_cloud_id}",
f"-I {host.inner_ip}",
f"-I6 {host.inner_ipv6}" if host.inner_ipv6 else "",
f"-AI {host.bk_agent_id}" if host.bk_agent_id else "",
"-N SERVER",
f"-p {install_path}",
f"-T {dest_dir}",
Expand Down
7 changes: 7 additions & 0 deletions apps/backend/components/collections/agent_new/components.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from .choose_access_point import ChooseAccessPointService
from .configure_policy import ConfigurePolicyService
from .delegate_plugin_proc import DelegatePluginProcService
from .get_agent_id import GetAgentIDService
from .get_agent_status import GetAgentStatusService
from .install import InstallService
from .install_plugins import InstallPluginsService
Expand Down Expand Up @@ -83,6 +84,12 @@ class GetAgentStatusComponent(Component):
bound_service = GetAgentStatusService


class GetAgentIDComponent(Component):
name = _("查询AgentID")
code = "get_agent_id"
bound_service = GetAgentIDService


class ReloadAgentConfigComponent(Component):
name = _("重载Agent配置")
code = "reload_agent_config"
Expand Down
82 changes: 82 additions & 0 deletions apps/backend/components/collections/agent_new/get_agent_id.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from typing import Any, Dict, List

from django.utils.translation import ugettext_lazy as _

from apps.backend.api.constants import POLLING_INTERVAL, POLLING_TIMEOUT
from apps.component.esbclient import client_v2
from apps.node_man import models
from apps.utils import batch_request
from pipeline.core.flow import Service, StaticIntervalGenerator

from .base import AgentBaseService, AgentCommonData


class GetAgentIDService(AgentBaseService):
"""安装后AgentID同步到CMDB后才认为是可用的"""

__need_schedule__ = True
interval = StaticIntervalGenerator(POLLING_INTERVAL)

def outputs_format(self):
return super().outputs_format() + [
Service.InputItem(name="polling_time", key="polling_time", type="int", required=True),
]

@staticmethod
def update_host_agent_id(cmdb_host_infos: List[Dict[str, Any]]):
need_update_hosts = [
models.Host(bk_host_id=host_info["bk_host_id"], bk_agent_id=host_info.get("bk_agent_id"))
for host_info in cmdb_host_infos
]
models.Host.objects.bulk_update(need_update_hosts, fields=["bk_agent_id"])

def _execute(self, data, parent_data, common_data: AgentCommonData):
self.log_info(sub_inst_ids=common_data.subscription_instance_ids, log_content=_("开始查询从 CMDB 查询主机的 Agent ID"))
data.outputs.polling_time = 0

def _schedule(self, data, parent_data, callback_data=None):
common_data: AgentCommonData = self.get_common_data(data)
list_cmdb_hosts_params = {
"fields": ["bk_host_id", "bk_agent_id"],
"host_property_filter": {
"condition": "AND",
"rules": [
{"field": "bk_host_id", "operator": "in", "value": common_data.bk_host_ids},
],
},
}
cmdb_host_infos: List[Dict[str, Any]] = batch_request.batch_request(
func=client_v2.cc.list_hosts_without_biz, params=list_cmdb_hosts_params
)

# CMDB 中 AgentID 为空的主机ID列表
no_agent_id_host_ids = [
host_info["bk_host_id"] for host_info in cmdb_host_infos if not host_info.get("bk_agent_id")
]
if not no_agent_id_host_ids:
self.update_host_agent_id(cmdb_host_infos)
self.finish_schedule()
return

polling_time = data.get_one_of_outputs("polling_time")
if polling_time + POLLING_INTERVAL > POLLING_TIMEOUT:
sub_inst_ids = [common_data.host_id__sub_inst_id_map[host_id] for host_id in no_agent_id_host_ids]
self.move_insts_to_failed(
sub_inst_ids=sub_inst_ids,
log_content=_("此主机在 CMDB 中未查到对应 Agent ID,请联系 GSE 及 CMDB 团队排查 Agent ID 上报处理是否异常!"),
)
self.update_host_agent_id(cmdb_host_infos)
self.finish_schedule()
return

data.outputs.polling_time = polling_time + POLLING_INTERVAL
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,13 @@ def add_hosts_to_biz_thread(
host_key = f"{host_info['bk_cloud_id']}-{host_info['bk_host_innerip']}"
register_params = {
index: {
"bk_host_innerip": host_info["bk_host_innerip"],
"bk_host_innerip": host_info.get("bk_host_innerip", ""),
"bk_host_innerip_v6": host_info.get("bk_host_innerip_v6", ""),
# "3" 表示API导入
"import_from": "3",
"bk_cloud_id": host_info["bk_cloud_id"],
"bk_host_outerip": host_info.get("bk_host_outerip", ""),
"bk_host_outerip_v6": host_info.get("bk_host_outerip_v6", ""),
"bk_os_type": constants.BK_OS_TYPE[host_info["os_type"]],
"bk_bak_operator": biz_info.get("bk_biz_maintainer") or host_info.get("username"),
"operator": biz_info.get("bk_biz_maintainer") or host_info.get("username"),
Expand Down
4 changes: 2 additions & 2 deletions apps/backend/components/collections/common/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ def conns_init_params(self) -> typing.Dict:
:return:
"""
if self.host.node_type == constants.NodeType.PROXY:
ip = self.host.login_ip or self.host.outer_ip
ip = self.host.login_ip or self.host.outer_ip or self.host.outer_ipv6
else:
ip = self.host.login_ip or self.host.inner_ip
ip = self.host.login_ip or self.host.inner_ip or self.host.inner_ipv6

client_key_strings = []
if self.identity_data.auth_type == constants.AuthType.KEY:
Expand Down
8 changes: 7 additions & 1 deletion apps/backend/components/collections/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -1144,7 +1144,10 @@ def _execute(self, data, parent_data, common_data: PluginCommonData):
gse_op_params = {
"meta": {"namespace": constants.GSE_NAMESPACE, "name": meta_name},
"op_type": op_type,
# GSE 1.0 目标对象
"hosts": [{"ip": host.inner_ip, "bk_cloud_id": host.bk_cloud_id}],
# GSE 2.0 目标对象
"agent_id_list": [host.bk_agent_id],
# 此字段是节点管理自用,仅用于标识,不会被GSE使用
"nodeman_spec": {
"process_status_id": process_status.id,
Expand Down Expand Up @@ -1240,7 +1243,10 @@ def _schedule(self, data, parent_data, callback_data=None):
subscription_instance = group_id_instance_map.get(process_status.group_id)

proc_name = self.get_plugin_meta_name(plugin, process_status)
gse_proc_key = f"{host.bk_cloud_id}:{host.inner_ip}:{constants.GSE_NAMESPACE}:{proc_name}"
if host.bk_agent_id:
gse_proc_key = f"{host.bk_agent_id}:{constants.GSE_NAMESPACE}:{proc_name}"
else:
gse_proc_key = f"{host.bk_cloud_id}:{host.inner_ip}:{constants.GSE_NAMESPACE}:{proc_name}"
proc_operate_result = result["data"].get(gse_proc_key)
if not proc_operate_result:
self.move_insts_to_failed(
Expand Down
1 change: 1 addition & 0 deletions apps/backend/plugin/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@ class PluginStartDebugSerializer(GatewaySerializer):
"""

class HostInfoSerializer(serializers.Serializer):
bk_host_id = serializers.IntegerField(required=False)
ip = serializers.CharField(required=True)
bk_cloud_id = serializers.IntegerField(required=True)
bk_supplier_id = serializers.IntegerField(default=constants.DEFAULT_SUPPLIER_ID)
Expand Down
17 changes: 9 additions & 8 deletions apps/backend/plugin/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
from apps.exceptions import AppBaseException, ValidationError
from apps.generic import APIViewSet
from apps.node_man import constants, models
from apps.node_man.exceptions import HostNotExists
from pipeline.engine.exceptions import InvalidOperationException
from pipeline.service import task_service
from pipeline.service.pipeline_engine_adapter.adapter_api import STATE_MAP
Expand Down Expand Up @@ -499,15 +500,15 @@ def start_debug(self, request):
host_info = params["host_info"]
plugin_name = params["plugin_name"]
plugin_version = params["version"]
query_host_params = dict(
bk_biz_id=host_info["bk_biz_id"], inner_ip=host_info["ip"], bk_cloud_id=host_info["bk_cloud_id"]
)
if params.get("bk_host_id"):
query_host_params.update(bk_host_id=params["bk_host_id"])

try:
host = models.Host.objects.get(
bk_biz_id=host_info["bk_biz_id"],
inner_ip=host_info["ip"],
bk_cloud_id=host_info["bk_cloud_id"],
)
except models.Host.DoesNotExist:
raise ValidationError("host does not exist")
host = models.Host.objects.get(**query_host_params).first()
if not host:
raise HostNotExists("host does not exist")

plugin_id = params.get("plugin_id")
if plugin_id:
Expand Down
2 changes: 1 addition & 1 deletion apps/backend/subscription/commons.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def get_host_object_attribute(bk_biz_id):
def list_biz_hosts(bk_biz_id, condition, func, split_params=False):
biz_custom_property = []
kwargs = {
"fields": constants.LIST_BIZ_HOSTS_KWARGS,
"fields": constants.CC_HOST_FIELDS,
}
if bk_biz_id:
biz_custom_property = get_host_object_attribute(bk_biz_id)
Expand Down
2 changes: 1 addition & 1 deletion apps/backend/subscription/render_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def get_host_detail_by_template(bk_obj_id, template_info_list: list, bk_biz_id:
if not template_info_list:
return []

fields = constants.FIND_HOST_BY_TEMPLATE_FIELD
fields = constants.CC_HOST_FIELDS

if bk_obj_id == models.Subscription.NodeType.SERVICE_TEMPLATE:
# 服务模板
Expand Down
6 changes: 6 additions & 0 deletions apps/backend/subscription/steps/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ def _generate_activities(self, agent_manager: AgentManager):
agent_manager.choose_ap(),
agent_manager.install(),
agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
agent_manager.get_agent_id(),
agent_manager.install_plugins() if self.is_install_latest_plugins else None,
]

Expand All @@ -180,6 +181,7 @@ def _generate_activities(self, agent_manager: AgentManager):
agent_manager.choose_ap(),
agent_manager.install(),
agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
agent_manager.get_agent_id(),
agent_manager.install_plugins() if self.is_install_latest_plugins else None,
]

Expand All @@ -199,6 +201,7 @@ def _generate_activities(self, agent_manager: AgentManager):
agent_manager.push_upgrade_package(),
agent_manager.run_upgrade_command(),
agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
agent_manager.get_agent_id(),
]
return activities, None

Expand All @@ -216,6 +219,7 @@ def _generate_activities(self, agent_manager: AgentManager):
agent_manager.restart(skip_polling_result=True),
agent_manager.wait(5),
agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
agent_manager.get_agent_id(),
]

return activities, None
Expand Down Expand Up @@ -310,6 +314,7 @@ def _generate_activities(self, agent_manager: AgentManager):
agent_manager.run_upgrade_command(),
agent_manager.wait(30),
agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
agent_manager.get_agent_id(),
]

# 推送文件到proxy
Expand Down Expand Up @@ -381,6 +386,7 @@ def _generate_activities(self, agent_manager: AgentManager):
agent_manager.reload_agent(skip_polling_result=True),
agent_manager.wait(5),
agent_manager.get_agent_status(expect_status=constants.ProcStateType.RUNNING),
agent_manager.get_agent_id(),
]
return activities, None

Expand Down
2 changes: 1 addition & 1 deletion apps/backend/subscription/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ def get_host_detail_by_template(bk_obj_id, template_info_list: list, bk_biz_id:
if not template_info_list:
return []

fields = constants.FIND_HOST_BY_TEMPLATE_FIELD
fields = constants.CC_HOST_FIELDS

if bk_obj_id == models.Subscription.NodeType.SERVICE_TEMPLATE:
# 服务模板
Expand Down
Loading

0 comments on commit 4430fe5

Please sign in to comment.