Skip to content

Commit

Permalink
feature: 提供 Agent 包管理后台接口 (closed TencentBlueKing#1683)
Browse files Browse the repository at this point in the history
  • Loading branch information
wyyalt authored and ping15 committed Nov 12, 2024
1 parent f14f01b commit 43d5fa4
Show file tree
Hide file tree
Showing 65 changed files with 3,407 additions and 205 deletions.
99 changes: 72 additions & 27 deletions apps/backend/agent/artifact_builder/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@
from apps.backend.agent.config_parser import GseConfigParser
from apps.core.files import core_files_constants
from apps.core.files.storage import get_storage
from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType
from apps.core.tag.constants import TargetType
from apps.core.tag.handlers import TagHandler
from apps.core.tag.targets import AgentTargetHelper
from apps.node_man import constants, models
from apps.utils import cache, files

Expand Down Expand Up @@ -65,6 +66,7 @@ def __init__(
overwrite_version: typing.Optional[str] = None,
tags: typing.Optional[typing.List[str]] = None,
enable_agent_pkg_manage: bool = False,
username: str = "",
):
"""
:param initial_artifact_path: 原始制品所在路径
Expand All @@ -84,6 +86,7 @@ def __init__(
self.applied_tmp_dirs = set()
# 文件源
self.storage = get_storage(file_overwrite=True)
self.username = username

@staticmethod
def download_file(file_path: str, target_path: str):
Expand Down Expand Up @@ -428,38 +431,89 @@ def _get_version(self, extract_dir: str) -> str:
raise exceptions.NotSemanticVersionError({"version": version_match})

@cache.class_member_cache()
def _get_changelog(self, extract_dir: str) -> str:
def _get_description(self, extract_dir: str) -> typing.Tuple[str, str]:
"""
获取版本日志
:param extract_dir: 解压目录
:return:
"""
changelog_file_path: str = os.path.join(extract_dir, "CHANGELOG.md")
if not os.path.exists(changelog_file_path):
raise exceptions.FileNotExistError(_("版本日志文件不存在"))
with open(changelog_file_path, "r", encoding="utf-8") as changelog_fs:
changelog: str = changelog_fs.read()
return changelog

def update_or_create_record(self, artifact_meta_info: typing.Dict[str, typing.Any]):
description_file_path: str = os.path.join(extract_dir, "DESCRIPTION")
description_en_file_path: str = os.path.join(extract_dir, "DESCRIPTION_EN")
description, description_en = "", ""
if os.path.exists(description_file_path):
with open(description_file_path, "r", encoding="utf-8") as description_fs:
description: str = description_fs.read()

if os.path.exists(description_en_file_path):
with open(description_en_file_path, "r", encoding="utf-8") as description_en_fs:
description_en: str = description_en_fs.read()
return description, description_en

def generate_location_path(self, upload_path: str, pkg_name: str) -> str:
if settings.STORAGE_TYPE == core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value:
location_path: str = f"{settings.BKREPO_ENDPOINT_URL}/generic/blueking/bknodeman/{upload_path}/{pkg_name}"
else:
location_path: str = f"http://{settings.BKAPP_LAN_IP}/{upload_path}/{pkg_name}"

return location_path

def update_or_create_package_records(self, package_infos: typing.List[typing.Dict[str, typing.Any]]):
"""
创建或更新制品记录,待 Agent 包管理完善
:param artifact_meta_info:
创建或更新制品记录
:param package_infos:
:return:
"""
pass
for package_info in package_infos:
models.GsePackages.objects.update_or_create(
defaults={
"pkg_size": package_info["package_upload_info"]["pkg_size"],
"pkg_path": package_info["package_upload_info"]["pkg_path"],
"md5": package_info["package_upload_info"]["md5"],
"location": self.generate_location_path(
package_info["package_upload_info"]["pkg_path"],
package_info["package_upload_info"]["pkg_name"],
),
"version_log": package_info["artifact_meta_info"]["description"],
"version_log_en": package_info["artifact_meta_info"]["description_en"],
},
pkg_name=package_info["package_upload_info"]["pkg_name"],
version=package_info["artifact_meta_info"]["version"],
project=package_info["artifact_meta_info"]["name"],
os=package_info["package_dir_info"]["os"],
cpu_arch=package_info["package_dir_info"]["cpu_arch"],
)
logger.info(
f"[update_or_create_package_record] "
f"package name -> {package_info['package_upload_info']['pkg_name']} success"
)

if package_infos:
models.GsePackageDesc.objects.update_or_create(
defaults={
"description": package_infos[0]["artifact_meta_info"]["description"],
},
project=package_infos[0]["artifact_meta_info"]["name"],
category=constants.CategoryType.official,
)

logger.info(
f"[update_or_create_package_record] "
f"package desc -> {package_info['package_upload_info']['pkg_name']}, "
f"project -> {package_infos[0]['artifact_meta_info']['name']} success"
)

def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]):
"""
创建或更新标签记录,待 Agent 包管理完善
:param artifact_meta_info:
:return:
"""
agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map()
for tag in self.tags:
TagHandler.publish_tag_version(
name=tag,
target_type=TargetType.AGENT.value,
target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME],
target_id=agent_name_target_id_map[self.NAME],
target_version=artifact_meta_info["version"],
)
logger.info(
Expand Down Expand Up @@ -517,14 +571,6 @@ def update_or_create_support_files(self, package_infos: typing.List[typing.Dict]
agent_name=self.NAME,
)

def update_or_create_package_records(self, v):
"""
创建或更新安装包记录,待 Agent 包管理完善
:param package_infos:
:return:
"""
pass

def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]:
"""
获取制品的基础信息、配置文件信息
Expand All @@ -535,13 +581,14 @@ def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.An
version_str: str = self._get_version(extract_dir)
# 配置文件
support_files_info = self._get_support_files_info(extract_dir)
# changelog
changelog: str = self._get_changelog(extract_dir)
# description
description, description_en = self._get_description(extract_dir)

return {
"name": self.NAME,
"version": version_str,
"changelog": changelog,
"description": description,
"description_en": description_en,
"support_files_info": support_files_info,
}

Expand Down Expand Up @@ -591,8 +638,6 @@ def make(
artifact_meta_info["operator"] = operator
# Agent 包先导入文件源 -> 写配置文件 -> 创建包记录 -> 创建 Tag
self.update_or_create_support_files(package_infos)
# TODO update_or_create_record & update_or_create_package_records 似乎是一样的功能?
self.update_or_create_record(artifact_meta_info)
self.update_or_create_package_records(package_infos)
self.update_or_create_tag(artifact_meta_info)

Expand Down
1 change: 1 addition & 0 deletions apps/backend/agent/artifact_builder/proxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class ProxyArtifactBuilder(base.BaseArtifactBuilder):
PROXY_SVR_EXES: typing.List[str] = ["gse_data", "gse_file"]

def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir: str):
# todo: 是否使用Archive(initial_artifact_local_path).extractall(extract_dir, auto_create_dir=True)
with tarfile.open(name=initial_artifact_local_path) as tf:
tf.extractall(path=extract_dir)

Expand Down
5 changes: 2 additions & 3 deletions apps/backend/agent/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ def check_run_commands(run_commands):


def batch_gen_commands(
base_agent_setup_info: AgentSetupInfo,
agent_step_adapter,
hosts: List[models.Host],
pipeline_id: str,
is_uninstall: bool,
Expand All @@ -350,7 +350,6 @@ def batch_gen_commands(
# 批量查出主机的属性并设置为property,避免在循环中进行ORM查询,提高效率
host_id__installation_tool_map = {}
bk_host_ids = [host.bk_host_id for host in hosts]
base_agent_setup_info_dict: Dict[str, Any] = asdict(base_agent_setup_info)
host_id_identity_map = {
identity.bk_host_id: identity for identity in models.IdentityData.objects.filter(bk_host_id__in=bk_host_ids)
}
Expand All @@ -368,7 +367,7 @@ def batch_gen_commands(
host_id__installation_tool_map[host.bk_host_id] = gen_commands(
agent_setup_info=AgentSetupInfo(
**{
**base_agent_setup_info_dict,
**asdict(agent_step_adapter.get_host_setup_info(host)),
"force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False),
}
),
Expand Down
89 changes: 89 additions & 0 deletions apps/backend/agent/views.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import typing

from django.utils.translation import get_language
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.status import HTTP_200_OK

from apps.generic import ApiMixinModelViewSet as ModelViewSet
from apps.generic import ValidationMixin
from apps.node_man.serializers import package_manage as pkg_manage
from apps.node_man.tools.gse_package import GsePackageTools
from apps.node_man.views.package_manage import PACKAGE_MANAGE_VIEW_TAGS
from common.utils.drf_utils import swagger_auto_schema


class AgentViewSet(ModelViewSet, ValidationMixin):
"""
agent相关API
"""

@swagger_auto_schema(
operation_summary="解析Agent包",
tags=PACKAGE_MANAGE_VIEW_TAGS,
responses={HTTP_200_OK: pkg_manage.ParseResponseSerializer},
)
@action(detail=False, methods=["POST"], serializer_class=pkg_manage.ParseSerializer)
def parse(self, request):
"""
return: {
"description": "test",
"packages": [
{
"pkg_abs_path": "xxx/xxxxx",
"pkg_name": "gseagent_2.1.7_linux_x86_64.tgz",
"module": "agent",
"version": "2.1.7",
"config_templates": [],
"os": "x86_64",
},
{
"pkg_abs_path": "xxx/xxxxx",
"pkg_name": "gseagent_2.1.7_linux_x86.tgz",
"module": "agent",
"version": "2.1.7",
"config_templates": [],
"os": "x86",
},
],
}
"""
validated_data = self.validated_data

# 获取最新的agent上传包记录
upload_package_obj = GsePackageTools.get_latest_upload_record(file_name=validated_data["file_name"])

# 区分agent和proxy包
project, artifact_builder_class = GsePackageTools.distinguish_gse_package(
file_path=upload_package_obj.file_path
)

# 解析包
with artifact_builder_class(initial_artifact_path=upload_package_obj.file_path) as builder:
extract_dir, package_dir_infos = builder.list_package_dir_infos()
artifact_meta_info: typing.Dict[str, typing.Any] = builder.get_artifact_meta_info(extract_dir)

language = get_language()
res = {
"description": artifact_meta_info["description"]
if language == "zh-hans"
else artifact_meta_info["description_en"],
"packages": package_dir_infos,
}

context = {
"project": project,
"version": artifact_meta_info["version"],
}

return Response(pkg_manage.ParseResponseSerializer(res, context=context).data)
4 changes: 2 additions & 2 deletions apps/backend/components/collections/agent_new/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def get_agent_pkg_name(
package_type = ("client", "proxy")[host.node_type == constants.NodeType.PROXY]
agent_step_adapter = common_data.agent_step_adapter
if not agent_step_adapter.is_legacy:
setup_info = agent_step_adapter.setup_info
setup_info = agent_step_adapter.get_host_setup_info(host)
return f"{setup_info.name}-{setup_info.version}.tgz"

# GSE1.0 的升级包是独立的,添加了 _upgrade 后缀
Expand Down Expand Up @@ -297,7 +297,7 @@ def get_host_id__installation_tool_map(
host for host in hosts_need_gen_commands if host.bk_host_id in host_id__install_channel_map
]
host_id__installation_tool_map = batch_gen_commands(
base_agent_setup_info=common_data.agent_step_adapter.setup_info,
agent_step_adapter=common_data.agent_step_adapter,
hosts=hosts_need_gen_commands,
pipeline_id=self.id,
is_uninstall=is_uninstall,
Expand Down
1 change: 1 addition & 0 deletions apps/backend/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ class InstNodeType(object):
"INSTALL_AGENT",
"REINSTALL_AGENT",
"UPGRADE_AGENT",
"DOWNGRADE_AGENT",
"RESTART_AGENT",
"UNINSTALL_AGENT",
"RELOAD_AGENT",
Expand Down
6 changes: 6 additions & 0 deletions apps/backend/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,9 @@ class AgentConfigTemplateEnvNotExistError(BackendBaseException):
MESSAGE = _("配置模板Env不存在")
MESSAGE_TPL = _("配置模板Env不存在[{name}-{version}-{os_type}-{cpu_arch}]不存在")
ERROR_CODE = 15


class ModelInstanceNotFoundError(BackendBaseException):
MESSAGE = _("模型对象不存在")
MESSAGE_TPL = _("模型对象 -> [{model_name}] 不存在")
ERROR_CODE = 16
8 changes: 8 additions & 0 deletions apps/backend/subscription/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,3 +169,11 @@ class SubscriptionIncludeGrayBizError(AppBaseException):
ERROR_CODE = 19
MESSAGE = _("订阅任务包含Gse2.0灰度业务,任务将暂缓执行无需重复点击")
MESSAGE_TPL = _("订阅任务包含Gse2.0灰度业务,任务将暂缓执行无需重复点击")


class AgentPackageValidationError(AppBaseException):
"""AgentPackage校验错误"""

ERROR_CODE = 20
MESSAGE = _("AgentPackage校验错误")
MESSAGE_TPL = _("{msg}")
27 changes: 27 additions & 0 deletions apps/backend/subscription/steps/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from apps.node_man import constants, models
from apps.node_man.constants import DEFAULT_CLOUD
from apps.node_man.models import GsePluginDesc, SubscriptionStep
from apps.node_man.tools.gse_package import GsePackageTools
from env.constants import GseVersion
from pipeline import builder
from pipeline.builder import Var
Expand Down Expand Up @@ -49,6 +50,7 @@ def get_supported_actions(self):
ReinstallAgent,
UninstallAgent,
UpgradeAgent,
DowngradeAgent,
RestartAgent,
InstallProxy,
ReinstallProxy,
Expand Down Expand Up @@ -115,6 +117,24 @@ def make_instances_migrate_actions(self, instances, auto_trigger=False, preview_
continue
instance_actions[instance_id] = job_type_map[self.subscription_step.config["job_type"]]

if instance_actions[instance_id] == backend_const.ActionNameType.UPGRADE_AGENT:
version_map: Dict[int, str] = {
version_map["bk_host_id"]: version_map["version"]
for version_map in self.subscription_step.config.get("version_map_list", [])
}

bk_host_id: int = instance["host"]["bk_host_id"]
agent_version: str = models.ProcessStatus.objects.get(
bk_host_id=bk_host_id, name=models.ProcessStatus.GSE_AGENT_PROCESS_NAME
).version
if all(
[
version_map.get(bk_host_id),
not GsePackageTools.compare_version(version_map[bk_host_id], agent_version),
]
):
instance_actions[instance_id] = backend_const.ActionNameType.DOWNGRADE_AGENT

return {"instance_actions": instance_actions, "migrate_reasons": migrate_reasons}


Expand Down Expand Up @@ -325,6 +345,13 @@ def _generate_activities(self, agent_manager: AgentManager):
return activities, None


class DowngradeAgent(UpgradeAgent):
"""回退Agent"""

ACTION_NAME = backend_const.ActionNameType.DOWNGRADE_AGENT
ACTION_DESCRIPTION = _("回退")


class RestartAgent(AgentAction):
"""
重启Agent
Expand Down
Loading

0 comments on commit 43d5fa4

Please sign in to comment.