feature: 提供 Agent 包管理后台接口 (closed TencentBlueKing#1683)

apps/backend/agent/artifact_builder/base.py

@@ -23,8 +23,9 @@
 from apps.backend.agent.config_parser import GseConfigParser
 from apps.core.files import core_files_constants
 from apps.core.files.storage import get_storage
-from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType
+from apps.core.tag.constants import TargetType
 from apps.core.tag.handlers import TagHandler
+from apps.core.tag.targets import AgentTargetHelper
 from apps.node_man import constants, models
 from apps.utils import cache, files
 
@@ -65,6 +66,7 @@ def __init__(
         overwrite_version: typing.Optional[str] = None,
         tags: typing.Optional[typing.List[str]] = None,
         enable_agent_pkg_manage: bool = False,
+        username: str = "",
     ):
         """
         :param initial_artifact_path: 原始制品所在路径
@@ -84,6 +86,7 @@ def __init__(
         self.applied_tmp_dirs = set()
         # 文件源
         self.storage = get_storage(file_overwrite=True)
+        self.username = username
 
     @staticmethod
     def download_file(file_path: str, target_path: str):
@@ -428,38 +431,89 @@ def _get_version(self, extract_dir: str) -> str:
             raise exceptions.NotSemanticVersionError({"version": version_match})
 
     @cache.class_member_cache()
-    def _get_changelog(self, extract_dir: str) -> str:
+    def _get_description(self, extract_dir: str) -> typing.Tuple[str, str]:
         """
         获取版本日志
         :param extract_dir: 解压目录
         :return:
         """
-        changelog_file_path: str = os.path.join(extract_dir, "CHANGELOG.md")
-        if not os.path.exists(changelog_file_path):
-            raise exceptions.FileNotExistError(_("版本日志文件不存在"))
-        with open(changelog_file_path, "r", encoding="utf-8") as changelog_fs:
-            changelog: str = changelog_fs.read()
-        return changelog
-
-    def update_or_create_record(self, artifact_meta_info: typing.Dict[str, typing.Any]):
+        description_file_path: str = os.path.join(extract_dir, "DESCRIPTION")
+        description_en_file_path: str = os.path.join(extract_dir, "DESCRIPTION_EN")
+        description, description_en = "", ""
+        if os.path.exists(description_file_path):
+            with open(description_file_path, "r", encoding="utf-8") as description_fs:
+                description: str = description_fs.read()
+
+        if os.path.exists(description_en_file_path):
+            with open(description_en_file_path, "r", encoding="utf-8") as description_en_fs:
+                description_en: str = description_en_fs.read()
+        return description, description_en
+
+    def generate_location_path(self, upload_path: str, pkg_name: str) -> str:
+        if settings.STORAGE_TYPE == core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value:
+            location_path: str = f"{settings.BKREPO_ENDPOINT_URL}/generic/blueking/bknodeman/{upload_path}/{pkg_name}"
+        else:
+            location_path: str = f"http://{settings.BKAPP_LAN_IP}/{upload_path}/{pkg_name}"
+
+        return location_path
+
+    def update_or_create_package_records(self, package_infos: typing.List[typing.Dict[str, typing.Any]]):
         """
-        创建或更新制品记录，待 Agent 包管理完善
-        :param artifact_meta_info:
+        创建或更新制品记录
+        :param package_infos:
         :return:
         """
-        pass
+        for package_info in package_infos:
+            models.GsePackages.objects.update_or_create(
+                defaults={
+                    "pkg_size": package_info["package_upload_info"]["pkg_size"],
+                    "pkg_path": package_info["package_upload_info"]["pkg_path"],
+                    "md5": package_info["package_upload_info"]["md5"],
+                    "location": self.generate_location_path(
+                        package_info["package_upload_info"]["pkg_path"],
+                        package_info["package_upload_info"]["pkg_name"],
+                    ),
+                    "version_log": package_info["artifact_meta_info"]["description"],
+                    "version_log_en": package_info["artifact_meta_info"]["description_en"],
+                },
+                pkg_name=package_info["package_upload_info"]["pkg_name"],
+                version=package_info["artifact_meta_info"]["version"],
+                project=package_info["artifact_meta_info"]["name"],
+                os=package_info["package_dir_info"]["os"],
+                cpu_arch=package_info["package_dir_info"]["cpu_arch"],
+            )
+            logger.info(
+                f"[update_or_create_package_record] "
+                f"package name -> {package_info['package_upload_info']['pkg_name']} success"
+            )
+
+        if package_infos:
+            models.GsePackageDesc.objects.update_or_create(
+                defaults={
+                    "description": package_infos[0]["artifact_meta_info"]["description"],
+                },
+                project=package_infos[0]["artifact_meta_info"]["name"],
+                category=constants.CategoryType.official,
+            )
+
+            logger.info(
+                f"[update_or_create_package_record] "
+                f"package desc -> {package_info['package_upload_info']['pkg_name']}, "
+                f"project -> {package_infos[0]['artifact_meta_info']['name']} success"
+            )
 
     def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]):
         """
         创建或更新标签记录，待 Agent 包管理完善
         :param artifact_meta_info:
         :return:
         """
+        agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map()
         for tag in self.tags:
             TagHandler.publish_tag_version(
                 name=tag,
                 target_type=TargetType.AGENT.value,
-                target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME],
+                target_id=agent_name_target_id_map[self.NAME],
                 target_version=artifact_meta_info["version"],
             )
             logger.info(
@@ -517,14 +571,6 @@ def update_or_create_support_files(self, package_infos: typing.List[typing.Dict]
                     agent_name=self.NAME,
                 )
 
-    def update_or_create_package_records(self, v):
-        """
-        创建或更新安装包记录，待 Agent 包管理完善
-        :param package_infos:
-        :return:
-        """
-        pass
-
     def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]:
         """
         获取制品的基础信息、配置文件信息
@@ -535,13 +581,14 @@ def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.An
         version_str: str = self._get_version(extract_dir)
         # 配置文件
         support_files_info = self._get_support_files_info(extract_dir)
-        # changelog
-        changelog: str = self._get_changelog(extract_dir)
+        # description
+        description, description_en = self._get_description(extract_dir)
 
         return {
             "name": self.NAME,
             "version": version_str,
-            "changelog": changelog,
+            "description": description,
+            "description_en": description_en,
             "support_files_info": support_files_info,
         }
 
@@ -591,8 +638,6 @@ def make(
         artifact_meta_info["operator"] = operator
         # Agent 包先导入文件源 -> 写配置文件 -> 创建包记录 -> 创建 Tag
         self.update_or_create_support_files(package_infos)
-        # TODO update_or_create_record & update_or_create_package_records 似乎是一样的功能？
-        self.update_or_create_record(artifact_meta_info)
         self.update_or_create_package_records(package_infos)
         self.update_or_create_tag(artifact_meta_info)
 

apps/backend/agent/artifact_builder/proxy.py

@@ -36,6 +36,7 @@ class ProxyArtifactBuilder(base.BaseArtifactBuilder):
     PROXY_SVR_EXES: typing.List[str] = ["gse_data", "gse_file"]
 
     def extract_initial_artifact(self, initial_artifact_local_path: str, extract_dir: str):
+        # todo: 是否使用Archive(initial_artifact_local_path).extractall(extract_dir, auto_create_dir=True)
         with tarfile.open(name=initial_artifact_local_path) as tf:
             tf.extractall(path=extract_dir)
 

apps/backend/agent/tools.py

@@ -334,7 +334,7 @@ def check_run_commands(run_commands):
 
 
 def batch_gen_commands(
-    base_agent_setup_info: AgentSetupInfo,
+    agent_step_adapter,
     hosts: List[models.Host],
     pipeline_id: str,
     is_uninstall: bool,
@@ -350,7 +350,6 @@ def batch_gen_commands(
     # 批量查出主机的属性并设置为property，避免在循环中进行ORM查询，提高效率
     host_id__installation_tool_map = {}
     bk_host_ids = [host.bk_host_id for host in hosts]
-    base_agent_setup_info_dict: Dict[str, Any] = asdict(base_agent_setup_info)
     host_id_identity_map = {
         identity.bk_host_id: identity for identity in models.IdentityData.objects.filter(bk_host_id__in=bk_host_ids)
     }
@@ -368,7 +367,7 @@ def batch_gen_commands(
         host_id__installation_tool_map[host.bk_host_id] = gen_commands(
             agent_setup_info=AgentSetupInfo(
                 **{
-                    **base_agent_setup_info_dict,
+                    **asdict(agent_step_adapter.get_host_setup_info(host)),
                     "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False),
                 }
             ),

apps/backend/agent/views.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import typing
+
+from django.utils.translation import get_language
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from rest_framework.status import HTTP_200_OK
+
+from apps.generic import ApiMixinModelViewSet as ModelViewSet
+from apps.generic import ValidationMixin
+from apps.node_man.serializers import package_manage as pkg_manage
+from apps.node_man.tools.gse_package import GsePackageTools
+from apps.node_man.views.package_manage import PACKAGE_MANAGE_VIEW_TAGS
+from common.utils.drf_utils import swagger_auto_schema
+
+
+class AgentViewSet(ModelViewSet, ValidationMixin):
+    """
+    agent相关API
+    """
+
+    @swagger_auto_schema(
+        operation_summary="解析Agent包",
+        tags=PACKAGE_MANAGE_VIEW_TAGS,
+        responses={HTTP_200_OK: pkg_manage.ParseResponseSerializer},
+    )
+    @action(detail=False, methods=["POST"], serializer_class=pkg_manage.ParseSerializer)
+    def parse(self, request):
+        """
+        return: {
+            "description": "test",
+            "packages": [
+                {
+                    "pkg_abs_path": "xxx/xxxxx",
+                    "pkg_name": "gseagent_2.1.7_linux_x86_64.tgz",
+                    "module": "agent",
+                    "version": "2.1.7",
+                    "config_templates": [],
+                    "os": "x86_64",
+                },
+                {
+                    "pkg_abs_path": "xxx/xxxxx",
+                    "pkg_name": "gseagent_2.1.7_linux_x86.tgz",
+                    "module": "agent",
+                    "version": "2.1.7",
+                    "config_templates": [],
+                    "os": "x86",
+                },
+            ],
+        }
+        """
+        validated_data = self.validated_data
+
+        # 获取最新的agent上传包记录
+        upload_package_obj = GsePackageTools.get_latest_upload_record(file_name=validated_data["file_name"])
+
+        # 区分agent和proxy包
+        project, artifact_builder_class = GsePackageTools.distinguish_gse_package(
+            file_path=upload_package_obj.file_path
+        )
+
+        # 解析包
+        with artifact_builder_class(initial_artifact_path=upload_package_obj.file_path) as builder:
+            extract_dir, package_dir_infos = builder.list_package_dir_infos()
+            artifact_meta_info: typing.Dict[str, typing.Any] = builder.get_artifact_meta_info(extract_dir)
+
+        language = get_language()
+        res = {
+            "description": artifact_meta_info["description"]
+            if language == "zh-hans"
+            else artifact_meta_info["description_en"],
+            "packages": package_dir_infos,
+        }
+
+        context = {
+            "project": project,
+            "version": artifact_meta_info["version"],
+        }
+
+        return Response(pkg_manage.ParseResponseSerializer(res, context=context).data)

apps/backend/components/collections/agent_new/base.py

@@ -141,7 +141,7 @@ def get_agent_pkg_name(
         package_type = ("client", "proxy")[host.node_type == constants.NodeType.PROXY]
         agent_step_adapter = common_data.agent_step_adapter
         if not agent_step_adapter.is_legacy:
-            setup_info = agent_step_adapter.setup_info
+            setup_info = agent_step_adapter.get_host_setup_info(host)
             return f"{setup_info.name}-{setup_info.version}.tgz"
 
         # GSE1.0 的升级包是独立的，添加了 _upgrade 后缀
@@ -297,7 +297,7 @@ def get_host_id__installation_tool_map(
             host for host in hosts_need_gen_commands if host.bk_host_id in host_id__install_channel_map
         ]
         host_id__installation_tool_map = batch_gen_commands(
-            base_agent_setup_info=common_data.agent_step_adapter.setup_info,
+            agent_step_adapter=common_data.agent_step_adapter,
             hosts=hosts_need_gen_commands,
             pipeline_id=self.id,
             is_uninstall=is_uninstall,

apps/backend/constants.py

@@ -64,6 +64,7 @@ class InstNodeType(object):
     "INSTALL_AGENT",
     "REINSTALL_AGENT",
     "UPGRADE_AGENT",
+    "DOWNGRADE_AGENT",
     "RESTART_AGENT",
     "UNINSTALL_AGENT",
     "RELOAD_AGENT",

apps/backend/exceptions.py

@@ -94,3 +94,9 @@ class AgentConfigTemplateEnvNotExistError(BackendBaseException):
     MESSAGE = _("配置模板Env不存在")
     MESSAGE_TPL = _("配置模板Env不存在[{name}-{version}-{os_type}-{cpu_arch}]不存在")
     ERROR_CODE = 15
+
+
+class ModelInstanceNotFoundError(BackendBaseException):
+    MESSAGE = _("模型对象不存在")
+    MESSAGE_TPL = _("模型对象 -> [{model_name}] 不存在")
+    ERROR_CODE = 16

apps/backend/subscription/errors.py

@@ -169,3 +169,11 @@ class SubscriptionIncludeGrayBizError(AppBaseException):
     ERROR_CODE = 19
     MESSAGE = _("订阅任务包含Gse2.0灰度业务，任务将暂缓执行无需重复点击")
     MESSAGE_TPL = _("订阅任务包含Gse2.0灰度业务，任务将暂缓执行无需重复点击")
+
+
+class AgentPackageValidationError(AppBaseException):
+    """AgentPackage校验错误"""
+
+    ERROR_CODE = 20
+    MESSAGE = _("AgentPackage校验错误")
+    MESSAGE_TPL = _("{msg}")

apps/backend/subscription/steps/agent.py

@@ -20,6 +20,7 @@
 from apps.node_man import constants, models
 from apps.node_man.constants import DEFAULT_CLOUD
 from apps.node_man.models import GsePluginDesc, SubscriptionStep
+from apps.node_man.tools.gse_package import GsePackageTools
 from env.constants import GseVersion
 from pipeline import builder
 from pipeline.builder import Var
@@ -49,6 +50,7 @@ def get_supported_actions(self):
             ReinstallAgent,
             UninstallAgent,
             UpgradeAgent,
+            DowngradeAgent,
             RestartAgent,
             InstallProxy,
             ReinstallProxy,
@@ -115,6 +117,24 @@ def make_instances_migrate_actions(self, instances, auto_trigger=False, preview_
                 continue
             instance_actions[instance_id] = job_type_map[self.subscription_step.config["job_type"]]
 
+            if instance_actions[instance_id] == backend_const.ActionNameType.UPGRADE_AGENT:
+                version_map: Dict[int, str] = {
+                    version_map["bk_host_id"]: version_map["version"]
+                    for version_map in self.subscription_step.config.get("version_map_list", [])
+                }
+
+                bk_host_id: int = instance["host"]["bk_host_id"]
+                agent_version: str = models.ProcessStatus.objects.get(
+                    bk_host_id=bk_host_id, name=models.ProcessStatus.GSE_AGENT_PROCESS_NAME
+                ).version
+                if all(
+                    [
+                        version_map.get(bk_host_id),
+                        not GsePackageTools.compare_version(version_map[bk_host_id], agent_version),
+                    ]
+                ):
+                    instance_actions[instance_id] = backend_const.ActionNameType.DOWNGRADE_AGENT
+
         return {"instance_actions": instance_actions, "migrate_reasons": migrate_reasons}
 
 
@@ -325,6 +345,13 @@ def _generate_activities(self, agent_manager: AgentManager):
         return activities, None
 
 
+class DowngradeAgent(UpgradeAgent):
+    """回退Agent"""
+
+    ACTION_NAME = backend_const.ActionNameType.DOWNGRADE_AGENT
+    ACTION_DESCRIPTION = _("回退")
+
+
 class RestartAgent(AgentAction):
     """
     重启Agent