Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: 安装预设插件锁定版本(closed #2482) #2493

Open
wants to merge 1 commit into
base: v2.4.8-dev
Choose a base branch
from
Open

feat: 安装预设插件锁定版本(closed #2482) #2493

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions apps/backend/components/collections/plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ def get_package_by_process_status(
"""通过进程状态得到插件包对象"""
host = self.get_host_by_process_status(process_status, common_data)
policy_step_adapter = common_data.policy_step_adapter
package = policy_step_adapter.get_matching_package_obj(host.os_type, host.cpu_arch)
package = policy_step_adapter.get_matching_package_obj(host.os_type, host.cpu_arch, host.bk_biz_id)
return package

def get_plugin_root_by_process_status(
Expand Down Expand Up @@ -280,11 +280,12 @@ def _execute(self, data, parent_data, common_data: PluginCommonData):
# target_host_objs 的长度通常为1或2，此处也不必担心时间复杂度问题
# 指定 target_host 主要用于远程采集的场景，常见于第三方插件，如拨测
for host in target_host_objs:
bk_biz_id = host.bk_biz_id
bk_host_id = host.bk_host_id
os_type = host.os_type.lower()
cpu_arch = host.cpu_arch
group_id = create_group_id(subscription, subscription_instance.instance_info)
package = self.get_package(subscription_instance, policy_step_adapter, os_type, cpu_arch)
package = self.get_package(subscription_instance, policy_step_adapter, os_type, cpu_arch, bk_biz_id)
ap_config = self.get_ap_config(ap_id_obj_map, host)
setup_path, pid_path, log_path, data_path = self.get_plugins_paths(
package, plugin_name, ap_config, group_id, subscription
Expand Down Expand Up @@ -340,10 +341,11 @@ def get_package(
policy_step_adapter: PolicyStepAdapter,
os_type: str,
cpu_arch: str,
bk_biz_id: int,
) -> models.Packages:
"""获取插件包对象"""
try:
return policy_step_adapter.get_matching_package_obj(os_type, cpu_arch)
return policy_step_adapter.get_matching_package_obj(os_type, cpu_arch, bk_biz_id)
except errors.PackageNotExists as error:
# 插件包不支持或不存在时，记录异常信息，此实例不参与后续流程
self.move_insts_to_failed([subscription_instance.id], str(error))
Expand Down Expand Up @@ -979,7 +981,9 @@ def _execute(self, data, parent_data, common_data: PluginCommonData):

# 根据配置模板和上下文变量渲染配置文件
rendered_configs = render_config_files_by_config_templates(
policy_step_adapter.get_matching_config_tmpl_objs(target_host.os_type, target_host.cpu_arch),
policy_step_adapter.get_matching_config_tmpl_objs(
target_host.os_type, target_host.cpu_arch, package, target_host.bk_biz_id, subscription_step.config
),
{"group_id": process_status.group_id},
context,
package_obj=package,
Expand Down
111 changes: 97 additions & 14 deletions apps/backend/subscription/steps/adapter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

from django.db.models import Max, Subquery, Value
from django.utils.translation import ugettext as _
from packaging import version
from rest_framework import exceptions, serializers

from apps.backend.subscription import errors
Expand Down Expand Up @@ -284,19 +285,19 @@ def max_ids_by_key(self, contained_os_cpu_items: List[Dict[str, Any]]) -> List[i
def format2policy_packages_new(
self, plugin_id: int, plugin_name: str, plugin_version: str, config_templates: List[Dict[str, Any]]
) -> List[Dict[str, Any]]:
latest_flag: str = "latest"
is_tag: bool = Tag.objects.filter(
target_id=plugin_id, name=latest_flag, target_type=TargetType.PLUGIN.value
).exists()
tags: List[str] = ["latest", "stable"]
is_tag: Dict[str, bool] = {
tag: Tag.objects.filter(target_id=plugin_id, name=tag, target_type=TargetType.PLUGIN.value).exists()
for tag in tags
}

if plugin_version != latest_flag or is_tag:
# 如果 latest 是 tag，走取指定版本的逻辑
packages = models.Packages.objects.filter(project=plugin_name, version=plugin_version)
else:
if plugin_version in tags and not is_tag[plugin_version]:
max_pkg_ids: List[int] = self.max_ids_by_key(
list(models.Packages.objects.filter(project=plugin_name).values("id", "os", "cpu_arch"))
)
packages = models.Packages.objects.filter(id__in=max_pkg_ids)
else:
packages = self.get_packages(plugin_name, plugin_version)

if not packages:
raise errors.PluginValidationError(
Expand All @@ -306,11 +307,11 @@ def format2policy_packages_new(
os_cpu__config_templates_map = defaultdict(list)
for template in config_templates:
is_main_template = template["is_main"]
if template["version"] != latest_flag or is_tag:
plugin_version_set = {plugin_version, "*"}
if plugin_version in tags and not is_tag[plugin_version]:
tag_packages_version_set = set(packages.values_list("version", flat=True))
plugin_version_set = tag_packages_version_set | {"*"}
else:
latest_packages_version_set = set(packages.values_list("version", flat=True))
plugin_version_set = latest_packages_version_set | {"*"}
plugin_version_set = {plugin_version, "*"}

max_config_tmpl_ids: typing.List[int] = self.max_ids_by_key(
list(
Expand Down Expand Up @@ -444,9 +445,18 @@ def get_matching_step_params(self, os_type: str = None, cpu_arch: str = None, os
return self.os_key_params_map.get(os_key)
return self.os_key_params_map.get(self.get_os_key(os_type, cpu_arch), {})

def get_matching_package_obj(self, os_type: str, cpu_arch: str) -> models.Packages:
def get_matching_package_obj(self, os_type: str, cpu_arch: str, bk_biz_id: int) -> models.Packages:
try:
package = self.os_key_pkg_map[self.get_os_key(os_type, cpu_arch)]
version_str = getattr(package, "version", "")
tag_name__obj_map: Dict[str, Tag] = PluginTargetHelper.get_tag_name__obj_map(
target_id=self.plugin_desc.id,
)
if version_str in tag_name__obj_map:
version_str = tag_name__obj_map[version_str].target_version
biz_version = self.get_biz_version(package, bk_biz_id)
if biz_version and version.Version(version_str) > version.Version(biz_version):
package = self.get_biz_package(package.project, os_type, cpu_arch, biz_version)
except KeyError:
msg = _("插件 [{name}] 不支持 系统:{os_type}-架构:{cpu_arch}-版本:{plugin_version}").format(
name=self.plugin_name,
Expand All @@ -466,5 +476,78 @@ def get_matching_package_obj(self, os_type: str, cpu_arch: str) -> models.Packag
raise errors.PluginValidationError(msg)
return package

def get_matching_config_tmpl_objs(self, os_type: str, cpu_arch: str) -> List[models.PluginConfigTemplate]:
def get_matching_config_tmpl_objs(
self, os_type: str, cpu_arch: str, package: models.Packages, bk_biz_id: int, config: Dict
) -> List[models.PluginConfigTemplate]:
# 如果当前业务设定了业务最大版本，重新从数据库中获取
if str(bk_biz_id) in self.plugin_version_config():
config_tmpl = (
models.PluginConfigTemplate.objects.filter(
name=config["config_templates"][0]["name"],
plugin_name=package.project,
plugin_version=package.version,
is_main=Value(1 if config["config_templates"][0]["is_main"] else 0),
)
.order_by("-id")
.first()
)
return config_tmpl
return self.config_tmpl_obj_gby_os_key.get(self.get_os_key(os_type, cpu_arch), [])

def get_biz_package(self, plugin_name: str, os_type: str, cpu_arch: str, biz_version: str):
"""获取业务锁定版本的插件包"""
packages_all = self.get_packages(plugin_name, biz_version)
packages = packages_all.filter(
id__in=[pkg.id for pkg in packages_all if version.Version(pkg.version) <= version.Version(biz_version)]
)
os_cpu__biz_pkg_map = {self.get_os_key(package.os, package.cpu_arch): package for package in packages}
if not os_cpu__biz_pkg_map:
raise errors.PluginValidationError(
msg="插件 [{name}-{versions}] 不存在".format(name=self.plugin_name, versions=biz_version)
)
package = os_cpu__biz_pkg_map[self.get_os_key(os_type, cpu_arch)]
return package

@staticmethod
def plugin_version_config():
"""业务锁定版本配置"""
plugin_version_config: Dict[str, Dict[str, str]] = models.GlobalSettings.get_config(
models.GlobalSettings.KeyEnum.PLUGIN_VERSION_CONFIG.value, default={}
)
return plugin_version_config

@staticmethod
def get_biz_version(package: models.Packages, bk_biz_id: int):
"""获取业务锁定版本"""
plugin_version_config = PolicyStepAdapter.plugin_version_config()
biz_version = None
if str(bk_biz_id) in plugin_version_config:
biz_version_config = plugin_version_config[str(bk_biz_id)]
biz_version = next(
(
biz_plugin_version
for biz_plugin_name, biz_plugin_version in biz_version_config.items()
if package.project == biz_plugin_name
),
None,
)
return biz_version

def get_packages(self, plugin_name: str, plugin_version: str, biz_version: str = None):
"""如果不存在某个版本，则取最大id的版本"""
all_packages = models.Packages.objects.filter(project=plugin_name).values("id", "os", "cpu_arch", "version")
version_packages = {pkg["id"]: pkg for pkg in all_packages if pkg["version"] == plugin_version}
package_ids = set(version_packages.keys())
for os in constants.PLUGIN_OS_TUPLE:
for cpu_arch in constants.CPU_TUPLE:
if not any(
pkg["os"] == os and pkg["cpu_arch"] == cpu_arch
for pkg in all_packages
if pkg["version"] == plugin_version
):
max_pkg_ids: List[int] = self.max_ids_by_key(
[pkg for pkg in all_packages if pkg["os"] == os and pkg["cpu_arch"] == cpu_arch]
)
package_ids.update(max_pkg_ids)
packages = models.Packages.objects.filter(id__in=package_ids)
return packages
74 changes: 17 additions & 57 deletions apps/backend/tests/plugin/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,45 +149,19 @@
"type": "object",
"required": True,
"properties": {
"token": {
"title": "token",
"type": "string",
"required": True
},
"logVerbosity": {
"title": "logVerbosity",
"type": "number",
"required": False,
"default": 5
},
"tempDir": {
"title": "tempDir",
"type": "string",
"required": False
},
"uid": {
"title": "uid",
"type": "string",
"required": False
},
"token": {"title": "token", "type": "string", "required": True},
"logVerbosity": {"title": "logVerbosity", "type": "number", "required": False, "default": 5},
"tempDir": {"title": "tempDir", "type": "string", "required": False},
"uid": {"title": "uid", "type": "string", "required": False},
"labels": {
"title": "labels",
"type": "array",
"items": {
"title": "label",
"type": "object",
"required": False,
"properties": {
"key": {
"title": "key",
"type": "string"
},
"value": {
"title": "value",
"type": "string"
}
}
}
"properties": {"key": {"title": "键", "type": "string"}, "value": {"title": "值", "type": "string"}},
},
},
"apps": {
"title": "apps",
Expand All @@ -196,16 +170,8 @@
"title": "named_label",
"type": "object",
"properties": {
"name": {
"title": "name",
"type": "string",
"required": True
},
"uid": {
"title": "uid",
"type": "string",
"required": False
},
"name": {"title": "name", "type": "string", "required": True},
"uid": {"title": "uid", "type": "string", "required": False},
"labels": {
"title": "labels",
"type": "array",
Expand All @@ -215,21 +181,15 @@
"type": "object",
"required": False,
"properties": {
"key": {
"title": "key",
"type": "string"
},
"value": {
"title": "value",
"type": "string"
}
}
}
}
}
}
}
}
"key": {"title": "键", "type": "string"},
"value": {"title": "值", "type": "string"},
},
},
},
},
},
},
},
}

# 插件名称
Expand Down
2 changes: 2 additions & 0 deletions apps/node_man/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ class KeyEnum(Enum):
INSTALL_CHANNEL_ID_NETWORK_SEGMENT = "INSTALL_CHANNEL_ID_NETWORK_SEGMENT"
# 需要执行清理订阅的APP_CODE
NEED_CLEAN_SUBSCRIPTION_APP_CODE = "NEED_CLEAN_SUBSCRIPTION_APP_CODE"
# 业务最大插件版本
PLUGIN_VERSION_CONFIG = "PLUGIN_VERSION_CONFIG"

key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True)
v_json = JSONField(_("值"))
Expand Down