diff --git a/apps/core/ipchooser/__init__.py b/apps/core/ipchooser/__init__.py index 8ce99929f..5f1dad09f 100644 --- a/apps/core/ipchooser/__init__.py +++ b/apps/core/ipchooser/__init__.py @@ -10,4 +10,4 @@ """ from . import constants -core_encrypt_constants = constants +core_ipchooser_constants = constants diff --git a/apps/core/ipchooser/tools/base.py b/apps/core/ipchooser/tools/base.py index 33baa57b3..bcb4d566c 100644 --- a/apps/core/ipchooser/tools/base.py +++ b/apps/core/ipchooser/tools/base.py @@ -129,6 +129,22 @@ def handle_plugin_conditions( return set(host_queryset) or [-1] return [] + @staticmethod + def fetch_match_node_types(is_proxy: bool, return_all_node_type: bool) -> typing.List[str]: + if is_proxy: + # 单独查代理 + return [node_man_constants.NodeType.PROXY] + elif return_all_node_type: + # 查插件，或者返回全部类型的情况 + return [ + node_man_constants.NodeType.AGENT, + node_man_constants.NodeType.PAGENT, + node_man_constants.NodeType.PROXY, + ] + else: + # 查 Agent + return [node_man_constants.NodeType.AGENT, node_man_constants.NodeType.PAGENT] + @classmethod def multiple_cond_sql( cls, @@ -169,24 +185,9 @@ def multiple_cond_sql( if params.get("bk_biz_id"): final_biz_scope = final_biz_scope & set(params["bk_biz_id"]) - filter_kwargs: typing.Dict[str, typing.Any] = { - "bk_host_id__in": params.get("bk_host_id"), - "bk_biz_id__in": final_biz_scope, - } - - if is_proxy: - # 单独查代理 - node_types = [node_man_constants.NodeType.PROXY] - elif return_all_node_type: - # 查插件，或者返回全部类型的情况 - node_types = [ - node_man_constants.NodeType.AGENT, - node_man_constants.NodeType.PAGENT, - node_man_constants.NodeType.PROXY, - ] - else: - # 查 Agent - node_types = [node_man_constants.NodeType.AGENT, node_man_constants.NodeType.PAGENT] + filter_q: Q = Q(bk_biz_id__in=final_biz_scope) + if params.get("bk_host_id") is not None: + filter_q &= Q(bk_host_id__in=params.get("bk_host_id")) # 条件搜索 where_or = [] @@ -195,12 +196,28 @@ def multiple_cond_sql( for condition in params.get("conditions", []): if condition["key"] in ["inner_ip", "inner_ipv6" "node_from", "node_type", "bk_addressing", "bk_host_name"]: + if condition["key"] in ["inner_ipv6"]: + condition["value"] = basic.ipv6s_formatter(condition["value"]) # host 精确搜索 - filter_kwargs[condition["key"] + "__in"] = condition["value"] + filter_q &= Q(**{f"{condition['key']}__in": condition["value"]}) + + elif condition["key"] in ["ip"]: + ipv6s: typing.Set[str] = set() + ipv4s: typing.Set[str] = set() + + for ip in condition["value"]: + if basic.is_v6(ip): + ipv6s.add(basic.exploded_ip(ip)) + else: + ipv4s.add(ip) + + filter_q &= Q(inner_ip__in=ipv4s) | Q(inner_ipv6__in=ipv6s) elif condition["key"] in ["os_type"]: # 如果传的是 none，替换成 "" - filter_kwargs[condition["key"] + "__in"] = list(map(lambda x: (x, "")[x == "none"], condition["value"])) + filter_q &= Q( + **{f"{condition['key']}__in": list(map(lambda x: (x, "")[x == "none"], condition["value"]))} + ) elif condition["key"] in ["status", "version"]: # process_status 精确搜索 @@ -216,7 +233,7 @@ def multiple_cond_sql( # 对于数字类过滤条件，保证过滤值全数字再拼生成 SQL，否则该条件置空 is_digit_list: bool = "".join([str(cond_val) for cond_val in condition["value"]]).isdigit() if is_digit_list: - filter_kwargs[condition["key"] + "__in"] = condition["value"] + filter_q &= Q(**{f"{condition['key']}__in": condition["value"]}) elif condition["key"] == "topology": # 集群与模块的精准搜索 @@ -289,11 +306,13 @@ def multiple_cond_sql( topo_query = topo_query | Q(bk_host_id__in=topo_host_ids) host_queryset: QuerySet = ( - node_man_models.Host.objects.filter(node_type__in=node_types, bk_biz_id__in=final_biz_scope) + node_man_models.Host.objects.filter( + node_type__in=cls.fetch_match_node_types(is_proxy, return_all_node_type), bk_biz_id__in=final_biz_scope + ) .extra( select=select, tables=[node_man_models.ProcessStatus._meta.db_table], where=wheres, params=sql_params ) - .filter(**basic.filter_values(filter_kwargs)) + .filter(filter_q) .filter(topo_query) ) @@ -439,6 +458,8 @@ def or_query_hosts(cls, host_queryset: QuerySet, or_conditions: typing.List[type or_query = Q() for or_condition in or_conditions: if or_condition["key"] in ["inner_ip", "inner_ipv6", "bk_host_name", "bk_host_id"]: + if or_condition["key"] in ["inner_ipv6"]: + or_condition["val"] = basic.ipv6s_formatter(or_condition["val"]) or_query = or_query | Q(**{f"{or_condition['key']}__in": or_condition["val"]}) elif or_condition["key"] in ["cloud_inner_ip", "cloud_inner_ipv6"]: __, key = or_condition["key"].split("_", 1) diff --git a/apps/node_man/handlers/host.py b/apps/node_man/handlers/host.py index 26c039c8b..7a4240949 100644 --- a/apps/node_man/handlers/host.py +++ b/apps/node_man/handlers/host.py @@ -15,6 +15,7 @@ from django.utils import timezone from django.utils.translation import ugettext_lazy as _ +from apps.core.ipchooser import core_ipchooser_constants from apps.core.ipchooser.tools.base import HostQuerySqlHelper from apps.node_man import constants as const from apps.node_man.constants import IamActionType @@ -142,31 +143,22 @@ def list(self, params: dict, username: str): hosts_status_count = hosts_status_sql.count() if params["only_ip"] is False: + host_fields = core_ipchooser_constants.CommonEnum.DEFAULT_HOST_FIELDS.value + [ + "bk_addressing", + "outer_ip", + "outer_ipv6", + "ap_id", + "install_channel_id", + "login_ip", + "data_ip", + "version", + "created_at", + "updated_at", + "is_manual", + "extra_data", + ] # sql分页查询获得数据 - hosts_status = list( - hosts_status_sql[begin:end].values( - "bk_cloud_id", - "bk_biz_id", - "bk_host_id", - "bk_host_name", - "bk_addressing", - "os_type", - "inner_ip", - "inner_ipv6", - "outer_ip", - "outer_ipv6", - "ap_id", - "install_channel_id", - "login_ip", - "data_ip", - "status", - "version", - "created_at", - "updated_at", - "is_manual", - "extra_data", - ) - ) + hosts_status = list(hosts_status_sql[begin:end].values(*set(host_fields))) else: # 如果仅需要IP数据 hosts_status = [host["inner_ip"] for host in list(hosts_status_sql[begin:end].values("inner_ip"))] diff --git a/apps/node_man/handlers/host_v2.py b/apps/node_man/handlers/host_v2.py index 11e541d30..4b7e11a46 100644 --- a/apps/node_man/handlers/host_v2.py +++ b/apps/node_man/handlers/host_v2.py @@ -11,6 +11,7 @@ from copy import deepcopy from typing import Any, Dict +from apps.core.ipchooser import core_ipchooser_constants from apps.core.ipchooser.tools.base import HostQueryHelper, HostQuerySqlHelper from apps.node_man import models, tools from apps.node_man.constants import DEFAULT_CLOUD_NAME, IamActionType @@ -83,20 +84,9 @@ def list( if only_queryset: return hosts_sql - fetch_fields = [ - "bk_biz_id", - "bk_host_id", - "bk_cloud_id", - "bk_host_name", - "bk_addressing", - "os_type", - "inner_ip", - "inner_ipv6", - "status", - "cpu_arch", - ] + fetch_fields = core_ipchooser_constants.CommonEnum.DEFAULT_HOST_FIELDS.value + ["bk_addressing", "cpu_arch"] - hosts = list(hosts_sql.values(*fetch_fields)[begin:end]) + hosts = list(hosts_sql.values(*set(fetch_fields))[begin:end]) bk_cloud_ids = [host["bk_cloud_id"] for host in hosts] diff --git a/apps/node_man/handlers/plugin.py b/apps/node_man/handlers/plugin.py index faf998d1b..dc95af1c8 100644 --- a/apps/node_man/handlers/plugin.py +++ b/apps/node_man/handlers/plugin.py @@ -14,6 +14,7 @@ from django.db.models import Q from django.utils.translation import get_language +from apps.core.ipchooser import core_ipchooser_constants from apps.core.ipchooser.tools.base import HostQueryHelper, HostQuerySqlHelper from apps.core.tag import targets from apps.core.tag.models import Tag @@ -217,22 +218,14 @@ def list(params: Dict[str, Any]): result = {"total": hosts_status_count, "list": hosts_status} return result else: + host_fields = core_ipchooser_constants.CommonEnum.DEFAULT_HOST_FIELDS.value + [ + "bk_addressing", + "cpu_arch", + "node_type", + "node_from", + ] # sql分页查询获得数据 - hosts_status = list( - hosts_status_sql[begin:end].values( - "bk_biz_id", - "bk_host_id", - "bk_cloud_id", - "bk_host_name", - "bk_addressing", - "inner_ip", - "inner_ipv6", - "os_type", - "cpu_arch", - "node_type", - "node_from", - ) - ) + hosts_status = list(hosts_status_sql[begin:end].values(*set(host_fields))) # 分页结果的Host_id, cloud_id集合 bk_host_ids = [hs["bk_host_id"] for hs in hosts_status] diff --git a/apps/node_man/tools/host_v2.py b/apps/node_man/tools/host_v2.py index 2c7426164..7e775e63e 100644 --- a/apps/node_man/tools/host_v2.py +++ b/apps/node_man/tools/host_v2.py @@ -88,16 +88,15 @@ def get_bk_host_id_plugin_version_map(cls, project: str, bk_host_ids: List[int]) @classmethod def retrieve_host_info(cls, cmdb_host_info: Dict, fields: List[str] = None) -> Dict: fields = fields or ["ip", "bk_biz_id", "bk_cloud_id", "os_type"] - field__cmdb_field__map = {"ip": "bk_host_innerip"} - host_info = {} for field in fields: if field == "os_type": host_info[field] = cls.get_os_type(cmdb_host_info) continue - - cmdb_field = field__cmdb_field__map.get(field, field) - host_info[field] = cmdb_host_info.get(cmdb_field) + elif field == "ip": + host_info[field] = cmdb_host_info.get("bk_host_innerip") or cmdb_host_info.get("bk_host_innerip_v6") + continue + host_info[field] = cmdb_host_info.get(field) return host_info @classmethod diff --git a/apps/node_man/tools/job.py b/apps/node_man/tools/job.py index 8d231e4c4..29da83c0c 100644 --- a/apps/node_man/tools/job.py +++ b/apps/node_man/tools/job.py @@ -9,9 +9,7 @@ specific language governing permissions and limitations under the License. """ import itertools -import operator from collections import Counter -from functools import reduce from typing import Any, Dict, Iterable, List, Optional, Union from django.conf import settings @@ -20,6 +18,7 @@ from django.utils.translation import ugettext_lazy as _ from apps.node_man import constants, models +from apps.utils import basic from apps.utils.local import get_request_username from common.api import NodeApi @@ -197,9 +196,14 @@ def parse2task_result_query_params(cls, job: models.Job, query_params: Dict[str, filter_key_value_list_map[condition["key"]].extend(condition["value"]) host_query = Q() - fuzzy_inner_ips = filter_key_value_list_map["ip"] - if fuzzy_inner_ips: - host_query &= reduce(operator.or_, (Q(inner_ip__contains=fuzzy_ip) for fuzzy_ip in fuzzy_inner_ips)) + for fuzzy_inner_ip in filter_key_value_list_map["ip"]: + if basic.is_v6(fuzzy_inner_ip): + host_query |= Q(inner_ipv6=basic.exploded_ip(fuzzy_inner_ip)) + elif basic.is_v4(fuzzy_inner_ip): + host_query |= Q(inner_ip=fuzzy_inner_ip) + else: + host_query |= Q(inner_ipv6__contains=fuzzy_inner_ip) | Q(inner_ip__contains=fuzzy_inner_ip) + instance_ids = filter_key_value_list_map["instance_id"] base_fields = { "node_type": models.Subscription.NodeType.INSTANCE, @@ -208,13 +212,15 @@ def parse2task_result_query_params(cls, job: models.Job, query_params: Dict[str, if host_query: from apps.backend.subscription.tools import create_node_id - for host in list(models.Host.objects.filter(host_query).values("inner_ip", "bk_cloud_id", "bk_host_id")): + for host in list( + models.Host.objects.filter(host_query).values("inner_ip", "inner_ipv6", "bk_cloud_id", "bk_host_id") + ): instance_ids.extend( [ create_node_id( { **base_fields, - "ip": host["inner_ip"], + "ip": host.get("inner_ip") or host.get("inner_ipv6"), "bk_cloud_id": host["bk_cloud_id"], "bk_supplier_id": constants.DEFAULT_CLOUD, } diff --git a/apps/utils/basic.py b/apps/utils/basic.py index 1b70c7f0f..29f5790ed 100644 --- a/apps/utils/basic.py +++ b/apps/utils/basic.py @@ -12,6 +12,7 @@ import json from collections import Counter, namedtuple from copy import deepcopy +from functools import partial from typing import Any, Dict, Iterable, List, Optional, Set, Union @@ -191,24 +192,60 @@ def get_chr_seq(begin_chr: str, end_chr: str) -> List[str]: return [chr(ascii_int) for ascii_int in range(ord(begin_chr), ord(end_chr) + 1)] -def is_v6(ip) -> bool: +def is_ip(ip: str, _version: Optional[int] = None) -> bool: + """ + 判断是否为合法 IP + :param ip: + :param _version: 是否为合法版本，缺省表示 both + :return: + """ try: ip_address = ipaddress.ip_address(ip) except ValueError: return False - - if ip_address.version == 6: + if _version is None: return True + return ip_address.version == _version + + +# 判断是否为合法 IPv6 +is_v6 = partial(is_ip, _version=6) - return False +# 判断是否为合法 IPv4 +is_v4 = partial(is_ip, _version=4) -def exploded_ip(ip): +def exploded_ip(ip: str) -> str: + """ + 如果 ip 为合法的 IPv6，转为标准格式 + :param ip: + :return: + """ if is_v6(ip): return ipaddress.ip_address(ip).exploded return ip +def compressed_ip(ip: str) -> str: + """ + 如果 ip 为合法的 IPv6，转为压缩格式 + :param ip: + :return: + """ + if is_v6(ip): + return ipaddress.ip_address(ip).compressed + return ip + + +def ipv6s_formatter(ips: List[str]) -> List[str]: + """ + 将 IPv6 列表转为标准格式 + :param ips: + :return: + """ + return [exploded_ip(ip) for ip in ips] + + def ipv6_formatter(data: Dict[str, Any], ipv6_field_names: List[str]): """ 将 data 中 ipv6_field_names 转为 IPv6 标准格式 diff --git a/dev_log/2.2.32/crayon_202211211230.yaml b/dev_log/2.2.32/crayon_202211211230.yaml new file mode 100644 index 000000000..ce8b62a2e --- /dev/null +++ b/dev_log/2.2.32/crayon_202211211230.yaml @@ -0,0 +1,3 @@ +--- +feature: + - "Agent / 插件状态支持 IPv4 / IPv6 混合搜索并提供 AgentID 信息 (closed #1249)" diff --git a/docs/apidoc/search_plugin.md b/docs/apidoc/search_plugin.md index 33ba4740a..aab54b8ad 100644 --- a/docs/apidoc/search_plugin.md +++ b/docs/apidoc/search_plugin.md @@ -26,25 +26,27 @@ 由指定关键词key和value组成的字典 示例：{"key": "inner_ip", "value": ["127.0.0.1"]} -| key | 类型 | value描述 | -| --------------------- | ------ | ----------------------------------------------------------- | -| inner_ip | string | 主机内网IPV4地址 | -| node_from | string | 节点来源，1: CMDB,  配置平台同步 2: EXCEL, saas页面表格导入 3: NODE_MAN，节点管理 | -| node_type | string | 节点类型，1: AGENT, 2: PROXY, 3: PAGENT | -| bk_addressing | string | 寻址方式，1: 0，静态 2: 1，动态 | -| bk_host_name | string | 主机名称 | -| os_type | string | 操作系统，1：LINUX 2：WINDOWS 3：AIX 4：SOLARIS | -| status | string | 进程状态，见status定义 | -| version | string | Agent版本号 | -| is_manual | string | 手动安装 | -| bk_cloud_id | string | 云区域ID | -| install_channel_id | string | 安装通道ID | -| topology | string | 集群与模块的精准搜索 | -| query | string | IP、操作系统、Agent状态、Agent版本、云区域模糊搜索,对应value为列表时为多模糊搜索 | -| source_id | string | 来源ID | -| plugin_name | string | 插件名，展开任务下所有的插件名称 | -| ${plugin_name} | string | 插件版本的精确搜索，${plugin_name}为对应的目标插件名称 | -| ${plugin_name}_status | string | 插件状态的精确搜索，${plugin_name}为对应的目标插件名称 | +| key | 类型 | value描述 | +|-----------------------|--------|------------------------------------------------------------| +| ip | list | 主机内网 IP，支持 v4 / v6 混合输入 | +| inner_ip | list | 主机内网IPv4地址 | +| inner_ipv6 | list | 主机内网IPv6地址 | +| node_from | list | 节点来源，1: CMDB, 配置平台同步 2: EXCEL, saas页面表格导入 3: NODE_MAN，节点管理 | +| node_type | list | 节点类型，1: AGENT, 2: PROXY, 3: PAGENT | +| bk_addressing | list | 寻址方式，1: 0，静态 2: 1，动态 | +| bk_host_name | list | 主机名称 | +| os_type | list | 操作系统，1：LINUX 2：WINDOWS 3：AIX 4：SOLARIS | +| status | list | 进程状态，见status定义 | +| version | list | Agent版本号 | +| is_manual | list | 手动安装 | +| bk_cloud_id | list | 云区域ID | +| install_channel_id | list | 安装通道ID | +| topology | string | 集群与模块的精准搜索 | +| query | string | IP、操作系统、Agent状态、Agent版本、云区域模糊搜索,对应value为列表时为多模糊搜索 | +| source_id | string | 来源ID | +| plugin_name | string | 插件名，展开任务下所有的插件名称 | +| ${plugin_name} | string | 插件版本的精确搜索，${plugin_name}为对应的目标插件名称 | +| ${plugin_name}_status | string | 插件状态的精确搜索，${plugin_name}为对应的目标插件名称 | ### 请求参数示例