feature: Agent / 插件状态支持 IPv4 / IPv6 混合搜索并提供 AgentID 信息 (closed Tencen…

…tBlueKing#1249)

apps/core/ipchooser/__init__.py

@@ -10,4 +10,4 @@
 """
 from . import constants
 
-core_encrypt_constants = constants
+core_ipchooser_constants = constants

apps/core/ipchooser/tools/base.py

@@ -129,6 +129,22 @@ def handle_plugin_conditions(
                 return set(host_queryset) or [-1]
         return []
 
+    @staticmethod
+    def fetch_match_node_types(is_proxy: bool, return_all_node_type: bool) -> typing.List[str]:
+        if is_proxy:
+            # 单独查代理
+            return [node_man_constants.NodeType.PROXY]
+        elif return_all_node_type:
+            # 查插件，或者返回全部类型的情况
+            return [
+                node_man_constants.NodeType.AGENT,
+                node_man_constants.NodeType.PAGENT,
+                node_man_constants.NodeType.PROXY,
+            ]
+        else:
+            # 查 Agent
+            return [node_man_constants.NodeType.AGENT, node_man_constants.NodeType.PAGENT]
+
     @classmethod
     def multiple_cond_sql(
         cls,
@@ -169,24 +185,9 @@ def multiple_cond_sql(
         if params.get("bk_biz_id"):
             final_biz_scope = final_biz_scope & set(params["bk_biz_id"])
 
-        filter_kwargs: typing.Dict[str, typing.Any] = {
-            "bk_host_id__in": params.get("bk_host_id"),
-            "bk_biz_id__in": final_biz_scope,
-        }
-
-        if is_proxy:
-            # 单独查代理
-            node_types = [node_man_constants.NodeType.PROXY]
-        elif return_all_node_type:
-            # 查插件，或者返回全部类型的情况
-            node_types = [
-                node_man_constants.NodeType.AGENT,
-                node_man_constants.NodeType.PAGENT,
-                node_man_constants.NodeType.PROXY,
-            ]
-        else:
-            # 查 Agent
-            node_types = [node_man_constants.NodeType.AGENT, node_man_constants.NodeType.PAGENT]
+        filter_q: Q = Q(bk_biz_id__in=final_biz_scope)
+        if params.get("bk_host_id") is not None:
+            filter_q &= Q(bk_host_id__id=params.get("bk_host_id"))
 
         # 条件搜索
         where_or = []
@@ -195,12 +196,28 @@ def multiple_cond_sql(
 
         for condition in params.get("conditions", []):
             if condition["key"] in ["inner_ip", "inner_ipv6" "node_from", "node_type", "bk_addressing", "bk_host_name"]:
+                if condition["key"] in ["inner_ipv6"]:
+                    condition["value"] = basic.ipv6s_formatter(condition["value"])
                 # host 精确搜索
-                filter_kwargs[condition["key"] + "__in"] = condition["value"]
+                filter_q &= Q(**{f"{condition['key']}__in": condition["value"]})
+
+            elif condition["key"] in ["ip"]:
+                ipv6s: typing.Set[str] = set()
+                ipv4s: typing.Set[str] = set()
+
+                for ip in condition["value"]:
+                    if basic.is_v6(ip):
+                        ipv6s.add(ip)
+                    else:
+                        ipv4s.add(ip)
+
+                filter_q &= Q(inner_ip__in=ipv4s) | Q(inner_ipv6__in=ipv6s)
 
             elif condition["key"] in ["os_type"]:
                 # 如果传的是 none，替换成 ""
-                filter_kwargs[condition["key"] + "__in"] = list(map(lambda x: (x, "")[x == "none"], condition["value"]))
+                filter_q &= Q(
+                    **{f"{condition['key']}__in": list(map(lambda x: (x, "")[x == "none"], condition["value"]))}
+                )
 
             elif condition["key"] in ["status", "version"]:
                 # process_status 精确搜索
@@ -216,7 +233,7 @@ def multiple_cond_sql(
                 # 对于数字类过滤条件，保证过滤值全数字再拼生成 SQL，否则该条件置空
                 is_digit_list: bool = "".join([str(cond_val) for cond_val in condition["value"]]).isdigit()
                 if is_digit_list:
-                    filter_kwargs[condition["key"] + "__in"] = condition["value"]
+                    filter_q &= Q(**{f"{condition['key']}__in": condition["value"]})
 
             elif condition["key"] == "topology":
                 # 集群与模块的精准搜索
@@ -289,11 +306,13 @@ def multiple_cond_sql(
             topo_query = topo_query | Q(bk_host_id__in=topo_host_ids)
 
         host_queryset: QuerySet = (
-            node_man_models.Host.objects.filter(node_type__in=node_types, bk_biz_id__in=final_biz_scope)
+            node_man_models.Host.objects.filter(
+                node_type__in=cls.fetch_match_node_types(is_proxy, return_all_node_type), bk_biz_id__in=final_biz_scope
+            )
             .extra(
                 select=select, tables=[node_man_models.ProcessStatus._meta.db_table], where=wheres, params=sql_params
             )
-            .filter(**basic.filter_values(filter_kwargs))
+            .filter(filter_q)
             .filter(topo_query)
         )
 
@@ -439,6 +458,8 @@ def or_query_hosts(cls, host_queryset: QuerySet, or_conditions: typing.List[type
         or_query = Q()
         for or_condition in or_conditions:
             if or_condition["key"] in ["inner_ip", "inner_ipv6", "bk_host_name", "bk_host_id"]:
+                if or_condition["key"] in ["inner_ipv6"]:
+                    or_condition["val"] = basic.ipv6s_formatter(or_condition["val"])
                 or_query = or_query | Q(**{f"{or_condition['key']}__in": or_condition["val"]})
             elif or_condition["key"] in ["cloud_inner_ip", "cloud_inner_ipv6"]:
                 __, key = or_condition["key"].split("_", 1)

apps/node_man/handlers/host.py

@@ -15,6 +15,7 @@
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
+from apps.core.ipchooser import core_ipchooser_constants
 from apps.core.ipchooser.tools.base import HostQuerySqlHelper
 from apps.node_man import constants as const
 from apps.node_man.constants import IamActionType
@@ -142,31 +143,22 @@ def list(self, params: dict, username: str):
         hosts_status_count = hosts_status_sql.count()
 
         if params["only_ip"] is False:
+            host_fields = core_ipchooser_constants.CommonEnum.DEFAULT_HOST_FIELDS.value + [
+                "bk_addressing",
+                "outer_ip",
+                "outer_ipv6",
+                "ap_id",
+                "install_channel_id",
+                "login_ip",
+                "data_ip",
+                "version",
+                "created_at",
+                "updated_at",
+                "is_manual",
+                "extra_data",
+            ]
             # sql分页查询获得数据
-            hosts_status = list(
-                hosts_status_sql[begin:end].values(
-                    "bk_cloud_id",
-                    "bk_biz_id",
-                    "bk_host_id",
-                    "bk_host_name",
-                    "bk_addressing",
-                    "os_type",
-                    "inner_ip",
-                    "inner_ipv6",
-                    "outer_ip",
-                    "outer_ipv6",
-                    "ap_id",
-                    "install_channel_id",
-                    "login_ip",
-                    "data_ip",
-                    "status",
-                    "version",
-                    "created_at",
-                    "updated_at",
-                    "is_manual",
-                    "extra_data",
-                )
-            )
+            hosts_status = list(hosts_status_sql[begin:end].values(*set(host_fields)))
         else:
             # 如果仅需要IP数据
             hosts_status = [host["inner_ip"] for host in list(hosts_status_sql[begin:end].values("inner_ip"))]

apps/node_man/handlers/host_v2.py

@@ -11,6 +11,7 @@
 from copy import deepcopy
 from typing import Any, Dict
 
+from apps.core.ipchooser import core_ipchooser_constants
 from apps.core.ipchooser.tools.base import HostQueryHelper, HostQuerySqlHelper
 from apps.node_man import models, tools
 from apps.node_man.constants import DEFAULT_CLOUD_NAME, IamActionType
@@ -83,20 +84,9 @@ def list(
         if only_queryset:
             return hosts_sql
 
-        fetch_fields = [
-            "bk_biz_id",
-            "bk_host_id",
-            "bk_cloud_id",
-            "bk_host_name",
-            "bk_addressing",
-            "os_type",
-            "inner_ip",
-            "inner_ipv6",
-            "status",
-            "cpu_arch",
-        ]
+        fetch_fields = core_ipchooser_constants.CommonEnum.DEFAULT_HOST_FIELDS.value + ["bk_addressing", "cpu_arch"]
 
-        hosts = list(hosts_sql.values(*fetch_fields)[begin:end])
+        hosts = list(hosts_sql.values(*set(fetch_fields))[begin:end])
 
         bk_cloud_ids = [host["bk_cloud_id"] for host in hosts]
 

apps/node_man/handlers/plugin.py

@@ -14,6 +14,7 @@
 from django.db.models import Q
 from django.utils.translation import get_language
 
+from apps.core.ipchooser import core_ipchooser_constants
 from apps.core.ipchooser.tools.base import HostQueryHelper, HostQuerySqlHelper
 from apps.core.tag import targets
 from apps.core.tag.models import Tag
@@ -217,22 +218,14 @@ def list(params: Dict[str, Any]):
             result = {"total": hosts_status_count, "list": hosts_status}
             return result
         else:
+            host_fields = core_ipchooser_constants.CommonEnum.DEFAULT_HOST_FIELDS.value + [
+                "bk_addressing",
+                "cpu_arch",
+                "node_type",
+                "node_from",
+            ]
             # sql分页查询获得数据
-            hosts_status = list(
-                hosts_status_sql[begin:end].values(
-                    "bk_biz_id",
-                    "bk_host_id",
-                    "bk_cloud_id",
-                    "bk_host_name",
-                    "bk_addressing",
-                    "inner_ip",
-                    "inner_ipv6",
-                    "os_type",
-                    "cpu_arch",
-                    "node_type",
-                    "node_from",
-                )
-            )
+            hosts_status = list(hosts_status_sql[begin:end].values(*set(host_fields)))
 
         # 分页结果的Host_id, cloud_id集合
         bk_host_ids = [hs["bk_host_id"] for hs in hosts_status]

apps/node_man/tools/host_v2.py

@@ -88,16 +88,15 @@ def get_bk_host_id_plugin_version_map(cls, project: str, bk_host_ids: List[int])
     @classmethod
     def retrieve_host_info(cls, cmdb_host_info: Dict, fields: List[str] = None) -> Dict:
         fields = fields or ["ip", "bk_biz_id", "bk_cloud_id", "os_type"]
-        field__cmdb_field__map = {"ip": "bk_host_innerip"}
-
         host_info = {}
         for field in fields:
             if field == "os_type":
                 host_info[field] = cls.get_os_type(cmdb_host_info)
                 continue
-
-            cmdb_field = field__cmdb_field__map.get(field, field)
-            host_info[field] = cmdb_host_info.get(cmdb_field)
+            elif field == "ip":
+                host_info[field] = cmdb_host_info.get("bk_host_innerip") or cmdb_host_info.get("bk_host_innerip_v6")
+                continue
+            host_info[field] = cmdb_host_info.get(field)
         return host_info
 
     @classmethod

apps/node_man/tools/job.py

@@ -9,9 +9,7 @@
 specific language governing permissions and limitations under the License.
 """
 import itertools
-import operator
 from collections import Counter
-from functools import reduce
 from typing import Any, Dict, Iterable, List, Optional, Union
 
 from django.conf import settings
@@ -20,6 +18,7 @@
 from django.utils.translation import ugettext_lazy as _
 
 from apps.node_man import constants, models
+from apps.utils import basic
 from apps.utils.local import get_request_username
 from common.api import NodeApi
 
@@ -197,9 +196,14 @@ def parse2task_result_query_params(cls, job: models.Job, query_params: Dict[str,
                     filter_key_value_list_map[condition["key"]].extend(condition["value"])
 
         host_query = Q()
-        fuzzy_inner_ips = filter_key_value_list_map["ip"]
-        if fuzzy_inner_ips:
-            host_query &= reduce(operator.or_, (Q(inner_ip__contains=fuzzy_ip) for fuzzy_ip in fuzzy_inner_ips))
+        for fuzzy_inner_ip in filter_key_value_list_map["ip"]:
+            if basic.is_v6(fuzzy_inner_ip):
+                host_query |= Q(inner_ipv6=basic.exploded_ip(fuzzy_inner_ip))
+            elif basic.is_v4(fuzzy_inner_ip):
+                host_query |= Q(inner_ip=fuzzy_inner_ip)
+            else:
+                host_query |= Q(inner_ipv6__contains=fuzzy_inner_ip) | Q(inner_ip__contains=fuzzy_inner_ip)
+
         instance_ids = filter_key_value_list_map["instance_id"]
         base_fields = {
             "node_type": models.Subscription.NodeType.INSTANCE,
@@ -208,13 +212,15 @@ def parse2task_result_query_params(cls, job: models.Job, query_params: Dict[str,
         if host_query:
             from apps.backend.subscription.tools import create_node_id
 
-            for host in list(models.Host.objects.filter(host_query).values("inner_ip", "bk_cloud_id", "bk_host_id")):
+            for host in list(
+                models.Host.objects.filter(host_query).values("inner_ip", "inner_ipv6", "bk_cloud_id", "bk_host_id")
+            ):
                 instance_ids.extend(
                     [
                         create_node_id(
                             {
                                 **base_fields,
-                                "ip": host["inner_ip"],
+                                "ip": host.get("inner_ip") or host.get("inner_ipv6"),
                                 "bk_cloud_id": host["bk_cloud_id"],
                                 "bk_supplier_id": constants.DEFAULT_CLOUD,
                             }

apps/utils/basic.py

@@ -12,6 +12,7 @@
 import json
 from collections import Counter, namedtuple
 from copy import deepcopy
+from functools import partial
 from typing import Any, Dict, Iterable, List, Optional, Set, Union
 
 
@@ -191,24 +192,60 @@ def get_chr_seq(begin_chr: str, end_chr: str) -> List[str]:
     return [chr(ascii_int) for ascii_int in range(ord(begin_chr), ord(end_chr) + 1)]
 
 
-def is_v6(ip) -> bool:
+def is_ip(ip: str, _version: Optional[int] = None) -> bool:
+    """
+    判断是否为合法 IP
+    :param ip:
+    :param _version: 是否为合法版本，缺省表示 both
+    :return:
+    """
     try:
         ip_address = ipaddress.ip_address(ip)
     except ValueError:
         return False
-
-    if ip_address.version == 6:
+    if _version is None:
         return True
+    return ip_address.version == _version
+
+
+# 判断是否为合法 IPv6
+is_v6 = partial(is_ip, _version=6)
 
-    return False
+# 判断是否为合法 IPv4
+is_v4 = partial(is_ip, _version=4)
 
 
-def exploded_ip(ip):
+def exploded_ip(ip: str) -> str:
+    """
+    如果 ip 为合法的 IPv6，转为标准格式
+    :param ip:
+    :return:
+    """
     if is_v6(ip):
         return ipaddress.ip_address(ip).exploded
     return ip
 
 
+def compressed_ip(ip: str) -> str:
+    """
+    如果 ip 为合法的 IPv6，转为压缩格式
+    :param ip:
+    :return:
+    """
+    if is_v6(ip):
+        return ipaddress.ip_address(ip).compressed
+    return ip
+
+
+def ipv6s_formatter(ips: List[str]) -> List[str]:
+    """
+    将 IPv6 列表转为标准格式
+    :param ips:
+    :return:
+    """
+    return [exploded_ip(ip) for ip in ips]
+
+
 def ipv6_formatter(data: Dict[str, Any], ipv6_field_names: List[str]):
     """
     将 data 中 ipv6_field_names 转为 IPv6 标准格式

dev_log/2.2.32/crayon_202211211230.yaml

@@ -0,0 +1,3 @@
+---
+feature:
+  - "Agent / 插件状态支持 IPv4 / IPv6 混合搜索并提供 AgentID 信息 (closed #1249)"