diff --git a/apps/mock_data/common_unit/host.py b/apps/mock_data/common_unit/host.py index 33411c20f..75d657968 100644 --- a/apps/mock_data/common_unit/host.py +++ b/apps/mock_data/common_unit/host.py @@ -30,9 +30,18 @@ "ap_type": "system", "region_id": "test", "city_id": "test", - "btfileserver": [{"inner_ip": DEFAULT_IP, "outer_ip": DEFAULT_IP}], - "dataserver": [{"inner_ip": DEFAULT_IP, "outer_ip": DEFAULT_IP}], - "taskserver": [{"inner_ip": DEFAULT_IP, "outer_ip": DEFAULT_IP}], + "btfileserver": { + "inner_ips": [{"inner_ip": DEFAULT_IP}], + "outer_ips": [{"outer_ip": DEFAULT_IP}], + }, + "dataserver": { + "inner_ips": [{"inner_ip": DEFAULT_IP}], + "outer_ips": [{"outer_ip": DEFAULT_IP}], + }, + "taskserver": { + "inner_ips": [{"inner_ip": DEFAULT_IP}], + "outer_ips": [{"outer_ip": DEFAULT_IP}], + }, "zk_hosts": [{"zk_ip": DEFAULT_IP, "zk_port": "2181"}], "zk_account": "zk_account", "zk_password": "zk_password", diff --git a/apps/node_man/management/commands/transform_ap_data.py b/apps/node_man/management/commands/transform_ap_data.py new file mode 100644 index 000000000..cf374c7d6 --- /dev/null +++ b/apps/node_man/management/commands/transform_ap_data.py @@ -0,0 +1,79 @@ +# coding: utf-8 +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import typing + +from django.core.management.base import BaseCommand + +from apps.node_man import models +from apps.node_man.utils.endpoint import EndPointTransform +from common.log import logger + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "-e", "--transform", type=bool, required=False, help="AP_ID create from V1 AP_ID", default=False + ) + parser.add_argument( + "-l", + "--transform_endpoint_to_leagcy", + action="store_true", + default=False, + help="Clean up the original mapping ID", + ) + parser.add_argument( + "-a", + "--all_ap", + action="store_true", + default=False, + help="Transform all the AP_IDs in the database", + ) + parser.add_argument( + "-t", + "--transform_ap_id", + type=int, + required=False, + help="Transform target AP_ID in the database", + ) + + def handle(self, **options): + transform_endpoint_to_leagcy = options.get("transform_endpoint_to_leagcy") + transform = options.get("transform") + if not transform_endpoint_to_leagcy and not transform: + raise ValueError("Please specify the AP_ID to be transformed") + if transform and transform_endpoint_to_leagcy: + raise ValueError("Please specify only one AP_ID to be transformed") + + all_ap_transform = options.get("all_ap") + transform_ap_id = options.get("transform_ap_id") + if all_ap_transform and transform_ap_id: + raise ValueError("Please specify only one AP_ID to be transformed") + if not all_ap_transform and not transform_ap_id: + raise ValueError("Please specify the AP_ID to be transformed") + + if all_ap_transform: + ap_objects: typing.List[models.AccessPoint] = models.AccessPoint.objects.all() + else: + ap_objects: typing.List[models.AccessPoint] = models.AccessPoint.objects.filter(id=transform_ap_id) + + if transform_endpoint_to_leagcy: + transform_func: typing.Callable = EndPointTransform().transform_endpoint_to_leagcy + elif transform: + transform_func: typing.Callable = EndPointTransform().transform + else: + raise ValueError("Please specify the transformation method") + + for ap_object in ap_objects: + logger.info(f"Transforming AP_ID: {ap_object.id}") + ap_object.taskserver = transform_func(ap_object.taskserver) + ap_object.dataserver = transform_func(ap_object.dataserver) + ap_object.btfileserver = transform_func(ap_object.btfileserver) + ap_object.save() diff --git a/apps/node_man/migrations/0081_covert_ap_data_20231109_1336.py b/apps/node_man/migrations/0081_covert_ap_data_20231109_1336.py new file mode 100644 index 000000000..5833a938f --- /dev/null +++ b/apps/node_man/migrations/0081_covert_ap_data_20231109_1336.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.4 on 2023-10-29 05:36 + +from django.db import migrations, models + +from apps.node_man.utils.endpoint import EndPointTransform + + +def covert_ap_data(apps, schema_editor): + AccessPoint = apps.get_model("node_man", "AccessPoint") + aps = AccessPoint.objects.all() + for ap in aps: + # 转换 gse 地址，从一对一关系，转换为两个列表 + ap.btfileserver = EndPointTransform().transform(legacy_endpoints=ap.btfileserver) + ap.dataserver = EndPointTransform().transform(legacy_endpoints=ap.dataserver) + ap.taskserver = EndPointTransform().transform(legacy_endpoints=ap.taskserver) + ap.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("node_man", "0080_auto_20231122_1552"), + ] + + operations = [ + migrations.RunPython(covert_ap_data), + ] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 4d8deae83..4e13f3181 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -518,6 +518,10 @@ class Meta: class AccessPoint(models.Model): + class ServersType(object): + OUTER_IPS = "outer_ips" + INNER_IPS = "inner_ips" + name = models.CharField(_("接入点名称"), max_length=255) ap_type = models.CharField(_("接入点类型"), max_length=255, default="user") region_id = models.CharField(_("区域id"), max_length=255, default="", blank=True, null=True) @@ -548,15 +552,15 @@ class AccessPoint(models.Model): @property def file_endpoint_info(self) -> EndpointInfo: - return EndpointInfo(inner_server_infos=self.btfileserver, outer_server_infos=self.btfileserver) + return EndpointInfo(endpoints=self.btfileserver) @property def data_endpoint_info(self) -> EndpointInfo: - return EndpointInfo(inner_server_infos=self.dataserver, outer_server_infos=self.dataserver) + return EndpointInfo(endpoints=self.dataserver) @property def cluster_endpoint_info(self) -> EndpointInfo: - return EndpointInfo(inner_server_infos=self.taskserver, outer_server_infos=self.taskserver) + return EndpointInfo(endpoints=self.taskserver) @classmethod def ap_id_obj_map(cls): @@ -881,7 +885,10 @@ class GsePluginDesc(models.Model): scenario_en = models.TextField(_("英文使用场景"), null=True, blank=True) category = models.CharField(_("所属范围"), max_length=32, choices=constants.CATEGORY_CHOICES) launch_node = models.CharField( - _("宿主节点类型要求"), max_length=32, choices=[("agent", "agent"), ("proxy", "proxy"), ("all", "all")], default="all" + _("宿主节点类型要求"), + max_length=32, + choices=[("agent", "agent"), ("proxy", "proxy"), ("all", "all")], + default="all", ) config_file = models.CharField(_("配置文件名称"), max_length=128, null=True, blank=True) @@ -1444,12 +1451,10 @@ class DownloadRecord(models.Model): @property def is_finish(self): - return self.task_status == self.TASK_STATUS_FAILED or self.task_status == self.TASK_STATUS_SUCCESS @property def is_failed(self): - return self.task_status == self.TASK_STATUS_FAILED @property @@ -1939,7 +1944,7 @@ def get_host_id__bk_obj_sub_map( host_id__bk_obj_sub_map[proc_status["bk_host_id"]].append( { "bk_obj_id": proc_status["bk_obj_id"], - "subscription": exist_subscription_id__obj_map.get(int(proc_status["source_id"])) + "subscription": exist_subscription_id__obj_map.get(int(proc_status["source_id"])), # "subscription_id": int(proc_status.source_id), # "name": exist_subscription_id__obj_map.get(int(proc_status.source_id)), } @@ -2195,7 +2200,10 @@ class SubscriptionInstanceRecord(models.Model): is_latest = models.BooleanField(_("是否为实例最新记录"), default=True, db_index=True) status = models.CharField( - _("任务状态"), max_length=45, choices=constants.JobStatusType.get_choices(), default=constants.JobStatusType.PENDING + _("任务状态"), + max_length=45, + choices=constants.JobStatusType.get_choices(), + default=constants.JobStatusType.PENDING, ) @property @@ -2344,7 +2352,10 @@ class SubscriptionInstanceStatusDetail(models.Model): subscription_instance_record_id = models.BigIntegerField(_("订阅实例ID"), db_index=True) node_id = models.CharField(_("Pipeline原子ID"), max_length=50, default="", blank=True, db_index=True) status = models.CharField( - _("任务状态"), max_length=45, choices=constants.JobStatusType.get_choices(), default=constants.JobStatusType.RUNNING + _("任务状态"), + max_length=45, + choices=constants.JobStatusType.get_choices(), + default=constants.JobStatusType.RUNNING, ) log = models.TextField(_("日志内容")) update_time = models.DateTimeField(_("更新时间"), null=True, blank=True, db_index=True) diff --git a/apps/node_man/periodic_tasks/gse_svr_discovery.py b/apps/node_man/periodic_tasks/gse_svr_discovery.py index 5a437ab92..416b8ae0f 100644 --- a/apps/node_man/periodic_tasks/gse_svr_discovery.py +++ b/apps/node_man/periodic_tasks/gse_svr_discovery.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from telnetlib import Telnet -from typing import Any, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple from celery.task import periodic_task from django.conf import settings @@ -21,7 +21,6 @@ def check_ip_ports_reachable(host: str, ports: List[int]) -> bool: - for port in ports: try: with Telnet(host=host, port=port, timeout=2): @@ -33,7 +32,6 @@ def check_ip_ports_reachable(host: str, ports: List[int]) -> bool: class ZkSafeClient: - zk_client: Optional[KazooClient] def __init__(self, hosts: str, auth_data: List[Tuple[str, str]], **kwargs): @@ -110,15 +108,9 @@ def gse_svr_discovery_periodic_task(): continue logger.info(f"zk_node_path -> {zk_node_path}, svr_ips -> {svr_ips}") - inner_ip__outer_ip_map: Dict[str, str] = {} - for svr_info in getattr(ap, ap_field, []): - inner_ip__outer_ip_map[svr_info.get("inner_ip")] = svr_info.get("outer_ip") + outer_ips = getattr(ap, ap_field, []).get("outer_ips" or []) - svr_infos: List[Dict[str, Any]] = [] - for svr_ip in svr_ips: - # svr_ip 通常解析为内网IP，外网IP允许自定义，如果为空再取 svr_ip - outer_ip = inner_ip__outer_ip_map.get(svr_ip) or svr_ip - svr_infos.append({"inner_ip": svr_ip, "outer_ip": outer_ip}) + svr_infos = {"inner_ips": [{"inner_ip": inner_ip} for inner_ip in svr_ips], "outer_ips": outer_ips} setattr(ap, ap_field, svr_infos) is_change = True if is_change: diff --git a/apps/node_man/serializers/ap.py b/apps/node_man/serializers/ap.py index 9adf3fd8a..d6daffb97 100644 --- a/apps/node_man/serializers/ap.py +++ b/apps/node_man/serializers/ap.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from typing import List +import typing from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers @@ -84,28 +84,66 @@ class UpdateOrCreateSerializer(serializers.ModelSerializer): """ class ServersSerializer(serializers.Serializer): - inner_ip = serializers.CharField(label=_("内网IP"), required=False) - inner_ipv6 = serializers.CharField(label=_("内网IPv6"), required=False) - outer_ip = serializers.CharField(label=_("外网IP"), required=False) - outer_ipv6 = serializers.CharField(label=_("外网IPv6"), required=False) - bk_host_id = serializers.IntegerField(label=_("主机ID"), required=False) + class ServerInnerSerializer(serializers.Serializer): + inner_ip = serializers.CharField(label=_("内网IP"), required=False) + inner_ipv6 = serializers.CharField(label=_("内网IPv6"), required=False) + bk_host_id = serializers.IntegerField(label=_("主机ID"), required=False) + + def validate(self, attrs): + inner_ip = attrs.get("inner_ip") + inner_ipv6 = attrs.get("inner_ipv6") + if not attrs.get("inner_ip") and not attrs.get("inner_ipv6") and not attrs.get("bk_host_id"): + raise ValidationError(_("请求参数必须包含 inner_ip inner_ipv6 or bk_host_id")) + if inner_ip and not basic.is_v4(inner_ip): + raise ValidationError(_("请求参数 inner_ip 不是有效的IP地址")) + if inner_ipv6 and not basic.is_v6(inner_ipv6): + raise ValidationError(_("请求参数 inner_ipv6 不是有效的IP 地址")) + return attrs + + class ServerOuterSerializer(serializers.Serializer): + outer_ip = serializers.CharField(label=_("外网IP"), required=False) + outer_ipv6 = serializers.CharField(label=_("外网IPv6"), required=False) + bk_host_id = serializers.IntegerField(label=_("主机ID"), required=False) + + def validate(self, attrs): + outer_ip = attrs.get("outer_ip") + outer_ipv6 = attrs.get("outer_ipv6") + if not attrs.get("outer_ip") and not attrs.get("outer_ipv6") and not attrs.get("bk_host_id"): + raise ValidationError(_("请求参数必须包含 outer_ip outer_ipv6 or bk_host_id")) + if outer_ip and not basic.is_v4(outer_ip): + raise ValidationError(_("请求参数 outer_ip 不是有效的IP地址")) + if outer_ipv6 and not basic.is_v6(outer_ipv6): + raise ValidationError(_("请求参数 outer_ipv6 不是有效的IP 地址")) + return attrs + + inner_ips = serializers.ListField(label=_("内网IP信息"), required=False, child=ServerInnerSerializer()) + outer_ips = serializers.ListField(label=_("外网IP信息"), required=False, child=ServerOuterSerializer()) def validate(self, attrs): - basic.ipv6_formatter(data=attrs, ipv6_field_names=["inner_ipv6", "outer_ipv6"]) + if not attrs.get("inner_ips") and not attrs.get("outer_ips"): + raise ValidationError(_("请求参数 inner_ips, outer_ips 不可同时为空")) + + for attr in attrs.keys(): + attrs[attr] = list({frozenset(d.items()): d for d in attrs[attr]}.values()) + + # 检查是否同时包含 ipv4 和 ipv6 + for attr in attrs.keys(): + v4_attr = attr.replace("ips", "ip") + v6_attr = attr.replace("ips", "ipv6") + v4_items = [item for item in attrs[attr] if item.get(v4_attr)] + v6_items = [item for item in attrs[attr] if item.get(v6_attr)] + if v4_items and v6_items: + raise ValidationError(_(f"{attr} 中不能同时包括 {v4_attr} 和 {v6_attr}")) - if not (attrs.get("inner_ip") or attrs.get("inner_ipv6")): - raise ValidationError(_("请求参数 inner_ip 和 inner_ipv6 不能同时为空")) - if not (attrs.get("outer_ip") or attrs.get("outer_ipv6")): - raise ValidationError(_("请求参数 outer_ip 和 outer_ipv6 不能同时为空")) return attrs class ZKSerializer(serializers.Serializer): zk_ip = serializers.CharField(label=_("ZK IP地址")) zk_port = serializers.CharField(label=_("ZK 端口")) - btfileserver = serializers.ListField(child=ServersSerializer()) - dataserver = serializers.ListField(child=ServersSerializer()) - taskserver = serializers.ListField(child=ServersSerializer()) + btfileserver = ServersSerializer() + dataserver = ServersSerializer() + taskserver = ServersSerializer() zk_hosts = serializers.ListField(child=ZKSerializer()) zk_account = serializers.CharField(label=_("ZK账号"), required=False, allow_blank=True) zk_password = serializers.CharField(label=_("ZK密码"), required=False, allow_blank=True) @@ -119,7 +157,7 @@ class ZKSerializer(serializers.Serializer): callback_url = serializers.CharField(label=_("节点管理内网回调地址"), required=False, allow_blank=True) def validate(self, data): - gse_version_list: List[str] = list(set(AccessPoint.objects.values_list("gse_version", flat=True))) + gse_version_list: typing.List[str] = list(set(AccessPoint.objects.values_list("gse_version", flat=True))) # 存量接入点版本全部为V2新建/更新版本也为V2版本 if GseVersion.V1.value not in gse_version_list: data["gse_version"] = GseVersion.V2.value diff --git a/apps/node_man/tests/test_pericdic_tasks/mock_data.py b/apps/node_man/tests/test_pericdic_tasks/mock_data.py index 0ec2a3c30..1fcc2f95d 100644 --- a/apps/node_man/tests/test_pericdic_tasks/mock_data.py +++ b/apps/node_man/tests/test_pericdic_tasks/mock_data.py @@ -128,8 +128,11 @@ "/gse/config/server/task/all": ["127.0.0.1", "127.0.0.2", "127.0.0.3"], "/gse/config/server/btfiles/all": ["127.0.0.1", "127.0.0.2", "127.0.0.3"], } -MOCK_AP_FIELD_MAP = [ - {"inner_ip": "127.0.0.1", "outer_ip": "127.0.0.1"}, - {"inner_ip": "127.0.0.2", "outer_ip": "127.0.0.2"}, - {"inner_ip": "127.0.0.3", "outer_ip": "127.0.0.3"}, -] +MOCK_AP_FIELD_MAP = { + "inner_ips": [ + {"inner_ip": "127.0.0.1"}, + {"inner_ip": "127.0.0.2"}, + {"inner_ip": "127.0.0.3"}, + ], + "outer_ips": [{"outer_ip": ""}], +} diff --git a/apps/node_man/tests/test_tools/test_ap_transform.py b/apps/node_man/tests/test_tools/test_ap_transform.py new file mode 100644 index 000000000..d4c209273 --- /dev/null +++ b/apps/node_man/tests/test_tools/test_ap_transform.py @@ -0,0 +1,534 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy + +from django.core.management import call_command +from django.test import TestCase + +from apps.mock_data.common_unit.host import AP_MODEL_DATA, DEFAULT_IP, DEFAULT_IPV6 +from apps.node_man import models +from apps.utils.unittest.testcase import CustomAPITestCase + + +class TestApTransform(TestCase): + def mock_outer_ipv6(self, ipv6: str): + return DEFAULT_IPV6.replace("6", "A") + + def setUp(self): + test_data_list = [ + { + "name": "公有云接入点", + "ap_type": "system", + "region_id": "2", + "city_id": "30", + "gse_version": "V2", + "btfileserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.20", "inner_ipv6": None, "outer_ipv6": None}, + { + "inner_ip": "127.0.0.27", + "outer_ip": "127.0.0.27", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "dataserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.120", "inner_ipv6": None, "outer_ipv6": None}, + { + "inner_ip": "127.0.0.27", + "outer_ip": "127.0.0.27", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "taskserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.120", "inner_ipv6": None, "outer_ipv6": None}, + { + "inner_ip": "127.0.0.27", + "outer_ip": "127.0.0.27", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "zk_hosts": [{"zk_ip": "127.0.0.190", "zk_port": "2182"}], + "zk_account": "", + "zk_password": "", + "package_inner_url": "http://nodeman.test.com/download/prod", + "package_outer_url": "http://127.0.0.161/download/", + "nginx_path": "/data/bkee/public/bknodeman/download", + "agent_config": { + "linux": { + "dataipc": "/usr/local/gse/agent/data/ipc.state.report", + "log_path": "/var/log/gse", + "run_path": "/var/run/gse", + "data_path": "/var/lib/gse", + "temp_path": "/tmp", + "setup_path": "/usr/local/gse", + "hostid_path": "/var/lib/gse/host/hostid", + }, + "windows": { + "dataipc": 27002, + "log_path": "C:\\gse\\logs", + "run_path": "C:\\gse\\data", + "data_path": "C:\\gse\\data", + "temp_path": "C:\\Temp", + "setup_path": "C:\\gse", + "hostid_path": "C:\\gse\\data\\host\\hostid", + }, + }, + "status": None, + "description": "GSE2_上海外网", + "is_enabled": True, + "is_default": True, + "creator": ["admin"], + "port_config": { + "bt_port": 20020, + "io_port": 28668, + "data_port": 28625, + "proc_port": 50000, + "trunk_port": 48329, + "bt_port_end": 60030, + "tracker_port": 20030, + "bt_port_start": 60020, + "db_proxy_port": 58859, + "file_svr_port": 28925, + "api_server_port": 50002, + "file_svr_port_v1": 58926, + "agent_thrift_port": 48669, + "btsvr_thrift_port": 58931, + "data_prometheus_port": 29402, + "file_metric_bind_port": 29404, + "file_topology_bind_port": 28930, + }, + "proxy_package": ["gse_client-windows-x86_64.tgz", "gse_client-linux-x86_64.tgz"], + "outer_callback_url": "", + "callback_url": "", + }, + { + "name": "公有云接入点_v1", + "ap_type": "system", + "region_id": "test", + "city_id": "test", + "gse_version": "V1", + "btfileserver": [ + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.69", + "inner_ipv6": None, + "outer_ipv6": None, + }, + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.76", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "dataserver": [ + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.69", + "inner_ipv6": None, + "outer_ipv6": None, + }, + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.76", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "taskserver": [ + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.69", + "inner_ipv6": None, + "outer_ipv6": None, + }, + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.76", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "zk_hosts": [{"zk_ip": "127.0.0.1", "zk_port": "2182"}], + "zk_account": "noneed", + "zk_password": "noneed", + "package_inner_url": "http://nodeman.test.com/download/prod-oa", + "package_outer_url": "http://127.0.0.1/download/prod-oa", + "nginx_path": "/data/bkee/public/bknodeman/download/prod-oa", + "agent_config": { + "linux": { + "dataipc": "/usr/local/gse/agent/data/ipc.state.report", + "log_path": "/var/log/gse", + "run_path": "/var/run/gse", + "data_path": "/var/lib/gse", + "temp_path": "/tmp", + "setup_path": "/usr/local/gse", + "hostid_path": "/var/lib/gse/host/hostid", + }, + "windows": { + "dataipc": "47000", + "log_path": "C:\\gse\\logs", + "run_path": "C:\\gse\\data", + "data_path": "C:\\gse\\data", + "temp_path": "C:\\Temp", + "setup_path": "C:\\gse", + "hostid_path": "C:\\gse\\data\\host\\hostid", + }, + }, + "status": None, + "description": "专用Proxy请勿选择", + "is_enabled": True, + "is_default": False, + "creator": ["admin"], + "port_config": { + "bt_port": 10020, + "io_port": 48668, + "data_port": 58625, + "proc_port": 50000, + "trunk_port": 48329, + "bt_port_end": 60030, + "tracker_port": 10030, + "bt_port_start": 60020, + "db_proxy_port": 58859, + "file_svr_port": 58925, + "api_server_port": 50002, + "agent_thrift_port": 48669, + "btsvr_thrift_port": 58930, + "data_prometheus_port": 59402, + }, + "proxy_package": [ + "gse_client-windows-x86_64.tgz", + "gse_client-linux-x86_64.tgz", + "gse_client-linux-aarch64.tgz", + ], + "outer_callback_url": "", + "callback_url": "", + }, + { + "name": "GSE2_IPV6", + "ap_type": "system", + "region_id": "2", + "city_id": "30", + "gse_version": "V2", + "btfileserver": [ + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + ], + "dataserver": [ + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + ], + "taskserver": [ + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + ], + "zk_hosts": [{"zk_ip": "127.0.0.1", "zk_port": "2182"}], + "zk_account": "noneed", + "zk_password": "noneed", + "package_inner_url": "http://nodeman.test.com/download/prod-oa", + "package_outer_url": "http://127.0.0.1/download/", + "nginx_path": "/data/bkee/public/bknodeman/download/", + "agent_config": { + "linux": { + "dataipc": "/usr/local/gse/agent/data/ipc.state.report", + "log_path": "/var/log/gse", + "run_path": "/var/run/gse", + "data_path": "/var/lib/gse", + "temp_path": "/tmp", + "setup_path": "/usr/local/gse", + "hostid_path": "/var/lib/gse/host/hostid", + }, + "windows": { + "dataipc": 27002, + "log_path": "C:\\gse\\logs", + "run_path": "C:\\gse\\data", + "data_path": "C:\\gse\\data", + "temp_path": "C:\\Temp", + "setup_path": "C:\\gse", + "hostid_path": "C:\\gse\\data\\host\\hostid", + }, + }, + "status": None, + "description": "专用Proxy请勿选择", + "is_enabled": True, + "is_default": False, + "creator": ["admin"], + "port_config": { + "bt_port": 20020, + "io_port": 28668, + "data_port": 28625, + "proc_port": 50000, + "trunk_port": 48329, + "bt_port_end": 60030, + "tracker_port": 20030, + "bt_port_start": 60020, + "db_proxy_port": 58859, + "file_svr_port": 28925, + "api_server_port": 50002, + "file_svr_port_v1": 58926, + "agent_thrift_port": 48669, + "btsvr_thrift_port": 58931, + "file_metric_bind_port": 29404, + "file_topology_bind_port": 28930, + }, + "proxy_package": ["gse_client-windows-x86_64.tgz", "gse_client-linux-x86_64.tgz"], + "outer_callback_url": "", + "callback_url": "", + }, + ] + + for ap_data in test_data_list: + models.AccessPoint.objects.update_or_create(**ap_data) + + def test_ap_transform(self): + from apps.node_man.utils.endpoint import EndPointTransform + from env.constants import GseVersion + + gse_v1_ap = models.AccessPoint.objects.get(gse_version=GseVersion.V1.value, name="公有云接入点_v1") + self.assertEqual( + EndPointTransform().transform(gse_v1_ap.taskserver), + { + "inner_ips": [{"inner_ip": "127.0.0.198"}], + "outer_ips": [{"outer_ip": "127.0.0.69"}, {"outer_ip": "127.0.0.76"}], + }, + ) + + gse_v2_ap = models.AccessPoint.objects.get(gse_version=GseVersion.V2.value, name="公有云接入点") + self.assertEqual( + EndPointTransform().transform(gse_v2_ap.btfileserver), + { + "inner_ips": [{"inner_ip": "127.0.0.120"}, {"inner_ip": "127.0.0.27"}], + "outer_ips": [{"outer_ip": "127.0.0.20"}, {"outer_ip": "127.0.0.27"}], + }, + ) + + def test_transfrom_command(self): + # 调用 django command transform_ap_data 进行数据转换 + default_ap_id = models.AccessPoint.objects.get(name="默认接入点").id + # 因为默认的接入点已经经过转换，所以这里需要先把默认接入点的数据转换为旧的格式 + call_command("transform_ap_data", transform_endpoint_to_leagcy=True, transform_ap_id=default_ap_id) + default_ap_obj = models.AccessPoint.objects.get(name="默认接入点") + self.assertEqual( + default_ap_obj.taskserver, + [ + { + "inner_ip": "", + "outer_ip": "", + } + ], + ) + # 把所有的接入点都转换一遍， 转换为新的格式 + call_command("transform_ap_data", transform=True, all_ap=True) + for ap in models.AccessPoint.objects.all(): + self.assertTrue(isinstance(ap.taskserver, dict)) + + gse_v1_ap = models.AccessPoint.objects.get(gse_version="V1", name="公有云接入点_v1") + self.assertEqual( + gse_v1_ap.btfileserver, + { + "inner_ips": [{"inner_ip": "127.0.0.198"}], + "outer_ips": [{"outer_ip": "127.0.0.69"}, {"outer_ip": "127.0.0.76"}], + }, + ) + v6_ap_obj = models.AccessPoint.objects.get(name="GSE2_IPV6") + self.assertEqual( + v6_ap_obj.btfileserver, + { + "inner_ips": [{"inner_ip": "", "inner_ipv6": DEFAULT_IPV6}], + "outer_ips": [{"outer_ip": "", "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6)}], + }, + ) + + # 转换回旧的数据 + call_command("transform_ap_data", transform_endpoint_to_leagcy=True, transform_ap_id=gse_v1_ap.id) + self.assertEqual( + # 转换回旧的数据，并且过滤为 None 的字段 + models.AccessPoint.objects.get(id=gse_v1_ap.id).btfileserver, + [ + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.69", + }, + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.76", + }, + ], + ) + call_command("transform_ap_data", transform=True, transform_ap_id=gse_v1_ap.id) + self.assertEqual( + models.AccessPoint.objects.get(id=gse_v1_ap.id).btfileserver, + { + "inner_ips": [{"inner_ip": "127.0.0.198"}], + "outer_ips": [{"outer_ip": "127.0.0.69"}, {"outer_ip": "127.0.0.76"}], + }, + ) + + # 校验参数 + self.assertRaises( + ValueError, call_command, "transform_ap_data", transform_endpoint_to_leagcy=True, transform=True + ) + self.assertRaises( + ValueError, call_command, "transform_ap_data", transform_endpoint_to_leagcy=True, transform=False + ) + self.assertRaises( + ValueError, + call_command, + "transform_ap_data", + transform_endpoint_to_leagcy=True, + transform=False, + all_ap=True, + transform_ap_id=gse_v1_ap.id, + ) + + +class ApViewTransformTestCase(CustomAPITestCase): + TEST_AP_NAME = "CUSTOM_AP" + CREATE_URL = "/api/ap/" + + def setUp(self): + ap_data = deepcopy(AP_MODEL_DATA) + ap_data["name"] = self.TEST_AP_NAME + self.ap_data = ap_data + + def test_ap_create(self): + self.client.post(path=self.CREATE_URL, data=self.ap_data) + ap = models.AccessPoint.objects.get(name=self.TEST_AP_NAME) + self.assertEqual(ap.name, self.TEST_AP_NAME) + self.assertEqual(ap.taskserver, AP_MODEL_DATA["taskserver"]) + + def test_mix_ip_ap_create(self): + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ip": DEFAULT_IP}, {"inner_ipv6": DEFAULT_IPV6}]} + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertFalse(result["result"]) + self.assertEqual(result["message"], "inner_ips 中不能同时包括 inner_ip 和 inner_ipv6（3800001）") + + # 支持 taskserver v6 & fileserver v4 混合 + mix_server_ap_data = deepcopy(self.ap_data) + mix_server_ap_data["taskserver"] = {"inner_ips": [{"inner_ipv6": DEFAULT_IPV6}]} + result = self.client.post(self.CREATE_URL, mix_server_ap_data) + self.assertTrue(result["result"]) + + def test_multi_v4_ap_create(self): + # 支持多个 v4 地址 + self.ap_data["taskserver"] = { + "inner_ips": [{"inner_ip": DEFAULT_IP}, {"inner_ip": DEFAULT_IP.replace("1", "2")}], + "outer_ips": [{"outer_ip": DEFAULT_IP}, {"outer_ip": DEFAULT_IP.replace("1", "2")}], + } + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertTrue(result["result"]) + ap = models.AccessPoint.objects.get(name=self.TEST_AP_NAME) + self.assertEqual( + ap.taskserver, + { + "inner_ips": [{"inner_ip": DEFAULT_IP}, {"inner_ip": DEFAULT_IP.replace("1", "2")}], + "outer_ips": [{"outer_ip": DEFAULT_IP}, {"outer_ip": DEFAULT_IP.replace("1", "2")}], + }, + ) + + def test_multi_v6_ap_create(self): + # 支持多个 v6 地址 + self.ap_data["taskserver"] = { + "inner_ips": [{"inner_ipv6": DEFAULT_IPV6}, {"inner_ipv6": DEFAULT_IPV6.replace("1", "2")}] + } + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertTrue(result["result"]) + + def test_v4_v6_mix_ap_create(self): + # 支持同一个 server inner & outer v4 v6 混合 + self.ap_data["taskserver"] = { + "inner_ips": [{"inner_ip": DEFAULT_IP}], + "outer_ips": [{"outer_ipv6": DEFAULT_IPV6}], + } + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertTrue(result["result"]) + ap = models.AccessPoint.objects.get(name=self.TEST_AP_NAME) + self.assertEqual( + ap.taskserver, + { + "inner_ips": [{"inner_ip": DEFAULT_IP}], + "outer_ips": [{"outer_ipv6": DEFAULT_IPV6}], + }, + ) + + def test_filter_ip_ap_crete(self): + # 支持过滤掉重复 ip + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ip": DEFAULT_IP}, {"inner_ip": DEFAULT_IP}]} + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertTrue(result["result"]) + self.assertEqual( + result["data"]["taskserver"], + { + "inner_ips": [{"inner_ip": DEFAULT_IP}], + }, + ) + + def test_illegal_ip_ap_create(self): + # 支持 v4 / v6 IP格式检测 + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ip": "11"}]} + self.assertFalse(self.client.post(self.CREATE_URL, self.ap_data)["result"]) + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ip": DEFAULT_IP}, {"inner_ip": "11"}]} + self.assertFalse(self.client.post(self.CREATE_URL, self.ap_data)["result"]) + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ipv6": DEFAULT_IP}]} + self.assertFalse(self.client.post(self.CREATE_URL, self.ap_data)["result"]) + + def test_update_ap(self): + update_ap_name = "默认update ap" + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ip": "111.1.1.1"}]} + self.ap_data["name"] = "默认update ap" + ap_id = models.AccessPoint.objects.get(name="默认接入点").id + update_url = f"/api/ap/{ap_id}/" + result = self.client.put(update_url, self.ap_data) + self.assertTrue(result["result"]) + ap_obj = models.AccessPoint.objects.get(name=update_ap_name) + self.assertEqual(ap_obj.taskserver, {"inner_ips": [{"inner_ip": "111.1.1.1"}]}) + + def test_ap_retrieve(self): + self.test_ap_create() + ap_id = models.AccessPoint.objects.get(name=self.TEST_AP_NAME).id + retrieve_url = f"/api/ap/{ap_id}/" + result = self.client.get(retrieve_url) + self.assertTrue(result["result"]) + self.assertEqual(result["data"]["name"], self.TEST_AP_NAME) + self.assertEqual(result["data"]["taskserver"], AP_MODEL_DATA["taskserver"]) diff --git a/apps/node_man/tests/utils.py b/apps/node_man/tests/utils.py index 0e950a3bc..200a0c303 100644 --- a/apps/node_man/tests/utils.py +++ b/apps/node_man/tests/utils.py @@ -191,7 +191,6 @@ def create_host( class Subscription: def create_subscription(self, job_type, nodes, *args, **kwargs): - cipher = tools.HostTools.get_asymmetric_cipher() subscription_id = random.randint(100, 1000) task_id = random.randint(10, 1000) @@ -654,9 +653,12 @@ def create_ap(number): "is_enabled": 1, "zk_account": "bkzk", "zk_password": "bkzk", - "btfileserver": [{"inner_ip": random_ip(), "outer_ip": random_ip()}], - "dataserver": [{"inner_ip": random_ip(), "outer_ip": random_ip()}], - "taskserver": [{"inner_ip": random_ip(), "outer_ip": random_ip()}], + "btfileserver": { + "inner_ips": [{"inner_ip": random_ip()}], + "outer_ips": [{"outer_ip": random_ip()}], + }, + "dataserver": {"inner_ips": [{"inner_ip": random_ip()}], "outer_ips": [{"outer_ip": random_ip()}]}, + "taskserver": {"inner_ips": [{"inner_ip": random_ip()}], "outer_ips": [{"outer_ip": random_ip()}]}, "package_inner_url": "http://127.0.0.1:80/download", "package_outer_url": "http://127.0.0.1:80/download", "nginx_path": "/data/bkee/public/bknodeman/download", diff --git a/apps/node_man/utils/endpoint.py b/apps/node_man/utils/endpoint.py index 62e8757f4..5d3eacc10 100644 --- a/apps/node_man/utils/endpoint.py +++ b/apps/node_man/utils/endpoint.py @@ -9,9 +9,14 @@ specific language governing permissions and limitations under the License. """ +import json import typing +from collections import defaultdict from dataclasses import dataclass, field +from apps.utils.basic import filter_values +from apps.utils.md5 import _count_md5 + @dataclass class Endpoint: @@ -27,31 +32,121 @@ def __post_init__(self): class EndpointInfo: def __init__( self, - inner_server_infos: typing.List[typing.Dict[str, typing.Any]], - outer_server_infos: typing.List[typing.Dict[str, typing.Any]], + endpoints: typing.Dict[str, typing.List[typing.Dict[str, typing.Union[str, int]]]], ): self.inner_endpoints: typing.List[Endpoint] = [] self.outer_endpoints: typing.List[Endpoint] = [] # shortcut self.outer_hosts: typing.List[str] = [] self.inner_hosts: typing.List[str] = [] + from apps.node_man.models import AccessPoint + + if isinstance(endpoints, dict): + raise ValueError("endpoints must be dict") + + for endpoint_type, endpoint_info_list in endpoints.items(): + for endpoint_info in endpoint_info_list: + if endpoint_type == AccessPoint.ServersType.INNER_IPS: + endpoint: Endpoint = Endpoint( + v4=endpoint_info.get("inner_ip"), + v6=endpoint_info.get("inner_ipv6"), + host_id=endpoint_info.get("bk_host_id"), + ) + self.inner_endpoints.append(endpoint) + if endpoint.host_str: + self.inner_hosts.append(endpoint.host_str) + elif endpoint_type == AccessPoint.ServersType.OUTER_IPS: + endpoint: Endpoint = Endpoint( + v4=endpoint_info.get("outer_ip"), + v6=endpoint_info.get("outer_ipv6"), + host_id=endpoint_info.get("bk_host_id"), + ) + self.outer_endpoints.append(endpoint) + if endpoint.host_str: + self.outer_hosts.append(endpoint.host_str) + - for inner_server_info in inner_server_infos: - endpoint: Endpoint = Endpoint( - v4=inner_server_info.get("inner_ip"), - v6=inner_server_info.get("inner_ipv6"), - host_id=inner_server_info.get("host_id"), +class EndPointTransform(object): + # legacy_endpoint: [{ "inner_ip": "10.0.6.44", "outer_ip": "10.0.6.44"}, {"inner_ip": "xxx", "outer_ip": "xxx"}] + # endpoint: { + # "inner_ips": [{"ip": "", "bk_host_id": x], + # "outer_ips": [{"ip": "", "bk_host_id": x] + # } + + INNER_IPS = "inner_ips" + OUTER_IPS = "outer_ips" + + def transform( + self, legacy_endpoints: typing.List[typing.Dict[str, typing.Any]] + ) -> typing.Dict[str, typing.List[typing.Dict[str, typing.Union[str, int]]]]: + endpoints = defaultdict(list) + if not isinstance(legacy_endpoints, list): + raise ValueError("legacy_endpoints must be list") + for legacy_endpoint in legacy_endpoints: + endpoints[self.INNER_IPS].append( + { + "inner_ip": legacy_endpoint.get("inner_ip"), + "bk_host_id": legacy_endpoint.get("bk_host_id"), + "inner_ipv6": legacy_endpoint.get("inner_ipv6"), + } ) - self.inner_endpoints.append(endpoint) - if endpoint.host_str: - self.inner_hosts.append(endpoint.host_str) - - for outer_server_info in outer_server_infos: - endpoint: Endpoint = Endpoint( - v4=outer_server_info.get("outer_ip"), - v6=outer_server_info.get("outer_ipv6"), - host_id=outer_server_info.get("host_id"), + endpoints[self.OUTER_IPS].append( + { + "outer_ip": legacy_endpoint.get("outer_ip"), + "bk_host_id": legacy_endpoint.get("bk_host_id"), + "outer_ipv6": legacy_endpoint.get("outer_ipv6"), + } ) - self.outer_endpoints.append(endpoint) - if endpoint.host_str: - self.outer_hosts.append(endpoint.host_str) + # 把 endpoints 的 values 包括的字典中的空字段去掉 并且去重 + unique_endpoints = defaultdict(list) + seen = set() + for endpoint_type in endpoints: + for endpoint in endpoints[endpoint_type]: + hash_value = _count_md5(json.dumps(endpoint)) + if hash_value not in seen: + unique_endpoints[endpoint_type].append(filter_values(endpoint)) + seen.add(hash_value) + return dict(unique_endpoints) + + def transform_endpoint_to_leagcy( + self, + endpoints: typing.Dict[str, typing.List[typing.Dict[str, typing.Union[str, int]]]], + ) -> typing.List[typing.Dict[str, typing.Any]]: + from apps.node_man.models import AccessPoint + + legacy_endpoints = [] + if not isinstance(endpoints, dict): + raise ValueError("endpoints must be dict") + + # endpoints 的 key 必须为这两个值，无序的 ["inner_ips", "outer_ips"]: + for endpoint_type in endpoints: + if endpoint_type not in [self.INNER_IPS, self.OUTER_IPS]: + raise ValueError( + f"endpoints key must be in {AccessPoint.ServersType.INNER_IPS}," + f"or {AccessPoint.ServersType.OUTER_IPS}" + ) + if not isinstance(endpoints[endpoint_type], list): + raise ValueError("endpoints value must be list") + + legacy_endpoints = [ + { + "inner_ip": inner_info.get("inner_ip"), + "outer_ip": outer_info.get("outer_ip"), + "inner_ipv6": inner_info.get("inner_ipv6"), + "outer_ipv6": outer_info.get("outer_ipv6"), + "bk_host_id": inner_info.get("bk_host_id"), + } + for inner_info in endpoints.get(self.INNER_IPS, []) + for outer_info in endpoints.get(self.OUTER_IPS, []) + ] + + # 把 legacy_endpoints 通过字典的 md5 去重 + seen = set() + unique_endpoints = [] + for endpoint in legacy_endpoints: + hash_value = _count_md5(json.dumps(endpoint)) + if hash_value not in seen: + unique_endpoints.append(filter_values(endpoint)) + seen.add(hash_value) + + return unique_endpoints diff --git a/apps/node_man/views/ap.py b/apps/node_man/views/ap.py index dca9ca9f4..4aeee94f1 100644 --- a/apps/node_man/views/ap.py +++ b/apps/node_man/views/ap.py @@ -57,12 +57,18 @@ def list(self, request, *args, **kwargs): ] "zk_user": "username", "zk_password": "zk_password", - "servers": [ - { - "inner_ip": "127.0.0.1", - "outer_ip": "127.0.0.2" - } - ], + "btfileserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "taskserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "dataserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, "package_inner_url": "http://127.0.0.1/download/", "package_outer_url": "http://127.0.0.2/download/", "agent_config": { @@ -125,12 +131,18 @@ def retrieve(self, request, *args, **kwargs): ] "zk_user": "username", "zk_password": "zk_password", - "servers": [ - { - "inner_ip": "127.0.0.1", - "outer_ip": "127.0.0.2" - } - ], + "btfileserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "taskserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "dataserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, "package_inner_url": "http://127.0.0.1/download/", "package_outer_url": "http://127.0.0.2/download/", "agent_config": { @@ -306,12 +318,18 @@ def update(self, request, *args, **kwargs): @apiParamExample {Json} 请求参数 { "name": "接入点名称", - "servers": [ - { - "inner_ip": "127.0.0.1", - "outer_ip": "127.0.0.2" - } - ], + "btfileserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "taskserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "dataserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, "package_inner_url": "http://127.0.0.1/download/", "package_outer_url": "http://127.0.0.2/download/", "agent_config": {