From e669fde9a4d14704e58ae6e16636ce9da8ddf188 Mon Sep 17 00:00:00 2001 From: dcd <1151627903@qq.com> Date: Wed, 5 Jun 2024 15:02:20 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20PROXY=E5=AE=89=E8=A3=85=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=A4=9A=E5=A4=96=E7=BD=91IP=20(closed=20#2198)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent_new/add_or_update_hosts.py | 5 ++ .../collections/agent_new/configure_policy.py | 10 +++- .../agent_new/test_add_or_update_hosts.py | 36 ++++++++++++++ .../agent_new/test_configure_policy.py | 33 +++++++++++++ apps/node_man/handlers/host.py | 5 ++ .../node_man/periodic_tasks/sync_cmdb_host.py | 20 ++++++++ apps/node_man/serializers/host.py | 10 +++- apps/node_man/serializers/job.py | 9 +++- .../node_man/tests/test_handlers/test_host.py | 25 ++++++++++ .../test_sync_cmdb_host.py | 48 ++++++++++++++++++- 10 files changed, 196 insertions(+), 5 deletions(-) diff --git a/apps/backend/components/collections/agent_new/add_or_update_hosts.py b/apps/backend/components/collections/agent_new/add_or_update_hosts.py index e0d31c91f..6e29390be 100644 --- a/apps/backend/components/collections/agent_new/add_or_update_hosts.py +++ b/apps/backend/components/collections/agent_new/add_or_update_hosts.py @@ -435,6 +435,11 @@ def handle_update_db(self, sub_insts: List[models.SubscriptionInstanceRecord]): } if host_info.get("data_path"): extra_data.update({"data_path": host_info.get("data_path")}) + outer_ip_list = outer_ip.split(",") + if len(outer_ip_list) > 1 and host_info["host_node_type"] == constants.NodeType.PROXY: + extra_data.update({"bk_host_multi_outerip": outer_ip}) + outer_ip = outer_ip_list[0] + host_obj = models.Host( bk_host_id=bk_host_id, bk_cloud_id=host_info["bk_cloud_id"], diff --git a/apps/backend/components/collections/agent_new/configure_policy.py b/apps/backend/components/collections/agent_new/configure_policy.py index b76fecc86..70ea7e929 100644 --- a/apps/backend/components/collections/agent_new/configure_policy.py +++ b/apps/backend/components/collections/agent_new/configure_policy.py @@ -13,7 +13,7 @@ from apps.backend.api.constants import POLLING_INTERVAL from apps.backend.components.collections.agent_new.base import AgentBaseService -from apps.node_man import models +from apps.node_man import constants, models from apps.node_man.handlers.security_group import get_security_group_factory from pipeline.core.flow import StaticIntervalGenerator @@ -34,7 +34,13 @@ def _execute(self, data, parent_data, common_data): security_group_factory = get_security_group_factory(security_group_type) ip_list = [] for host in host_id_obj_map.values(): - ip_list.extend([host.outer_ip, host.login_ip]) + bk_host_multi_outerip = host.extra_data.get("bk_host_multi_outerip", "") + if bk_host_multi_outerip and host.node_type == constants.NodeType.PROXY: + outer_ip_list = bk_host_multi_outerip.split(",") + ip_list.extend(outer_ip_list) + ip_list.append(host.login_ip) + else: + ip_list.extend([host.outer_ip, host.login_ip]) # 不同的安全组工厂添加策略后得到的输出可能是不同的，输出到outputs中，在schedule中由工厂对应的check_result方法来校验结果 creator: str = common_data.subscription.creator data.outputs.add_ip_output = security_group_factory.add_ips_to_security_group(ip_list, creator=creator) diff --git a/apps/backend/tests/components/collections/agent_new/test_add_or_update_hosts.py b/apps/backend/tests/components/collections/agent_new/test_add_or_update_hosts.py index 4ef65490e..40397ecab 100644 --- a/apps/backend/tests/components/collections/agent_new/test_add_or_update_hosts.py +++ b/apps/backend/tests/components/collections/agent_new/test_add_or_update_hosts.py @@ -348,3 +348,39 @@ class UpdateOldWhenIncludeLoginInParamTestCase(AddOrUpdateHostsTestCase): def assert_in_teardown(self): super().assert_in_teardown() self.assertEqual(models.Host.objects.filter(login_ip="").count(), 0) + + +class MultiOuterIpHostsTestCase(AddOrUpdateHostsTestCase): + @classmethod + def adjust_test_data_in_db(cls): + super().adjust_test_data_in_db() + + # 多外网IP情况 + for sub_inst_obj in cls.obj_factory.sub_inst_record_objs: + sub_inst_obj.instance_info["host"]["bk_host_outerip"] += ",1.2.3.4" + sub_inst_obj.instance_info["host"]["host_node_type"] = constants.NodeType.PROXY + models.SubscriptionInstanceRecord.objects.bulk_update( + cls.obj_factory.sub_inst_record_objs, fields=["instance_info"] + ) + + def assert_in_teardown(self): + # 更新主机情况 + for host_info in self.cmdb_mock_client.batch_update_host.call_args[0][0]["update"]: + bk_host_outerip = host_info["properties"]["bk_host_outerip"] + outer_ips = bk_host_outerip.split(",") + self.assertEqual(len(outer_ips), 2) + # 新增主机情况 + for bk_host_info in self.cmdb_mock_client.add_host_to_business_idle.call_args[0][0]["bk_host_list"]: + bk_host_outerip = bk_host_info["bk_host_outerip"] + outer_ips = bk_host_outerip.split(",") + self.assertEqual(len(outer_ips), 2) + # 更新DB情况 + host_datas = models.Host.objects.filter(bk_cloud_id=0)[:2].values("outer_ip", "extra_data") + for host in host_datas: + outer_ip = host["outer_ip"].split(",") + self.assertEqual(len(outer_ip), 1) + bk_host_multi_outerip = host["extra_data"]["bk_host_multi_outerip"] + outer_ips = bk_host_multi_outerip.split(",") + self.assertEqual(len(outer_ips), 2) + + super().assert_in_teardown() diff --git a/apps/backend/tests/components/collections/agent_new/test_configure_policy.py b/apps/backend/tests/components/collections/agent_new/test_configure_policy.py index 629bd88f7..e31d83a1c 100644 --- a/apps/backend/tests/components/collections/agent_new/test_configure_policy.py +++ b/apps/backend/tests/components/collections/agent_new/test_configure_policy.py @@ -303,3 +303,36 @@ def cases(self): execute_call_assertion=None, ), ] + + +class YunTiConfigureMultiOuterIpPolicyTestCase(YunTiConfigurePolicyComponentBaseTest): + def cases(self): + outer_ip = self.obj_factory.host_objs[0].outer_ip + login_ip = self.obj_factory.host_objs[0].login_ip + return [ + ComponentTestCase( + name="通过云梯配置多外网IP策略", + inputs=self.common_inputs, + parent_data={}, + execute_assertion=ExecuteAssertion( + success=bool(self.common_inputs["subscription_instance_ids"]), + outputs={ + "add_ip_output": {"ip_list": [outer_ip, login_ip]}, + "polling_time": 0, + "succeeded_subscription_instance_ids": self.common_inputs["subscription_instance_ids"], + }, + ), + schedule_assertion=[ + ScheduleAssertion( + success=True, + schedule_finished=True, + outputs={ + "add_ip_output": {"ip_list": [outer_ip, login_ip]}, + "polling_time": 0, + "succeeded_subscription_instance_ids": self.common_inputs["subscription_instance_ids"], + }, + ), + ], + execute_call_assertion=None, + ), + ] diff --git a/apps/node_man/handlers/host.py b/apps/node_man/handlers/host.py index c6ed507bd..b32cacdf2 100644 --- a/apps/node_man/handlers/host.py +++ b/apps/node_man/handlers/host.py @@ -347,6 +347,7 @@ def proxies(bk_cloud_id: int): proxy["re_certification"] = host_id_identities.get(proxy["bk_host_id"], {}).get("re_certification", "") proxy["job_result"] = host_id_job_status.get(proxy["bk_host_id"], {}) proxy["pagent_count"] = pagent_upstream_nodes.get(proxy["inner_ip"], 0) + # proxy["bk_host_multi_outerip"] = proxy["extra_data"].get("bk_host_multi_outerip", "") return proxies @@ -488,6 +489,10 @@ def update_proxy_info(params: dict): ).exists() ): CmdbHandler().cmdb_update_host(kwargs["bk_host_id"], update_properties) + outer_ips = kwargs.get("outer_ip", "").split(",") + if len(outer_ips) > 1: + extra_data["bk_host_multi_outerip"] = kwargs["outer_ip"] + kwargs["outer_ip"] = outer_ips[0] # 如果需要更新Host的Cloud信息 if ( diff --git a/apps/node_man/periodic_tasks/sync_cmdb_host.py b/apps/node_man/periodic_tasks/sync_cmdb_host.py index 96ace2c75..cf04b0177 100644 --- a/apps/node_man/periodic_tasks/sync_cmdb_host.py +++ b/apps/node_man/periodic_tasks/sync_cmdb_host.py @@ -243,6 +243,8 @@ def update_or_create_host_base(biz_id, ap_map_config, is_gse2_gray, task_id, cmd need_update_hosts: typing.List[models.Host] = [] need_update_hosts_with_arch: typing.List[models.Host] = [] + host_id__multi_outer_ip_map: typing.Dict[int, str] = {} + multi_outer_ip_host_ids: typing.List[int] = [] need_create_hosts: typing.List[models.Host] = [] need_create_host_identity_objs: typing.List[models.IdentityData] = [] @@ -275,6 +277,12 @@ def update_or_create_host_base(biz_id, ap_map_config, is_gse2_gray, task_id, cmd "inner_ipv6": (host.get("bk_host_innerip_v6") or "").split(",")[0], "outer_ipv6": (host.get("bk_host_outerip_v6") or "").split(",")[0], } + outer_ip = host.get("bk_host_outerip") or "" + outer_ip_list = outer_ip.split(",") + if len(outer_ip_list) > 1 and host["bk_host_id"] in exist_proxy_host_ids: + bk_host_id = host["bk_host_id"] + multi_outer_ip_host_ids.append(bk_host_id) + host_id__multi_outer_ip_map[bk_host_id] = outer_ip if host["bk_host_id"] in exist_agent_host_ids: host_params["os_type"] = tools.HostV2Tools.get_os_type(host, is_os_type_priority) @@ -346,6 +354,18 @@ def update_or_create_host_base(biz_id, ap_map_config, is_gse2_gray, task_id, cmd ) if need_create_process_status_objs: models.ProcessStatus.objects.bulk_create(need_create_process_status_objs, batch_size=500) + if multi_outer_ip_host_ids: + host_queryset = models.Host.objects.filter(bk_host_id__in=multi_outer_ip_host_ids) + need_update_hosts_with_extra_data: typing.List[models.Host] = [] + for host_obj in host_queryset: + host_obj.extra_data["bk_host_multi_outerip"] = host_id__multi_outer_ip_map[host_obj.bk_host_id] + need_update_hosts_with_extra_data.append(host_obj) + models.Host.objects.bulk_update(need_update_hosts_with_extra_data, fields=["extra_data"], batch_size=500) + logger.info( + f"[sync_cmdb_host] update_or_create_host: task_id -> {task_id}, bk_biz_id -> {biz_id}, " + f"bk_host_ids -> {multi_outer_ip_host_ids} update proxy host_extra_data success, " + f"len -> {len(multi_outer_ip_host_ids)}" + ) return bk_host_ids diff --git a/apps/node_man/serializers/host.py b/apps/node_man/serializers/host.py index b5d2e3695..4b70b4c4a 100644 --- a/apps/node_man/serializers/host.py +++ b/apps/node_man/serializers/host.py @@ -8,6 +8,8 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from ipaddress import IPv4Address + from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers @@ -37,7 +39,7 @@ class HostUpdateSerializer(serializers.Serializer): bk_cloud_id = serializers.IntegerField(label=_("管控区域ID"), required=False) inner_ip = serializers.IPAddressField(label=_("内网IP"), required=False, protocol="ipv4") inner_ipv6 = serializers.IPAddressField(label=_("内网IPv6"), required=False, protocol="ipv6") - outer_ip = serializers.IPAddressField(label=_("外网IP"), required=False, protocol="ipv4") + outer_ip = serializers.CharField(label=_("外网IP"), required=False) outer_ipv6 = serializers.IPAddressField(label=_("外网IPv6"), required=False, protocol="ipv6") # 登录 IP & 数据 IP 支持多 IP 协议 login_ip = serializers.IPAddressField(label=_("登录IP"), required=False, protocol="both") @@ -63,6 +65,12 @@ def validate(self, attrs): cipher=cipher, encrypt_message=attrs[field_need_encrypt], raise_exec=ValidationError ) basic.ipv6_formatter(data=attrs, ipv6_field_names=["inner_ipv6", "outer_ipv6", "login_ip", "data_ip"]) + if not attrs.get("outer_ipv6") and attrs.get("outer_ip"): + outer_ips = attrs["outer_ip"].split(",") + try: + [IPv4Address(outer_ip) for outer_ip in outer_ips] + except ValueError: + raise ValidationError(_("更新Proxy信息请求参数 outer_ip:请输入一个合法的IPv4地址")) return attrs diff --git a/apps/node_man/serializers/job.py b/apps/node_man/serializers/job.py index ad96adb8c..dba4db497 100644 --- a/apps/node_man/serializers/job.py +++ b/apps/node_man/serializers/job.py @@ -10,6 +10,7 @@ """ import typing from collections import defaultdict +from ipaddress import IPv4Address from django.conf import settings from django.db.models import Q @@ -171,7 +172,7 @@ class HostSerializer(InstallBaseSerializer): ap_id = serializers.IntegerField(label=_("接入点ID"), required=False) install_channel_id = serializers.IntegerField(label=_("安装通道ID"), required=False, allow_null=True) inner_ip = serializers.IPAddressField(label=_("内网IP"), required=False, allow_blank=True, protocol="ipv4") - outer_ip = serializers.IPAddressField(label=_("外网IP"), required=False, allow_blank=True, protocol="ipv4") + outer_ip = serializers.CharField(label=_("外网IP"), required=False, allow_blank=True) login_ip = serializers.IPAddressField(label=_("登录IP"), required=False, allow_blank=True, protocol="both") data_ip = serializers.IPAddressField(label=_("数据IP"), required=False, allow_blank=True, protocol="both") inner_ipv6 = serializers.IPAddressField(label=_("内网IPv6"), required=False, allow_blank=True, protocol="ipv6") @@ -203,6 +204,12 @@ def validate(self, attrs): raise ValidationError(_("请求参数 inner_ip 和 inner_ipv6 不能同时为空")) if node_type == constants.NodeType.PROXY and not (attrs.get("outer_ip") or attrs.get("outer_ipv6")): raise ValidationError(_("Proxy 操作的请求参数 outer_ip 和 outer_ipv6 不能同时为空")) + if not attrs.get("outer_ipv6") and attrs.get("outer_ip"): + outer_ips = attrs["outer_ip"].split(",") + try: + [IPv4Address(outer_ip) for outer_ip in outer_ips] + except ValueError: + raise ValidationError(_("Proxy 操作的请求参数 outer_ip:请输入一个合法的IPv4地址")) basic.ipv6_formatter(data=attrs, ipv6_field_names=["inner_ipv6", "outer_ipv6", "login_ip", "data_ip"]) diff --git a/apps/node_man/tests/test_handlers/test_host.py b/apps/node_man/tests/test_handlers/test_host.py index c3acd148f..0d11f97e0 100644 --- a/apps/node_man/tests/test_handlers/test_host.py +++ b/apps/node_man/tests/test_handlers/test_host.py @@ -666,3 +666,28 @@ def test_export_cloud_area_colon_ip(self): } res = HostHandler().list(params, "admin") self.assertEqual(len(res["list"]), 0) + + @patch("apps.node_man.handlers.cmdb.CmdbHandler.cmdb_or_cache_biz", cmdb_or_cache_biz) + @patch("apps.node_man.handlers.cmdb.client_v2", MockClient) + def test_update_multi_outer_ip_proxy_info(self): + host_to_create, identity_to_create, _ = create_host( + number=1, + outer_ip="255.255.255.258", + login_ip="255.255.255.259", + bk_cloud_id=5, + bk_host_id=10002, + node_type=const.NodeType.PROXY, + ip="127.0.0.1", + ) + update_data = { + "bk_host_id": host_to_create[0].bk_host_id, + "outer_ip": "255.255.255.260,255.255.255.261", + "bk_cloud_id": 5, + } + self.assertEqual(HostHandler().update_proxy_info(update_data), None) + proxy_extra_data = Host.objects.get(bk_host_id=10002) + extra_data = proxy_extra_data.extra_data + # 验证多外网IP存入extra_data中 + self.assertEqual(extra_data["bk_host_multi_outerip"], "255.255.255.260,255.255.255.261") + # 验证传入多外网IP时，保存传入的第一个IP + self.assertEqual(proxy_extra_data.outer_ip, "255.255.255.260") diff --git a/apps/node_man/tests/test_pericdic_tasks/test_sync_cmdb_host.py b/apps/node_man/tests/test_pericdic_tasks/test_sync_cmdb_host.py index 03b279dad..f9845eadc 100644 --- a/apps/node_man/tests/test_pericdic_tasks/test_sync_cmdb_host.py +++ b/apps/node_man/tests/test_pericdic_tasks/test_sync_cmdb_host.py @@ -26,7 +26,8 @@ class TestSyncCMDBHost(CustomBaseTestCase): - def init_db(self): + @staticmethod + def init_db(): """ 先在数据库提前构造一些数据 """ @@ -97,3 +98,48 @@ def test_clear_need_delete_host_ids(self): clear_need_delete_host_ids_task() # 验证ProcessStatus中信息是否删除成功 self.assertEqual(ProcessStatus.objects.count(), 1) + + +class TestSyncCMDBMultiOuterIPHost(CustomBaseTestCase): + @staticmethod + def init_db(): + """ + 先在数据库提前构造一些数据 + """ + # 构造待更新数据 + host_data = copy.deepcopy(MOCK_HOST) + host_list = [] + for index in range(1, MOCK_HOST_NUM): + host_data["node_type"] = constants.NodeType.PROXY if index % 2 else constants.NodeType.AGENT + host_data["inner_ip"] = f"127.0.0.{index}" + host_data["bk_host_id"] = index + host_list.append(Host(**host_data)) + + Host.objects.bulk_create(host_list) + + @staticmethod + def list_biz_hosts(*args, **kwargs): + return_value = MockClient.cc.list_resource_pool_hosts(*args, **kwargs) + host_info = return_value["info"] + for host in host_info: + host["bk_host_outerip"] += ",1.2.3.4" + return return_value + + def start_patch(self): + MockClient.cc.list_biz_hosts = self.list_biz_hosts + + @patch("apps.node_man.periodic_tasks.sync_cmdb_host.client_v2", MockClient) + def test_sync_multi_outer_ip_host(self): + self.init_db() + self.start_patch() + sync_cmdb_host_periodic_task(bk_biz_id=2) + # 验证proxy多外网IP将数据存至extra_data中 + proxy_extra_data = Host.objects.filter(node_type=constants.NodeType.PROXY).values("extra_data") + for extra_data in proxy_extra_data: + bk_host_multi_outerip = extra_data["extra_data"]["bk_host_multi_outerip"] + self.assertEqual(len(bk_host_multi_outerip.split(",")), 2) + # 验证不影响Agent存在多外网IP的情况 + agent_extra_data = Host.objects.filter(node_type=constants.NodeType.AGENT).values("extra_data") + for data in agent_extra_data: + extra_data = data["extra_data"] + self.assertEqual(extra_data.get("bk_host_multi_outerip"), None)