feature: 云梯安全组添加白名单功能 (closed TencentBlueKing#1760)

apps/backend/components/collections/agent_new/configure_policy.py

@@ -36,7 +36,8 @@ def _execute(self, data, parent_data, common_data):
         for host in host_id_obj_map.values():
             ip_list.extend([host.outer_ip, host.login_ip])
         # 不同的安全组工厂添加策略后得到的输出可能是不同的，输出到outputs中，在schedule中由工厂对应的check_result方法来校验结果
-        data.outputs.add_ip_output = security_group_factory.add_ips_to_security_group(ip_list)
+        creator: str = common_data.subscription.creator
+        data.outputs.add_ip_output = security_group_factory.add_ips_to_security_group(ip_list, creator=creator)
         data.outputs.polling_time = 0
         return True
 

apps/backend/utils/dataclass.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import copy
+from dataclasses import _is_dataclass_instance, fields
+
+
+def _asdict_inner(obj, dict_factory):
+    if _is_dataclass_instance(obj):
+        result = []
+        for f in fields(obj):
+            value = _asdict_inner(getattr(obj, f.name), dict_factory)
+            # 过滤掉为空或者None的字段
+            if not value:
+                continue
+            result.append((f.name, value))
+        return dict_factory(result)
+    elif isinstance(obj, (list, tuple)):
+        return type(obj)(_asdict_inner(v, dict_factory) for v in obj)
+    elif isinstance(obj, dict):
+        return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items())
+    else:
+        return copy.deepcopy(obj)
+
+
+def asdict(obj, *, dict_factory=dict):
+    if not _is_dataclass_instance(obj):
+        raise TypeError("asdict() should be called on dataclass instances")
+    return _asdict_inner(obj, dict_factory)

apps/node_man/handlers/security_group.py

@@ -1,12 +1,26 @@
 # -*- coding: utf-8 -*-
 import abc
-from typing import Dict, List, Optional
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
 
 from django.conf import settings
 
+from apps.backend.utils.dataclass import asdict
 from apps.node_man.exceptions import ConfigurationPolicyError
 from apps.node_man.policy.tencent_vpc_client import VpcClient
-from common.api import SopsApi
+from apps.utils.batch_request import request_multi_thread
+from common.api import SopsApi, YunTiApi
+from common.log import logger
+
+
+@dataclass
+class YunTiPolicyData:
+    Protocol: str
+    CidrBlock: str
+    Port: str
+    Action: str
+    PolicyDescription: Optional[str] = None
+    Ipv6CidrBlock: Optional[str] = None
 
 
 class BaseSecurityGroupFactory(abc.ABC):
@@ -16,7 +30,7 @@ def describe_security_group_address(self) -> List[str]:
         """获取安全组IP地址"""
         raise NotImplementedError()
 
-    def add_ips_to_security_group(self, ip_list: List[str]) -> Dict:
+    def add_ips_to_security_group(self, ip_list: List[str], creator: str = None) -> Dict:
         """添加IP到安全组中，输出的字典用作check_result的入参"""
         raise NotImplementedError()
 
@@ -31,7 +45,7 @@ class SopsSecurityGroupFactory(BaseSecurityGroupFactory):
     def describe_security_group_address(self):
         pass
 
-    def add_ips_to_security_group(self, ip_list: List[str]) -> Dict:
+    def add_ips_to_security_group(self, ip_list: List[str], creator: str = None) -> Dict:
         task_id = SopsApi.create_task(
             {
                 "name": "NodeMan Configure SecurityGroup",
@@ -76,7 +90,7 @@ def describe_security_group_address(self) -> List:
                 ip_set = ip_set & set(client.describe_address_templates(template))
         return list(ip_set)
 
-    def add_ips_to_security_group(self, ip_list: List[str]):
+    def add_ips_to_security_group(self, ip_list: List[str], creator: str = None):
         client = VpcClient()
         for template in client.ip_templates:
             using_ip_list = client.describe_address_templates(template)
@@ -94,6 +108,123 @@ def check_result(self, add_ip_output: Dict) -> bool:
         return set(add_ip_output["ip_list"]).issubset(set(current_ip_list))
 
 
+class YunTiSecurityGroupFactory(BaseSecurityGroupFactory):
+    SECURITY_GROUP_TYPE: str = "YUNTI"
+    DEPT_ID: int = int(settings.BKAPP_REQUEST_YUNTI_DEPT_ID)
+    REGION: str = settings.BKAPP_REQUEST_YUNTI_REGION
+    # BKAPP_REQUEST_YUNTI_SID_LIST Example: name:id,name:id
+    SID_INFO: Dict[str, str] = {
+        s_info.split(":")[0]: s_info.split(":")[1] for s_info in settings.BKAPP_REQUEST_YUNTI_SID_LIST.split(",")
+    }
+
+    def describe_security_group_address(self) -> Dict:
+        # 批量获取当前安全组策略详情
+        params_list: List = []
+        for _, sid in self.SID_INFO.items():
+            params_list.append(
+                {
+                    "params": {
+                        "method": "get-security-group-policies",
+                        "params": {
+                            "deptId": self.DEPT_ID,
+                            "region": self.REGION,
+                            "sid": sid,
+                        },
+                        "no_request": True,
+                    },
+                }
+            )
+        # 批量请求
+        result: List[Dict[str, Any]] = request_multi_thread(
+            func=YunTiApi.get_security_group_details,
+            params_list=params_list,
+            get_data=lambda x: [x],
+        )
+        return {sid_info["SecurityGroupId"]: sid_info for sid_info in result}
+
+    def add_ips_to_security_group(self, ip_list: List[str], creator: str = None):
+        result: Dict[str, Dict[str, Any]] = self.describe_security_group_address()
+        params_list: List[Dict[str, Any]] = []
+
+        # 新策略列表
+        new_in_gress: Dict[str, Dict[str, Any]] = {}
+        for ip in ip_list:
+            new_in_gress[ip] = asdict(
+                YunTiPolicyData(
+                    Protocol="ALL",
+                    CidrBlock=ip,
+                    Port="ALL",
+                    Action="ACCEPT",
+                    PolicyDescription="",
+                    Ipv6CidrBlock="",
+                )
+            )
+
+        for name, sid in self.SID_INFO.items():
+            version: str = result[sid]["pilicies"]["SecurityGroupPolicySet"]["Version"]
+            current_policies: List[Dict[str, Any]] = result[sid]["pilicies"]["SecurityGroupPolicySet"]["Ingress"]
+
+            # 已有策略列表
+            in_gress: Dict[str, Dict[str, Any]] = {}
+            for policy in current_policies:
+                in_gress[policy["CidrBlock"]] = asdict(
+                    YunTiPolicyData(
+                        Protocol=policy["Protocol"],
+                        CidrBlock=policy["CidrBlock"],
+                        Port=policy["Port"],
+                        Action=policy["Action"],
+                        PolicyDescription=policy["PolicyDescription"],
+                        Ipv6CidrBlock=policy["Ipv6CidrBlock"],
+                    )
+                )
+            # 增加新IP
+            in_gress.update(new_in_gress)
+
+            params_list.append(
+                {
+                    "params": {
+                        "method": "createSecurityGroupForm",
+                        "params": {
+                            "deptId": self.DEPT_ID,
+                            "region": self.REGION,
+                            "sid": sid,
+                            "type": "modify",
+                            "mark": "Add proxy whitelist to Shangyun security group security.",
+                            "groupName": name,
+                            "groupDesc": "Proxy whitelist for Shangyun",
+                            "creator": creator,
+                            "ext": {
+                                "Version": version,
+                                "Egress": [],
+                                "Ingress": list(in_gress.values()),
+                            },
+                        },
+                        "no_request": True,
+                    },
+                }
+            )
+        # 批量请求
+        logger.info(f"Add proxy whitelist to Shangyun security group security. params: {params_list}")
+        request_multi_thread(func=YunTiApi.operate_security_group, params_list=params_list)
+        return {"ip_list": ip_list}
+
+    def check_result(self, add_ip_output: Dict) -> bool:
+        """检查IP列表是否已添加到安全组中"""
+        result: Dict[str, Dict[str, Any]] = self.describe_security_group_address()
+        is_success: bool = True
+        for _, sid in self.SID_INFO.items():
+            current_policies: List[Dict[str, Any]] = result[sid]["pilicies"]["SecurityGroupPolicySet"]["Ingress"]
+            current_ip_list = [policy["CidrBlock"] for policy in current_policies]
+            logger.info(
+                f"check_result: Add proxy whitelist to Shangyun security group security. "
+                f"sid: {sid} ip_list: {add_ip_output['ip_list']}"
+            )
+            # 需添加的IP列表是已有IP的子集，则认为已添加成功
+            is_success: bool = is_success and set(add_ip_output["ip_list"]).issubset(set(current_ip_list))
+
+        return is_success
+
+
 def get_security_group_factory(security_group_type: Optional[str]) -> BaseSecurityGroupFactory:
     """获取安全组工厂，返回None表示无需配置安全组"""
     factory_map = {factory.SECURITY_GROUP_TYPE: factory for factory in BaseSecurityGroupFactory.__subclasses__()}

common/api/__init__.py

@@ -41,6 +41,8 @@ def new_api_module(module_name, api_name, module_dir="modules"):
 
 # ESB
 EsbApi = SimpleLazyObject(lambda: new_api_module("esb", "_ESBApi"))
+# YUNTI
+YunTiApi = SimpleLazyObject(lambda: new_api_module("yunti", "_YunTiApi"))
 
 __all__ = [
     "CCApi",
@@ -50,6 +52,7 @@ def new_api_module(module_name, api_name, module_dir="modules"):
     "SopsApi",
     "CmsiApi",
     "NodeApi",
+    "YunTiApi",
 ]
 
 

common/api/base.py

@@ -163,7 +163,7 @@ def __call__(
             # 统一处理返回内容，根据平台既定规则，断定成功与否
             if raise_exception and not response.is_success():
                 raise ApiResultError(
-                    self.get_error_message(response.message),
+                    self.get_error_message(response.message or response.response),
                     code=response.code,
                     errors=response.errors,
                     data=response.data,

common/api/domains.py

@@ -39,3 +39,4 @@ def gen_api_root(api_gw_env_key: str, suffix: str) -> str:
 JOB_APIGATEWAY_ROOT_V3 = gen_api_root("BKAPP_BK_JOB_APIGATEWAY", "jobv3")
 SOPS_APIGATEWAY_ROOT = gen_api_root("BKAPP_BK_SOPS_APIGATEWAY", "sops")
 BK_NODE_APIGATEWAY_ROOT = gen_api_root("BKAPP_BK_NODE_APIGATEWAY", "nodeman")
+BKAPP_YUNTI_API_ROOT = os.getenv("BKAPP_YUNTI_API_ROOT")

common/api/modules/yunti.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import hashlib
+import hmac
+import time
+import typing
+
+from django.conf import settings
+from django.utils.translation import ugettext_lazy as _
+
+from ..base import DataAPI
+from ..domains import BKAPP_YUNTI_API_ROOT
+
+
+def yunti_api_before_request(params) -> typing.Dict[str, typing.Any]:
+    api_ts: int = int(time.time())
+    api_key_name: str = settings.BKAPP_REQUEST_YUNTI_APP_KEY
+    api_key_secret: str = settings.BKAPP_REQUEST_YUNTI_SECRET
+    api_sign: str = hmac.new(
+        api_key_secret.encode("utf8"),
+        (str(api_ts) + api_key_name).encode("utf8"),
+        hashlib.sha1,
+    ).hexdigest()
+
+    params["api_key"] = api_key_name
+    params["api_ts"] = api_ts
+    params["api_sigin"] = api_sign
+    params["jsonrpc"] = "2.0"
+    params["id"] = str(api_ts)
+    return params
+
+
+def yunti_api_after_request(result):
+    data = result.pop("result")
+    result["data"] = data
+    result["result"] = True
+    return result
+
+
+class _YunTiApi(object):
+    MODULE = _("云梯")
+
+    def __init__(self):
+
+        self.get_security_group_details = DataAPI(
+            method="POST",
+            url=BKAPP_YUNTI_API_ROOT + "/account/manage?api_key={api_key}&api_sign={api_sigin}&api_ts={api_ts}",
+            module=self.MODULE,
+            description="查询安全组详情",
+            before_request=yunti_api_before_request,
+            after_request=yunti_api_after_request,
+        )
+
+        self.operate_security_group = DataAPI(
+            method="POST",
+            url=BKAPP_YUNTI_API_ROOT + "/apply/api/sg?api_key={api_key}&api_sign={api_sigin}&api_ts={api_ts}",
+            module=self.MODULE,
+            description="修改安全组",
+            before_request=yunti_api_before_request,
+            after_request=yunti_api_after_request,
+        )

config/default.py

@@ -727,6 +727,12 @@ def get_standard_redis_mode(cls, config_redis_mode: str, default: Optional[str]
 BKAPP_EE_SOPS_TEMPLATE_ID = os.getenv("BKAPP_EE_SOPS_TEMPLATE_ID")
 BKAPP_REQUEST_EE_SOPS_BK_BIZ_ID = os.getenv("BKAPP_REQUEST_EE_SOPS_BK_BIZ_ID")
 
+# 使用云梯开通策略相关变量
+BKAPP_REQUEST_YUNTI_APP_KEY = os.getenv("BKAPP_REQUEST_YUNTI_APP_KEY")
+BKAPP_REQUEST_YUNTI_SECRET = os.getenv("BKAPP_REQUEST_YUNTI_SECRET")
+BKAPP_REQUEST_YUNTI_REGION = os.getenv("BKAPP_REQUEST_YUNTI_REGION")
+BKAPP_REQUEST_YUNTI_SID_LIST = os.getenv("BKAPP_REQUEST_YUNTI_SID_LIST")
+BKAPP_REQUEST_YUNTI_DEPT_ID = os.getenv("BKAPP_REQUEST_YUNTI_DEPT_ID")
 
 # 管控平台平台版本
 GSE_VERSION = env.GSE_VERSION