diff --git a/apps/backend/components/collections/agent_new/configure_policy.py b/apps/backend/components/collections/agent_new/configure_policy.py index 0043cd283..3ff55f475 100644 --- a/apps/backend/components/collections/agent_new/configure_policy.py +++ b/apps/backend/components/collections/agent_new/configure_policy.py @@ -36,7 +36,8 @@ def _execute(self, data, parent_data, common_data): for host in host_id_obj_map.values(): ip_list.extend([host.outer_ip, host.login_ip]) # 不同的安全组工厂添加策略后得到的输出可能是不同的，输出到outputs中，在schedule中由工厂对应的check_result方法来校验结果 - data.outputs.add_ip_output = security_group_factory.add_ips_to_security_group(ip_list) + creator: str = common_data.subscription.creator + data.outputs.add_ip_output = security_group_factory.add_ips_to_security_group(ip_list, creator=creator) data.outputs.polling_time = 0 return True diff --git a/apps/backend/utils/dataclass.py b/apps/backend/utils/dataclass.py new file mode 100644 index 000000000..c89b4364f --- /dev/null +++ b/apps/backend/utils/dataclass.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import copy +from dataclasses import _is_dataclass_instance, fields + + +def _asdict_inner(obj, dict_factory): + if _is_dataclass_instance(obj): + result = [] + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name), dict_factory) + # 过滤掉为空或者None的字段 + if not value: + continue + result.append((f.name, value)) + return dict_factory(result) + elif isinstance(obj, (list, tuple)): + return type(obj)(_asdict_inner(v, dict_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items()) + else: + return copy.deepcopy(obj) + + +def asdict(obj, *, dict_factory=dict): + if not _is_dataclass_instance(obj): + raise TypeError("asdict() should be called on dataclass instances") + return _asdict_inner(obj, dict_factory) diff --git a/apps/node_man/handlers/security_group.py b/apps/node_man/handlers/security_group.py index 3ed88365f..7cfabe963 100644 --- a/apps/node_man/handlers/security_group.py +++ b/apps/node_man/handlers/security_group.py @@ -1,12 +1,26 @@ # -*- coding: utf-8 -*- import abc -from typing import Dict, List, Optional +from dataclasses import dataclass +from typing import Any, Dict, List, Optional from django.conf import settings +from apps.backend.utils.dataclass import asdict from apps.node_man.exceptions import ConfigurationPolicyError from apps.node_man.policy.tencent_vpc_client import VpcClient -from common.api import SopsApi +from apps.utils.batch_request import request_multi_thread +from common.api import SopsApi, YunTiApi +from common.log import logger + + +@dataclass +class YunTiPolicyData: + Protocol: str + CidrBlock: str + Port: str + Action: str + PolicyDescription: Optional[str] = None + Ipv6CidrBlock: Optional[str] = None class BaseSecurityGroupFactory(abc.ABC): @@ -16,7 +30,7 @@ def describe_security_group_address(self) -> List[str]: """获取安全组IP地址""" raise NotImplementedError() - def add_ips_to_security_group(self, ip_list: List[str]) -> Dict: + def add_ips_to_security_group(self, ip_list: List[str], creator: str = None) -> Dict: """添加IP到安全组中，输出的字典用作check_result的入参""" raise NotImplementedError() @@ -31,7 +45,7 @@ class SopsSecurityGroupFactory(BaseSecurityGroupFactory): def describe_security_group_address(self): pass - def add_ips_to_security_group(self, ip_list: List[str]) -> Dict: + def add_ips_to_security_group(self, ip_list: List[str], creator: str = None) -> Dict: task_id = SopsApi.create_task( { "name": "NodeMan Configure SecurityGroup", @@ -76,7 +90,7 @@ def describe_security_group_address(self) -> List: ip_set = ip_set & set(client.describe_address_templates(template)) return list(ip_set) - def add_ips_to_security_group(self, ip_list: List[str]): + def add_ips_to_security_group(self, ip_list: List[str], creator: str = None): client = VpcClient() for template in client.ip_templates: using_ip_list = client.describe_address_templates(template) @@ -94,6 +108,123 @@ def check_result(self, add_ip_output: Dict) -> bool: return set(add_ip_output["ip_list"]).issubset(set(current_ip_list)) +class YunTiSecurityGroupFactory(BaseSecurityGroupFactory): + SECURITY_GROUP_TYPE: str = "YUNTI" + DEPT_ID: int = int(settings.BKAPP_REQUEST_YUNTI_DEPT_ID) + REGION: str = settings.BKAPP_REQUEST_YUNTI_REGION + # BKAPP_REQUEST_YUNTI_SID_LIST Example: name:id,name:id + SID_INFO: Dict[str, str] = { + s_info.split(":")[0]: s_info.split(":")[1] for s_info in settings.BKAPP_REQUEST_YUNTI_SID_LIST.split(",") + } + + def describe_security_group_address(self) -> Dict: + # 批量获取当前安全组策略详情 + params_list: List = [] + for _, sid in self.SID_INFO.items(): + params_list.append( + { + "params": { + "method": "get-security-group-policies", + "params": { + "deptId": self.DEPT_ID, + "region": self.REGION, + "sid": sid, + }, + "no_request": True, + }, + } + ) + # 批量请求 + result: List[Dict[str, Any]] = request_multi_thread( + func=YunTiApi.get_security_group_details, + params_list=params_list, + get_data=lambda x: [x], + ) + return {sid_info["SecurityGroupId"]: sid_info for sid_info in result} + + def add_ips_to_security_group(self, ip_list: List[str], creator: str = None): + result: Dict[str, Dict[str, Any]] = self.describe_security_group_address() + params_list: List[Dict[str, Any]] = [] + + # 新策略列表 + new_in_gress: Dict[str, Dict[str, Any]] = {} + for ip in ip_list: + new_in_gress[ip] = asdict( + YunTiPolicyData( + Protocol="ALL", + CidrBlock=ip, + Port="ALL", + Action="ACCEPT", + PolicyDescription="", + Ipv6CidrBlock="", + ) + ) + + for name, sid in self.SID_INFO.items(): + version: str = result[sid]["pilicies"]["SecurityGroupPolicySet"]["Version"] + current_policies: List[Dict[str, Any]] = result[sid]["pilicies"]["SecurityGroupPolicySet"]["Ingress"] + + # 已有策略列表 + in_gress: Dict[str, Dict[str, Any]] = {} + for policy in current_policies: + in_gress[policy["CidrBlock"]] = asdict( + YunTiPolicyData( + Protocol=policy["Protocol"], + CidrBlock=policy["CidrBlock"], + Port=policy["Port"], + Action=policy["Action"], + PolicyDescription=policy["PolicyDescription"], + Ipv6CidrBlock=policy["Ipv6CidrBlock"], + ) + ) + # 增加新IP + in_gress.update(new_in_gress) + + params_list.append( + { + "params": { + "method": "createSecurityGroupForm", + "params": { + "deptId": self.DEPT_ID, + "region": self.REGION, + "sid": sid, + "type": "modify", + "mark": "Add proxy whitelist to Shangyun security group security.", + "groupName": name, + "groupDesc": "Proxy whitelist for Shangyun", + "creator": creator, + "ext": { + "Version": version, + "Egress": [], + "Ingress": list(in_gress.values()), + }, + }, + "no_request": True, + }, + } + ) + # 批量请求 + logger.info(f"Add proxy whitelist to Shangyun security group security. params: {params_list}") + request_multi_thread(func=YunTiApi.operate_security_group, params_list=params_list) + return {"ip_list": ip_list} + + def check_result(self, add_ip_output: Dict) -> bool: + """检查IP列表是否已添加到安全组中""" + result: Dict[str, Dict[str, Any]] = self.describe_security_group_address() + is_success: bool = True + for _, sid in self.SID_INFO.items(): + current_policies: List[Dict[str, Any]] = result[sid]["pilicies"]["SecurityGroupPolicySet"]["Ingress"] + current_ip_list = [policy["CidrBlock"] for policy in current_policies] + logger.info( + f"check_result: Add proxy whitelist to Shangyun security group security. " + f"sid: {sid} ip_list: {add_ip_output['ip_list']}" + ) + # 需添加的IP列表是已有IP的子集，则认为已添加成功 + is_success: bool = is_success and set(add_ip_output["ip_list"]).issubset(set(current_ip_list)) + + return is_success + + def get_security_group_factory(security_group_type: Optional[str]) -> BaseSecurityGroupFactory: """获取安全组工厂，返回None表示无需配置安全组""" factory_map = {factory.SECURITY_GROUP_TYPE: factory for factory in BaseSecurityGroupFactory.__subclasses__()} diff --git a/common/api/__init__.py b/common/api/__init__.py index c20b861a4..27c35a8cc 100644 --- a/common/api/__init__.py +++ b/common/api/__init__.py @@ -41,6 +41,8 @@ def new_api_module(module_name, api_name, module_dir="modules"): # ESB EsbApi = SimpleLazyObject(lambda: new_api_module("esb", "_ESBApi")) +# YUNTI +YunTiApi = SimpleLazyObject(lambda: new_api_module("yunti", "_YunTiApi")) __all__ = [ "CCApi", @@ -50,6 +52,7 @@ def new_api_module(module_name, api_name, module_dir="modules"): "SopsApi", "CmsiApi", "NodeApi", + "YunTiApi", ] diff --git a/common/api/base.py b/common/api/base.py index bf6107ff8..5b6c5295c 100644 --- a/common/api/base.py +++ b/common/api/base.py @@ -163,7 +163,7 @@ def __call__( # 统一处理返回内容，根据平台既定规则，断定成功与否 if raise_exception and not response.is_success(): raise ApiResultError( - self.get_error_message(response.message), + self.get_error_message(response.message or response.response), code=response.code, errors=response.errors, data=response.data, diff --git a/common/api/domains.py b/common/api/domains.py index 51fcf5563..4737ee22a 100644 --- a/common/api/domains.py +++ b/common/api/domains.py @@ -39,3 +39,4 @@ def gen_api_root(api_gw_env_key: str, suffix: str) -> str: JOB_APIGATEWAY_ROOT_V3 = gen_api_root("BKAPP_BK_JOB_APIGATEWAY", "jobv3") SOPS_APIGATEWAY_ROOT = gen_api_root("BKAPP_BK_SOPS_APIGATEWAY", "sops") BK_NODE_APIGATEWAY_ROOT = gen_api_root("BKAPP_BK_NODE_APIGATEWAY", "nodeman") +BKAPP_YUNTI_API_ROOT = os.getenv("BKAPP_YUNTI_API_ROOT") diff --git a/common/api/modules/yunti.py b/common/api/modules/yunti.py new file mode 100644 index 000000000..6037f5486 --- /dev/null +++ b/common/api/modules/yunti.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import hashlib +import hmac +import time +import typing + +from django.conf import settings +from django.utils.translation import ugettext_lazy as _ + +from ..base import DataAPI +from ..domains import BKAPP_YUNTI_API_ROOT + + +def yunti_api_before_request(params) -> typing.Dict[str, typing.Any]: + api_ts: int = int(time.time()) + api_key_name: str = settings.BKAPP_REQUEST_YUNTI_APP_KEY + api_key_secret: str = settings.BKAPP_REQUEST_YUNTI_SECRET + api_sign: str = hmac.new( + api_key_secret.encode("utf8"), + (str(api_ts) + api_key_name).encode("utf8"), + hashlib.sha1, + ).hexdigest() + + params["api_key"] = api_key_name + params["api_ts"] = api_ts + params["api_sigin"] = api_sign + params["jsonrpc"] = "2.0" + params["id"] = str(api_ts) + return params + + +def yunti_api_after_request(result): + data = result.pop("result") + result["data"] = data + result["result"] = True + return result + + +class _YunTiApi(object): + MODULE = _("云梯") + + def __init__(self): + + self.get_security_group_details = DataAPI( + method="POST", + url=BKAPP_YUNTI_API_ROOT + "/account/manage?api_key={api_key}&api_sign={api_sigin}&api_ts={api_ts}", + module=self.MODULE, + description="查询安全组详情", + before_request=yunti_api_before_request, + after_request=yunti_api_after_request, + ) + + self.operate_security_group = DataAPI( + method="POST", + url=BKAPP_YUNTI_API_ROOT + "/apply/api/sg?api_key={api_key}&api_sign={api_sigin}&api_ts={api_ts}", + module=self.MODULE, + description="修改安全组", + before_request=yunti_api_before_request, + after_request=yunti_api_after_request, + ) diff --git a/config/default.py b/config/default.py index 6a597cade..6f8c7b9a0 100644 --- a/config/default.py +++ b/config/default.py @@ -727,6 +727,12 @@ def get_standard_redis_mode(cls, config_redis_mode: str, default: Optional[str] BKAPP_EE_SOPS_TEMPLATE_ID = os.getenv("BKAPP_EE_SOPS_TEMPLATE_ID") BKAPP_REQUEST_EE_SOPS_BK_BIZ_ID = os.getenv("BKAPP_REQUEST_EE_SOPS_BK_BIZ_ID") +# 使用云梯开通策略相关变量 +BKAPP_REQUEST_YUNTI_APP_KEY = os.getenv("BKAPP_REQUEST_YUNTI_APP_KEY") +BKAPP_REQUEST_YUNTI_SECRET = os.getenv("BKAPP_REQUEST_YUNTI_SECRET") +BKAPP_REQUEST_YUNTI_REGION = os.getenv("BKAPP_REQUEST_YUNTI_REGION") +BKAPP_REQUEST_YUNTI_SID_LIST = os.getenv("BKAPP_REQUEST_YUNTI_SID_LIST") +BKAPP_REQUEST_YUNTI_DEPT_ID = os.getenv("BKAPP_REQUEST_YUNTI_DEPT_ID") # 管控平台平台版本 GSE_VERSION = env.GSE_VERSION