From e252dd29631a5b231dbb59142b5b2c574e3ff199 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Fri, 13 Dec 2024 14:59:49 +0800 Subject: [PATCH] =?UTF-8?q?feat(backend):=20=E5=A4=96=E9=83=A8=E9=9B=86?= =?UTF-8?q?=E7=BE=A4=E8=BF=87=E6=9C=9F=E8=AE=BF=E9=97=AE=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=20#8616?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/backend/bk_dataview/grafana/views.py | 2 +- dbm-ui/backend/bk_web/middleware.py | 4 +- dbm-ui/backend/configuration/constants.py | 2 + dbm-ui/backend/configuration/models/system.py | 70 ++++++++++++++++--- dbm-ui/backend/configuration/serializers.py | 7 ++ dbm-ui/backend/configuration/views/system.py | 11 +++ .../builders/tendbcluster/tendb_dump_data.py | 3 +- dbm-ui/backend/utils/time.py | 4 ++ 8 files changed, 88 insertions(+), 15 deletions(-) diff --git a/dbm-ui/backend/bk_dataview/grafana/views.py b/dbm-ui/backend/bk_dataview/grafana/views.py index ba41802d7e..017079fd9e 100644 --- a/dbm-ui/backend/bk_dataview/grafana/views.py +++ b/dbm-ui/backend/bk_dataview/grafana/views.py @@ -416,7 +416,7 @@ def _auth(self, request): result = IAMPermission(actions, resources).has_permission(request, "") # 针对外部查询，需在判断是否集群是否在允许的白名单内 - if env.ENABLE_EXTERNAL_PROXY and cluster.id not in SystemSettings.get_external_whitelist_cluster_ids(): + if env.ENABLE_EXTERNAL_PROXY and not SystemSettings.check_access_external_cluster(cluster.id): raise ExternalClusterIdInvalidException(cluster_id=cluster.id) if not result: diff --git a/dbm-ui/backend/bk_web/middleware.py b/dbm-ui/backend/bk_web/middleware.py index 6f03358786..0c04b703d9 100644 --- a/dbm-ui/backend/bk_web/middleware.py +++ b/dbm-ui/backend/bk_web/middleware.py @@ -172,7 +172,7 @@ def check_create_ticket(): # 目前只放开数据导出 if data["ticket_type"] not in EXTERNAL_TICKET_TYPE_WHITELIST: raise ExternalRouteInvalidException(_("单据类型[{}]非法，未开通白名单").format(data["ticket_type"])) - if data["details"]["cluster_id"] not in SystemSettings.get_external_whitelist_cluster_ids(): + if SystemSettings.check_access_external_cluster(data["details"]["cluster_id"]): raise ExternalClusterIdInvalidException(cluster_id=data["cluster_id"]) # 单据过滤校验函数 @@ -185,7 +185,7 @@ def check_list_ticket(): def check_webconsole(): data = json.loads(request.body.decode("utf-8")) # 校验集群是否在白名单中 - if data["cluster_id"] not in SystemSettings.get_external_whitelist_cluster_ids(): + if SystemSettings.check_access_external_cluster(data["cluster_id"]): raise ExternalClusterIdInvalidException(cluster_id=data["cluster_id"]) check_action_func_map = { diff --git a/dbm-ui/backend/configuration/constants.py b/dbm-ui/backend/configuration/constants.py index 9a77ceadfe..26e278ebdd 100644 --- a/dbm-ui/backend/configuration/constants.py +++ b/dbm-ui/backend/configuration/constants.py @@ -23,6 +23,8 @@ MYSQL_DATA_RESTORE_TIME = 259200 MYSQL_USUAL_JOB_TIME = 7200 MYSQL8_VER_PARSE_NUM = 8000000 +# 外部集群访问过期时间 - 30天 +EXTERNAL_CLUSTER_EXPIRE = 30 class DBPrivSecurityType(str, StructuredEnum): diff --git a/dbm-ui/backend/configuration/models/system.py b/dbm-ui/backend/configuration/models/system.py index 9698f1b66c..a25465c0ba 100644 --- a/dbm-ui/backend/configuration/models/system.py +++ b/dbm-ui/backend/configuration/models/system.py @@ -9,17 +9,21 @@ specific language governing permissions and limitations under the License. """ import logging +from collections import defaultdict +from datetime import datetime, timedelta from typing import Any, Dict, List, Optional, Union from django.conf import settings -from django.db import connection, models +from django.db import connection, models, transaction from django.utils.translation import ugettext_lazy as _ from backend import env from backend.bk_web.constants import LEN_LONG, LEN_NORMAL from backend.bk_web.models import AuditedModel from backend.configuration import constants +from backend.configuration.constants import EXTERNAL_CLUSTER_EXPIRE, SystemSettingsEnum from backend.db_meta.enums import ClusterType +from backend.utils.time import date2str, str2date logger = logging.getLogger("root") @@ -33,10 +37,13 @@ class AbstractSettings(AuditedModel): desc = models.CharField(_("描述"), max_length=LEN_LONG) @classmethod - def get_setting_value(cls, key: dict, default: Optional[Any] = None) -> Union[str, Dict, List]: + def get_setting_value(cls, key: dict, default: Optional[Any] = None, lock: bool = False) -> Union[str, Dict, List]: """插入一条配置记录""" try: - setting_value = cls.objects.get(**key).value + if lock: + setting_value = cls.objects.select_for_update().get(**key).value + else: + setting_value = cls.objects.get(**key).value except cls.DoesNotExist: if default is None: setting_value = "" @@ -103,11 +110,13 @@ def register_system_settings(cls): setattr(settings, system_setting.key, system_setting.value) @classmethod - def get_setting_value(cls, key: str, default: Optional[Any] = None) -> Union[str, Dict, List]: + def get_setting_value(cls, key: str, default: Optional[Any] = None, lock: bool = False) -> Union[str, Dict, List]: return super().get_setting_value(key={"key": key}, default=default) @classmethod - def insert_setting_value(cls, key: str, value: Any, value_type: str = "str", user: str = "admin") -> None: + def insert_setting_value( + cls, key: str, value: Any, value_type: str = "str", user: str = "admin", desc: str = "" + ) -> None: return super().insert_setting_value( key={"key": key}, value=value, @@ -117,13 +126,52 @@ def insert_setting_value(cls, key: str, value: Any, value_type: str = "str", use ) @classmethod - def get_external_whitelist_cluster_ids(cls) -> List: - return [ - conf["cluster_id"] - for conf in cls.get_setting_value( - key=constants.SystemSettingsEnum.EXTERNAL_WHITELIST_CLUSTER_IDS.value, default=[] + def check_access_external_cluster(cls, cluster_id) -> bool: + """ + 获取是否可访问外部合法白名单集群，数据格式 + "$cluster_id": {"bk_biz_id": 123, "operator": "somebody", "update_at": "2024-12-13 10:23:33", "remark": "xxx"} + """ + today = datetime.today().date() + cluster_id = str(cluster_id) + with transaction.atomic(): + # 用行锁控制并发时更新请求的不一致 + whitelist = cls.get_setting_value( + key=SystemSettingsEnum.EXTERNAL_WHITELIST_CLUSTER_IDS, default={}, lock=True + ) + if cluster_id not in whitelist: + return False + + # 判断集群时间是否过期，如果过期则删除该key并报错，否则更新访问时间 + access_date = str2date(whitelist[cluster_id]["update_at"]) + if access_date - timedelta(days=EXTERNAL_CLUSTER_EXPIRE) > today: + check_flag = False + else: + if access_date != today: + whitelist[cluster_id]["update_at"] = date2str(today) + cls.insert_setting_value(key=SystemSettingsEnum.EXTERNAL_WHITELIST_CLUSTER_IDS, value=whitelist) + check_flag = True + + return check_flag + + @classmethod + def update_external_cluster(cls, bk_biz_id, operator, cluster_id, remark=""): + """ + 更新外部可访问集群名单 + """ + with transaction.atomic(): + # 用行锁控制并发时更新请求的不一致 + whitelist = cls.get_setting_value( + key=SystemSettingsEnum.EXTERNAL_WHITELIST_CLUSTER_IDS, default=defaultdict(dict), lock=True + ) + whitelist[cluster_id] = { + "bk_biz_id": bk_biz_id, + "operator": operator, + "remark": remark, + "update_at": date2str(datetime.today()), + } + cls.insert_setting_value( + key=SystemSettingsEnum.EXTERNAL_WHITELIST_CLUSTER_IDS, value=whitelist, value_type="dict" ) - ] class BizSettings(AbstractSettings): diff --git a/dbm-ui/backend/configuration/serializers.py b/dbm-ui/backend/configuration/serializers.py index 03100531ad..e6afe68d3f 100644 --- a/dbm-ui/backend/configuration/serializers.py +++ b/dbm-ui/backend/configuration/serializers.py @@ -254,3 +254,10 @@ class FunctionControllerSerializer(serializers.Serializer): class Meta: model = FunctionController fields = "__all__" + + +class UpdateExternalClusterSerializer(serializers.Serializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + bk_biz_id = serializers.IntegerField(help_text=_("业务ID")) + operator = serializers.CharField(help_text=_("更新人")) + remark = serializers.CharField(help_text=_("备注"), required=False, default="") diff --git a/dbm-ui/backend/configuration/views/system.py b/dbm-ui/backend/configuration/views/system.py index 308e085047..16c24f2ecd 100644 --- a/dbm-ui/backend/configuration/views/system.py +++ b/dbm-ui/backend/configuration/views/system.py @@ -26,6 +26,7 @@ ListBizSettingsSerializer, UpdateBizSettingsSerializer, UpdateDutyNoticeSerializer, + UpdateExternalClusterSerializer, ) from backend.db_meta.models import AppCache from backend.db_services.ipchooser.constants import IDLE_HOST_MODULE @@ -126,6 +127,16 @@ def sensitive_environ(self, request): } ) + @common_swagger_auto_schema( + operation_summary=_("更新外部集群白名单"), + tags=tags, + request_body=UpdateExternalClusterSerializer(), + ) + @action(methods=["POST"], detail=False, pagination_class=None, serializer_class=UpdateExternalClusterSerializer) + def update_external_cluster(self, request): + SystemSettings.update_external_cluster(**self.validated_data) + return Response() + class BizSettingsViewSet(viewsets.AuditedModelViewSet): """业务设置视图""" diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_dump_data.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_dump_data.py index 15d05c91c0..17d718f2a9 100644 --- a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_dump_data.py +++ b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_dump_data.py @@ -14,6 +14,7 @@ from backend.ticket import builders from backend.ticket.builders.mysql.mysql_dump_data import ( MySQLDumpDataDetailSerializer, + MySQLDumpDataFlowBuilder, MySQLDumpDataFlowParamBuilder, MySQLDumpDataItsmFlowParamsBuilder, ) @@ -34,7 +35,7 @@ class TendbClusterDumpDataItsmFlowParamsBuilder(MySQLDumpDataItsmFlowParamsBuild @builders.BuilderFactory.register(TicketType.TENDBCLUSTER_DUMP_DATA) -class TendbClusterDumpDataFlowBuilder(BaseTendbTicketFlowBuilder): +class TendbClusterDumpDataFlowBuilder(BaseTendbTicketFlowBuilder, MySQLDumpDataFlowBuilder): serializer = TendbClusterDumpDataDetailSerializer itsm_flow_builder = TendbClusterDumpDataItsmFlowParamsBuilder inner_flow_builder = TendbClusterDumpDataFlowParamBuilder diff --git a/dbm-ui/backend/utils/time.py b/dbm-ui/backend/utils/time.py index 90f809bb1d..8c755eb8b4 100644 --- a/dbm-ui/backend/utils/time.py +++ b/dbm-ui/backend/utils/time.py @@ -102,6 +102,10 @@ def date2str(o_date: datetime.date, fmt: str = DATE_PATTERN) -> str: return datetime.date.strftime(o_date, fmt) +def str2date(date_str: str, fmt: str = DATE_PATTERN) -> datetime.date: + return datetime.datetime.strptime(date_str, fmt).date() + + def calculate_cost_time( end_time: Optional[Union[datetime.datetime, str]], start_time: Optional[Union[datetime.datetime, str]],