diff --git a/dbm-ui/backend/db_meta/admin.py b/dbm-ui/backend/db_meta/admin.py index 44ef249c2e..b694372f35 100644 --- a/dbm-ui/backend/db_meta/admin.py +++ b/dbm-ui/backend/db_meta/admin.py @@ -126,6 +126,12 @@ class MachineAdmin(admin.ModelAdmin): search_fields = ("ip", "db_module_id") +@admin.register(models.machine.DeviceClass) +class DeviceClassAdmin(admin.ModelAdmin): + list_display = ("device_type", "cpu", "mem", "disk") + search_fields = ("device_type",) + + @admin.register(models.storage_instance_tuple.StorageInstanceTuple) class StorageInstanceTupleAdmin(DynamicRawIDMixin, admin.ModelAdmin): list_display = ( diff --git a/dbm-ui/backend/db_meta/enums/spec.py b/dbm-ui/backend/db_meta/enums/spec.py index 29c97c50cd..b99a5a797f 100644 --- a/dbm-ui/backend/db_meta/enums/spec.py +++ b/dbm-ui/backend/db_meta/enums/spec.py @@ -51,6 +51,10 @@ class SpecMachineType(str, StructuredEnum): MONGODB = EnumField("mongodb", _("mongodb")) MONOG_CONFIG = EnumField("mongo_config", _("mongo_config")) + DORIS_FOLLOWER = EnumField("doris_follower", _("doris_follower")) + DORIS_OBSERVER = EnumField("doris_observer", _("doris_observer")) + DORIS_BACKEND = EnumField("doris_backend", _("doris_backend")) + # TODO: 规格迁移脚本函数，迁移完成后删除 def migrate_spec(): diff --git a/dbm-ui/backend/db_meta/migrations/0044_deviceclass.py b/dbm-ui/backend/db_meta/migrations/0044_deviceclass.py new file mode 100644 index 0000000000..9124fc4326 --- /dev/null +++ b/dbm-ui/backend/db_meta/migrations/0044_deviceclass.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.25 on 2024-11-19 07:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("db_meta", "0043_auto_20241014_1042"), + ] + + operations = [ + migrations.CreateModel( + name="DeviceClass", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("device_type", models.CharField(help_text="机型类型", max_length=128)), + ("cpu", models.IntegerField(default=0, help_text="机型cpu")), + ("mem", models.IntegerField(default=0, help_text="机型内存")), + ("disk", models.IntegerField(default=0, help_text="机型磁盘")), + ], + options={ + "verbose_name": "机型(DeviceClass)", + "verbose_name_plural": "机型(DeviceClass)", + }, + ), + ] diff --git a/dbm-ui/backend/db_meta/models/machine.py b/dbm-ui/backend/db_meta/models/machine.py index 3254aa4016..f81f8593cd 100644 --- a/dbm-ui/backend/db_meta/models/machine.py +++ b/dbm-ui/backend/db_meta/models/machine.py @@ -19,10 +19,13 @@ from backend.bk_web.models import AuditedModel from backend.components import CCApi +from backend.components.hcm.client import HCMApi from backend.constants import CommonHostDBMeta from backend.db_meta.enums import AccessLayer, ClusterType, MachineType from backend.db_meta.exceptions import HostDoseNotExistInCmdbException from backend.db_meta.models import AppCache, BKCity +from backend.exceptions import ApiRequestError +from backend.utils.batch_request import batch_request from backend.utils.string import base64_encode @@ -185,3 +188,39 @@ def is_refer_spec(cls, spec_ids): @property def simple_desc(self): return model_to_dict(self) + + +class DeviceClass(models.Model): + device_type = models.CharField(max_length=128, help_text=_("机型类型")) + cpu = models.IntegerField(default=0, help_text=_("机型cpu")) + mem = models.IntegerField(default=0, help_text=_("机型内存")) + disk = models.IntegerField(default=0, help_text=_("机型磁盘")) + + class Meta: + verbose_name = verbose_name_plural = _("机型(DeviceClass)") + + def __str__(self): + return f"{self.device_type}({self.cpu}C{self.mem}G)" + + @classmethod + def sync_device_from_hcm(cls): + """从hcm同步机型，仅适用于内部环境""" + + # 默认过滤条件是支持申请的机型，且是常规项目 + rules = [ + {"field": "enable_apply", "operator": "equal", "value": True}, + {"field": "require_type", "operator": "equal", "value": 1}, + ] + dvc_filter = {"condition": "AND", "rules": rules} + # 全量请求hcm平台获取机型列表，若报错则忽略 + try: + device_class_list = batch_request(func=HCMApi.list_cvm_device, params={"filter": dvc_filter}) + except (Exception, ApiRequestError): + device_class_list = [] + + # 过滤不重复的机型，并且更新到数据库 + device_class_dict = { + d["device_type"]: cls(device_type=d["device_type"], cpu=d["cpu"], mem=d["mem"], disk=d["disk"]) + for d in device_class_list + } + cls.objects.bulk_create(list(device_class_dict.values())) diff --git a/dbm-ui/backend/db_services/dbresource/filters.py b/dbm-ui/backend/db_services/dbresource/filters.py index 4e0316a128..176854d094 100644 --- a/dbm-ui/backend/db_services/dbresource/filters.py +++ b/dbm-ui/backend/db_services/dbresource/filters.py @@ -12,6 +12,7 @@ from django.utils.translation import ugettext_lazy as _ from django_filters import rest_framework as filters +from backend.db_meta.models.machine import DeviceClass from backend.db_meta.models.spec import Spec @@ -47,3 +48,11 @@ class Meta: "spec_db_type", "spec_ids", ] + + +class DeviceClassFilter(filters.FilterSet): + device_type = filters.CharFilter(field_name="device_type", lookup_expr="icontains", label=_("机型名称")) + + class Meta: + model = DeviceClass + fields = ["device_type"] diff --git a/dbm-ui/backend/db_services/dbresource/serializers.py b/dbm-ui/backend/db_services/dbresource/serializers.py index 9d3edd49ec..b37bd30464 100644 --- a/dbm-ui/backend/db_services/dbresource/serializers.py +++ b/dbm-ui/backend/db_services/dbresource/serializers.py @@ -18,6 +18,7 @@ from backend.db_meta.enums import InstanceRole from backend.db_meta.enums.spec import SpecClusterType, SpecMachineType from backend.db_meta.models import Spec +from backend.db_meta.models.machine import DeviceClass from backend.db_services.dbresource import mock from backend.db_services.dbresource.constants import ResourceGroupByEnum, ResourceOperation from backend.db_services.dbresource.mock import ( @@ -191,7 +192,7 @@ class ResourceUpdateSerializer(serializers.Serializer): bk_host_ids = serializers.ListField(help_text=_("主机ID列表"), child=serializers.IntegerField()) labels = serializers.DictField(help_text=_("Labels"), required=False) for_biz = serializers.IntegerField(help_text=_("专用业务ID"), required=False) - resource_type = serializers.CharField(help_text=_("专属DB"), allow_blank=True, allow_null=True) + resource_type = serializers.CharField(help_text=_("专属DB"), allow_blank=True, allow_null=True, required=False) storage_device = serializers.JSONField(help_text=_("磁盘挂载点信息"), required=False) rack_id = serializers.CharField(help_text=_("机架ID"), required=False, allow_null=True, allow_blank=True) @@ -431,14 +432,10 @@ class Meta: swagger_schema_fields = {"example": {"spec1": 10, "spec2": 10}} -class ListCvmDeviceClassSerializer(serializers.Serializer): - offset = serializers.IntegerField(help_text=_("起始位置"), required=False, default=0) - limit = serializers.IntegerField(help_text=_("分页限制"), required=False, default=10, max_value=200) - name = serializers.CharField(help_text=_("名称"), required=False, default="", allow_null=True, allow_blank=True) - - -class ListCvmDeviceClassResponseSerializer(serializers.Serializer): +class ListCvmDeviceClassSerializer(serializers.ModelSerializer): class Meta: + model = DeviceClass + fields = "__all__" swagger_schema_fields = {"example": mock.DEVICE_CLASS_DATA} diff --git a/dbm-ui/backend/db_services/dbresource/views/resource.py b/dbm-ui/backend/db_services/dbresource/views/resource.py index cc97f81e3a..6484076f15 100644 --- a/dbm-ui/backend/db_services/dbresource/views/resource.py +++ b/dbm-ui/backend/db_services/dbresource/views/resource.py @@ -20,25 +20,26 @@ from backend import env from backend.bk_web import viewsets +from backend.bk_web.pagination import AuditedLimitOffsetPagination from backend.bk_web.swagger import common_swagger_auto_schema from backend.components import CCApi from backend.components.dbresource.client import DBResourceApi -from backend.components.hcm.client import HCMApi from backend.components.uwork.client import UWORKApi from backend.configuration.constants import SystemSettingsEnum from backend.configuration.models import SystemSettings from backend.db_meta.models import AppCache +from backend.db_meta.models.machine import DeviceClass from backend.db_services.dbresource.constants import ( GSE_AGENT_RUNNING_CODE, RESOURCE_IMPORT_EXPIRE_TIME, RESOURCE_IMPORT_TASK_FIELD, SWAGGER_TAG, ) +from backend.db_services.dbresource.filters import DeviceClassFilter from backend.db_services.dbresource.handlers import ResourceHandler from backend.db_services.dbresource.serializers import ( GetDiskTypeResponseSerializer, GetMountPointResponseSerializer, - ListCvmDeviceClassResponseSerializer, ListCvmDeviceClassSerializer, ListDBAHostsSerializer, ListSubzonesSerializer, @@ -63,7 +64,6 @@ from backend.db_services.ipchooser.handlers.topo_handler import TopoHandler from backend.db_services.ipchooser.query.resource import ResourceQueryHelper from backend.db_services.ipchooser.types import ScopeList -from backend.exceptions import ApiRequestError from backend.flow.consts import FAILED_STATES, SUCCEED_STATES from backend.flow.engine.controller.base import BaseController from backend.flow.models import FlowTree @@ -97,6 +97,8 @@ class DBResourceViewSet(viewsets.SystemViewSet): ("query_operation_list",): [ResourceActionPermission([ActionEnum.RESOURCE_OPERATION_VIEW])], } default_permission_class = [ResourceActionPermission([ActionEnum.RESOURCE_MANAGE])] + filter_class = None + pagination_class = None @common_swagger_auto_schema( operation_summary=_("资源池资源列表"), @@ -312,31 +314,20 @@ def get_subzones(self, request): @common_swagger_auto_schema( operation_summary=_("获取机型列表"), - request_body=ListCvmDeviceClassSerializer(), - responses={status.HTTP_200_OK: ListCvmDeviceClassResponseSerializer()}, tags=[SWAGGER_TAG], ) - @action(detail=False, methods=["POST"], serializer_class=ListCvmDeviceClassSerializer) + @action( + detail=False, + methods=["GET"], + serializer_class=ListCvmDeviceClassSerializer, + filter_class=DeviceClassFilter, + pagination_class=AuditedLimitOffsetPagination, + queryset=DeviceClass.objects.all(), + ) def get_device_class(self, request): - data = self.params_validate(self.get_serializer_class()) - page = {"start": data["offset"], "limit": data["limit"]} - # 默认过滤条件是支持申请的机型，且是常规项目 - rules = [ - {"field": "enable_apply", "operator": "equal", "value": True}, - {"field": "require_type", "operator": "equal", "value": 1}, - ] - # 如果有名称过滤，则加上 - if data.get("name"): - rules.append({"field": "device_type", "operator": "contains", "value": data["name"]}) - dvc_filter = {"condition": "AND", "rules": rules} - # 请求hcm平台获取机型列表，若报错则忽略 - try: - device_list = HCMApi.list_cvm_device(params={"filter": dvc_filter, "page": page}) - device_list["results"] = device_list.pop("info") - except (Exception, ApiRequestError): - device_list = {"count": 0, "results": []} - - return Response(device_list) + page_device_qs = self.paginate_queryset(self.filter_queryset(self.get_queryset())) + page_device_data = self.serializer_class(instance=page_device_qs, many=True).data + return self.paginator.get_paginated_response(data=page_device_data) @common_swagger_auto_schema( operation_summary=_("资源预申请"), diff --git a/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py b/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py index 11ca17de5a..8e70d8640e 100644 --- a/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py +++ b/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py @@ -276,25 +276,10 @@ def query_instance_backup_logs(self, end_time: datetime, **kwargs) -> Dict: backup_instance_record: Dict[str, Any] = {} def init_backup_record(log): - # 如果备份ID相同或者当前备份ID时间更接近，则忽略 - if backup_instance_record.get("backup_id") == log["backup_id"]: - return - if backup_instance_record.get("backup_consistent_time", "") > log["backup_consistent_time"]: + if backup_instance_record: return # 保留聚合需要的通用字段 - common_fields = [ - "backup_id", - "backup_type", - "cluster_id", - "cluster_address", - "bill_id", - "bk_biz_id", - "mysql_version", - "backup_begin_time", - "backup_end_time", - "backup_consistent_time", - "bk_cloud_id", - ] + common_fields = ["cluster_id", "cluster_address", "bk_biz_id", "bk_cloud_id"] for field in common_fields: backup_instance_record[field] = log[field] # 初始化实例的file_list @@ -303,15 +288,22 @@ def init_backup_record(log): def init_file_list(log): # 找到备份日志的priv文件 priv_file = next((file for file in log["file_list"] if file["file_type"] == "priv"), None) - inst = f"{log['backup_host']}:{log['backup_port']}" if not priv_file: return - priv_file.update(mysql_role=log["mysql_role"]) - # 覆盖更新，priv_file在同一个实例，同一个备份仅有一条记录的 - backup_instance_record["file_list"][inst] = priv_file + inst = f"{log['backup_host']}:{log['backup_port']}" + priv_file.update(mysql_role=log["mysql_role"], backup_consistent_time=log["backup_consistent_time"]) + # 更新实例的priv备份记录，如果有重复，则取时间更近的一份 + file_list = backup_instance_record["file_list"] + if inst not in file_list or log["backup_consistent_time"] > file_list[inst]["backup_consistent_time"]: + backup_instance_record["file_list"][inst] = priv_file + + # 不存在权限备份记录，返回为空 + if not backup_logs: + return backup_instance_record + # 初始化权限备份记录数据结构，填充priv文件列表 + init_backup_record(backup_logs[0]) for log in backup_logs: - init_backup_record(log) init_file_list(log) return backup_instance_record