feat(backend): 机型表改为自平台维护 TencentBlueKing#8043

dbm-ui/backend/db_meta/admin.py

@@ -126,6 +126,12 @@ class MachineAdmin(admin.ModelAdmin):
     search_fields = ("ip", "db_module_id")
 
 
+@admin.register(models.machine.DeviceClass)
+class DeviceClassAdmin(admin.ModelAdmin):
+    list_display = ("device_type", "cpu", "mem", "disk")
+    search_fields = ("device_type",)
+
+
 @admin.register(models.storage_instance_tuple.StorageInstanceTuple)
 class StorageInstanceTupleAdmin(DynamicRawIDMixin, admin.ModelAdmin):
     list_display = (

dbm-ui/backend/db_meta/enums/spec.py

@@ -51,6 +51,10 @@ class SpecMachineType(str, StructuredEnum):
     MONGODB = EnumField("mongodb", _("mongodb"))
     MONOG_CONFIG = EnumField("mongo_config", _("mongo_config"))
 
+    DORIS_FOLLOWER = EnumField("doris_follower", _("doris_follower"))
+    DORIS_OBSERVER = EnumField("doris_observer", _("doris_observer"))
+    DORIS_BACKEND = EnumField("doris_backend", _("doris_backend"))
+
 
 # TODO: 规格迁移脚本函数，迁移完成后删除
 def migrate_spec():

dbm-ui/backend/db_meta/migrations/0044_deviceclass.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.25 on 2024-11-19 07:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("db_meta", "0043_auto_20241014_1042"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DeviceClass",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("device_type", models.CharField(help_text="机型类型", max_length=128)),
+                ("cpu", models.IntegerField(default=0, help_text="机型cpu")),
+                ("mem", models.IntegerField(default=0, help_text="机型内存")),
+                ("disk", models.IntegerField(default=0, help_text="机型磁盘")),
+            ],
+            options={
+                "verbose_name": "机型(DeviceClass)",
+                "verbose_name_plural": "机型(DeviceClass)",
+            },
+        ),
+    ]

dbm-ui/backend/db_meta/models/machine.py

@@ -19,10 +19,13 @@
 
 from backend.bk_web.models import AuditedModel
 from backend.components import CCApi
+from backend.components.hcm.client import HCMApi
 from backend.constants import CommonHostDBMeta
 from backend.db_meta.enums import AccessLayer, ClusterType, MachineType
 from backend.db_meta.exceptions import HostDoseNotExistInCmdbException
 from backend.db_meta.models import AppCache, BKCity
+from backend.exceptions import ApiRequestError
+from backend.utils.batch_request import batch_request
 from backend.utils.string import base64_encode
 
 
@@ -185,3 +188,39 @@ def is_refer_spec(cls, spec_ids):
     @property
     def simple_desc(self):
         return model_to_dict(self)
+
+
+class DeviceClass(models.Model):
+    device_type = models.CharField(max_length=128, help_text=_("机型类型"))
+    cpu = models.IntegerField(default=0, help_text=_("机型cpu"))
+    mem = models.IntegerField(default=0, help_text=_("机型内存"))
+    disk = models.IntegerField(default=0, help_text=_("机型磁盘"))
+
+    class Meta:
+        verbose_name = verbose_name_plural = _("机型(DeviceClass)")
+
+    def __str__(self):
+        return f"{self.device_type}({self.cpu}C{self.mem}G)"
+
+    @classmethod
+    def sync_device_from_hcm(cls):
+        """从hcm同步机型，仅适用于内部环境"""
+
+        # 默认过滤条件是支持申请的机型，且是常规项目
+        rules = [
+            {"field": "enable_apply", "operator": "equal", "value": True},
+            {"field": "require_type", "operator": "equal", "value": 1},
+        ]
+        dvc_filter = {"condition": "AND", "rules": rules}
+        # 全量请求hcm平台获取机型列表，若报错则忽略
+        try:
+            device_class_list = batch_request(func=HCMApi.list_cvm_device, params={"filter": dvc_filter})
+        except (Exception, ApiRequestError):
+            device_class_list = []
+
+        # 过滤不重复的机型，并且更新到数据库
+        device_class_dict = {
+            d["device_type"]: cls(device_type=d["device_type"], cpu=d["cpu"], mem=d["mem"], disk=d["disk"])
+            for d in device_class_list
+        }
+        cls.objects.bulk_create(list(device_class_dict.values()))

dbm-ui/backend/db_services/dbresource/filters.py

@@ -12,6 +12,7 @@
 from django.utils.translation import ugettext_lazy as _
 from django_filters import rest_framework as filters
 
+from backend.db_meta.models.machine import DeviceClass
 from backend.db_meta.models.spec import Spec
 
 
@@ -47,3 +48,11 @@ class Meta:
             "spec_db_type",
             "spec_ids",
         ]
+
+
+class DeviceClassFilter(filters.FilterSet):
+    device_type = filters.CharFilter(field_name="device_type", lookup_expr="icontains", label=_("机型名称"))
+
+    class Meta:
+        model = DeviceClass
+        fields = ["device_type"]

dbm-ui/backend/db_services/dbresource/serializers.py

@@ -18,6 +18,7 @@
 from backend.db_meta.enums import InstanceRole
 from backend.db_meta.enums.spec import SpecClusterType, SpecMachineType
 from backend.db_meta.models import Spec
+from backend.db_meta.models.machine import DeviceClass
 from backend.db_services.dbresource import mock
 from backend.db_services.dbresource.constants import ResourceGroupByEnum, ResourceOperation
 from backend.db_services.dbresource.mock import (
@@ -191,7 +192,7 @@ class ResourceUpdateSerializer(serializers.Serializer):
     bk_host_ids = serializers.ListField(help_text=_("主机ID列表"), child=serializers.IntegerField())
     labels = serializers.DictField(help_text=_("Labels"), required=False)
     for_biz = serializers.IntegerField(help_text=_("专用业务ID"), required=False)
-    resource_type = serializers.CharField(help_text=_("专属DB"), allow_blank=True, allow_null=True)
+    resource_type = serializers.CharField(help_text=_("专属DB"), allow_blank=True, allow_null=True, required=False)
     storage_device = serializers.JSONField(help_text=_("磁盘挂载点信息"), required=False)
     rack_id = serializers.CharField(help_text=_("机架ID"), required=False, allow_null=True, allow_blank=True)
 
@@ -431,14 +432,10 @@ class Meta:
         swagger_schema_fields = {"example": {"spec1": 10, "spec2": 10}}
 
 
-class ListCvmDeviceClassSerializer(serializers.Serializer):
-    offset = serializers.IntegerField(help_text=_("起始位置"), required=False, default=0)
-    limit = serializers.IntegerField(help_text=_("分页限制"), required=False, default=10, max_value=200)
-    name = serializers.CharField(help_text=_("名称"), required=False, default="", allow_null=True, allow_blank=True)
-
-
-class ListCvmDeviceClassResponseSerializer(serializers.Serializer):
+class ListCvmDeviceClassSerializer(serializers.ModelSerializer):
     class Meta:
+        model = DeviceClass
+        fields = "__all__"
         swagger_schema_fields = {"example": mock.DEVICE_CLASS_DATA}
 
 

dbm-ui/backend/db_services/dbresource/views/resource.py

@@ -20,25 +20,26 @@
 
 from backend import env
 from backend.bk_web import viewsets
+from backend.bk_web.pagination import AuditedLimitOffsetPagination
 from backend.bk_web.swagger import common_swagger_auto_schema
 from backend.components import CCApi
 from backend.components.dbresource.client import DBResourceApi
-from backend.components.hcm.client import HCMApi
 from backend.components.uwork.client import UWORKApi
 from backend.configuration.constants import SystemSettingsEnum
 from backend.configuration.models import SystemSettings
 from backend.db_meta.models import AppCache
+from backend.db_meta.models.machine import DeviceClass
 from backend.db_services.dbresource.constants import (
     GSE_AGENT_RUNNING_CODE,
     RESOURCE_IMPORT_EXPIRE_TIME,
     RESOURCE_IMPORT_TASK_FIELD,
     SWAGGER_TAG,
 )
+from backend.db_services.dbresource.filters import DeviceClassFilter
 from backend.db_services.dbresource.handlers import ResourceHandler
 from backend.db_services.dbresource.serializers import (
     GetDiskTypeResponseSerializer,
     GetMountPointResponseSerializer,
-    ListCvmDeviceClassResponseSerializer,
     ListCvmDeviceClassSerializer,
     ListDBAHostsSerializer,
     ListSubzonesSerializer,
@@ -63,7 +64,6 @@
 from backend.db_services.ipchooser.handlers.topo_handler import TopoHandler
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
 from backend.db_services.ipchooser.types import ScopeList
-from backend.exceptions import ApiRequestError
 from backend.flow.consts import FAILED_STATES, SUCCEED_STATES
 from backend.flow.engine.controller.base import BaseController
 from backend.flow.models import FlowTree
@@ -97,6 +97,8 @@ class DBResourceViewSet(viewsets.SystemViewSet):
         ("query_operation_list",): [ResourceActionPermission([ActionEnum.RESOURCE_OPERATION_VIEW])],
     }
     default_permission_class = [ResourceActionPermission([ActionEnum.RESOURCE_MANAGE])]
+    filter_class = None
+    pagination_class = None
 
     @common_swagger_auto_schema(
         operation_summary=_("资源池资源列表"),
@@ -312,31 +314,20 @@ def get_subzones(self, request):
 
     @common_swagger_auto_schema(
         operation_summary=_("获取机型列表"),
-        request_body=ListCvmDeviceClassSerializer(),
-        responses={status.HTTP_200_OK: ListCvmDeviceClassResponseSerializer()},
         tags=[SWAGGER_TAG],
     )
-    @action(detail=False, methods=["POST"], serializer_class=ListCvmDeviceClassSerializer)
+    @action(
+        detail=False,
+        methods=["GET"],
+        serializer_class=ListCvmDeviceClassSerializer,
+        filter_class=DeviceClassFilter,
+        pagination_class=AuditedLimitOffsetPagination,
+        queryset=DeviceClass.objects.all(),
+    )
     def get_device_class(self, request):
-        data = self.params_validate(self.get_serializer_class())
-        page = {"start": data["offset"], "limit": data["limit"]}
-        # 默认过滤条件是支持申请的机型，且是常规项目
-        rules = [
-            {"field": "enable_apply", "operator": "equal", "value": True},
-            {"field": "require_type", "operator": "equal", "value": 1},
-        ]
-        # 如果有名称过滤，则加上
-        if data.get("name"):
-            rules.append({"field": "device_type", "operator": "contains", "value": data["name"]})
-        dvc_filter = {"condition": "AND", "rules": rules}
-        # 请求hcm平台获取机型列表，若报错则忽略
-        try:
-            device_list = HCMApi.list_cvm_device(params={"filter": dvc_filter, "page": page})
-            device_list["results"] = device_list.pop("info")
-        except (Exception, ApiRequestError):
-            device_list = {"count": 0, "results": []}
-
-        return Response(device_list)
+        page_device_qs = self.paginate_queryset(self.filter_queryset(self.get_queryset()))
+        page_device_data = self.serializer_class(instance=page_device_qs, many=True).data
+        return self.paginator.get_paginated_response(data=page_device_data)
 
     @common_swagger_auto_schema(
         operation_summary=_("资源预申请"),

dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py

@@ -276,25 +276,10 @@ def query_instance_backup_logs(self, end_time: datetime, **kwargs) -> Dict:
         backup_instance_record: Dict[str, Any] = {}
 
         def init_backup_record(log):
-            # 如果备份ID相同或者当前备份ID时间更接近，则忽略
-            if backup_instance_record.get("backup_id") == log["backup_id"]:
-                return
-            if backup_instance_record.get("backup_consistent_time", "") > log["backup_consistent_time"]:
+            if backup_instance_record:
                 return
             # 保留聚合需要的通用字段
-            common_fields = [
-                "backup_id",
-                "backup_type",
-                "cluster_id",
-                "cluster_address",
-                "bill_id",
-                "bk_biz_id",
-                "mysql_version",
-                "backup_begin_time",
-                "backup_end_time",
-                "backup_consistent_time",
-                "bk_cloud_id",
-            ]
+            common_fields = ["cluster_id", "cluster_address", "bk_biz_id", "bk_cloud_id"]
             for field in common_fields:
                 backup_instance_record[field] = log[field]
             # 初始化实例的file_list
@@ -303,15 +288,22 @@ def init_backup_record(log):
         def init_file_list(log):
             # 找到备份日志的priv文件
             priv_file = next((file for file in log["file_list"] if file["file_type"] == "priv"), None)
-            inst = f"{log['backup_host']}:{log['backup_port']}"
             if not priv_file:
                 return
-            priv_file.update(mysql_role=log["mysql_role"])
-            # 覆盖更新，priv_file在同一个实例，同一个备份仅有一条记录的
-            backup_instance_record["file_list"][inst] = priv_file
+            inst = f"{log['backup_host']}:{log['backup_port']}"
+            priv_file.update(mysql_role=log["mysql_role"], backup_consistent_time=log["backup_consistent_time"])
+            # 更新实例的priv备份记录，如果有重复，则取时间更近的一份
+            file_list = backup_instance_record["file_list"]
+            if inst not in file_list or log["backup_consistent_time"] > file_list[inst]["backup_consistent_time"]:
+                backup_instance_record["file_list"][inst] = priv_file
+
+        # 不存在权限备份记录，返回为空
+        if not backup_logs:
+            return backup_instance_record
 
+        # 初始化权限备份记录数据结构，填充priv文件列表
+        init_backup_record(backup_logs[0])
         for log in backup_logs:
-            init_backup_record(log)
             init_file_list(log)
 
         return backup_instance_record