feat(backend): 主机资源池流转 #7747

dbm-ui/backend/db_dirty/admin.py

@@ -15,6 +15,13 @@
 
 @admin.register(models.DirtyMachine)
 class DirtyMachineAdmin(admin.ModelAdmin):
-    list_display = ("ip", "bk_biz_id", "bk_host_id", "flow", "ticket")
-    list_filter = ("ip", "bk_biz_id", "bk_host_id", "flow", "ticket")
-    search_fields = ("ip", "bk_biz_id", "bk_host_id", "flow", "ticket")
+    list_display = ("ip", "bk_biz_id", "bk_host_id", "ticket", "pool")
+    list_filter = ("ip", "ticket", "pool")
+    search_fields = ("ip", "bk_biz_id", "bk_host_id")
+
+
+@admin.register(models.MachineEvent)
+class MachineEventAdmin(admin.ModelAdmin):
+    list_display = ("ip", "bk_biz_id", "bk_host_id", "event", "to", "ticket")
+    list_filter = ("ip", "bk_biz_id", "to")
+    search_fields = ("ip", "bk_biz_id", "bk_host_id")

dbm-ui/backend/db_dirty/constants.py

@@ -11,4 +11,42 @@
 
 from django.utils.translation import ugettext_lazy as _
 
-SWAGGER_TAG = _("污点池")
+from blue_krill.data_types.enum import EnumField, StructuredEnum
+
+SWAGGER_TAG = _("主机池")
+
+
+class PoolType(str, StructuredEnum):
+    # 池管理：污点池，故障池，待回收池
+    Dirty = EnumField("dirty", _("污点池"))
+    Fault = EnumField("fault", _("故障池"))
+    Recycle = EnumField("recycle", _("待回收池"))
+    # 资源池不由saas维护，单独由资源池服务维护
+    Resource = EnumField("resource", _("资源池"))
+    # 回收池表示已经挪到cc待回收，不在dbm流转
+    Recycled = EnumField("recycled", _("已回收"))
+    # 已部署表示主机正在使用
+    APPLY = EnumField("apply", _("已部署"))
+
+
+class MachineEventType(str, StructuredEnum):
+    ImportResource = EnumField("import_resource", _("导入资源池"))
+    ApplyResource = EnumField("apply_resource", _("申请资源"))
+    ReturnResource = EnumField("return_resource", _("退回资源"))
+    ToDirty = EnumField("to_dirty", _("转入污点池"))
+    ToRecycle = EnumField("to_recycle", _("转入待回收池"))
+    ToFault = EnumField("to_fault", _("转入故障池"))
+    UndoImport = EnumField("undo_import", _("撤销导入"))
+    Recycled = EnumField("recycled", _("回收"))
+
+
+MACHINE_EVENT__POOL_MAP = {
+    MachineEventType.ToDirty: PoolType.Dirty,
+    MachineEventType.ToRecycle: PoolType.Recycle,
+    MachineEventType.ToFault: PoolType.Fault,
+    MachineEventType.ImportResource: PoolType.Resource,
+    MachineEventType.ReturnResource: PoolType.Resource,
+    MachineEventType.Recycled: PoolType.Recycled,
+    MachineEventType.UndoImport: PoolType.Recycled,
+    MachineEventType.ApplyResource: PoolType.APPLY,
+}

dbm-ui/backend/db_dirty/exceptions.py

@@ -0,0 +1,24 @@
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.utils.translation import ugettext_lazy as _
+
+from backend.exceptions import AppBaseException, ErrorCode
+
+
+class DBDirtyPoolBaseException(AppBaseException):
+    MODULE_CODE = ErrorCode.DB_DIRTY_POOL_CODE
+    MESSAGE = _("主机池异常")
+
+
+class PoolTransferException(DBDirtyPoolBaseException):
+    ERROR_CODE = "001"
+    MESSAGE = _("主机池转移异常")
+    MESSAGE_TPL = _("主机池转移异常")

dbm-ui/backend/db_dirty/filters.py

@@ -13,7 +13,7 @@
 from django_filters import rest_framework as filters
 from django_filters.filters import BaseInFilter, NumberFilter
 
-from backend.db_dirty.models import DirtyMachine
+from backend.db_dirty.models import DirtyMachine, MachineEvent
 
 
 class NumberInFilter(BaseInFilter, NumberFilter):
@@ -50,3 +50,29 @@ def filter_task_ids(self, queryset, name, value):
     class Meta:
         model = DirtyMachine
         fields = ["ticket_types", "ticket_ids", "task_ids", "operator", "ip"]
+
+
+class MachineEventFilter(filters.FilterSet):
+    operator = filters.CharFilter(field_name="creator", lookup_expr="icontains", label=_("操作者"))
+    bk_biz_id = filters.NumberFilter(field_name="bk_biz_id", label=_("业务"))
+    event = filters.CharFilter(field_name="event", lookup_expr="exact", label=_("事件类型"))
+    ips = filters.CharFilter(field_name="ip", method="filter_ips", label=_("过滤IP"))
+
+    def filter_ips(self, queryset, name, value):
+        return queryset.filter(ip__in=value.split(","))
+
+    class Meta:
+        model = MachineEvent
+        fields = ["operator", "bk_biz_id", "event", "ips"]
+
+
+class DirtyMachinePoolFilter(filters.FilterSet):
+    ips = filters.CharFilter(field_name="ip", method="filter_ips", label=_("过滤IP"))
+    pool = filters.CharFilter(field_name="pool", lookup_expr="exact", label=_("主机池类型"))
+
+    def filter_ips(self, queryset, name, value):
+        return queryset.filter(ip__in=value.split(","))
+
+    class Meta:
+        model = DirtyMachine
+        fields = ["ips", "pool"]

dbm-ui/backend/db_dirty/handlers.py

@@ -17,19 +17,20 @@
 
 from backend import env
 from backend.components import CCApi
-from backend.components.dbresource.client import DBResourceApi
 from backend.configuration.constants import SystemSettingsEnum
 from backend.configuration.models import SystemSettings
-from backend.db_dirty.models import DirtyMachine
+from backend.db_dirty.constants import MachineEventType, PoolType
+from backend.db_dirty.exceptions import PoolTransferException
+from backend.db_dirty.models import DirtyMachine, MachineEvent
 from backend.db_meta.models import AppCache
 from backend.db_services.ipchooser.constants import IDLE_HOST_MODULE
 from backend.db_services.ipchooser.handlers.topo_handler import TopoHandler
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
 from backend.flow.consts import FAILED_STATES
 from backend.flow.utils.cc_manage import CcManage
 from backend.ticket.builders import BuilderFactory
+from backend.ticket.builders.common.base import fetch_apply_hosts
 from backend.ticket.models import Flow, Ticket
-from backend.utils.basic import get_target_items_from_details
 from backend.utils.batch_request import request_multi_thread
 
 logger = logging.getLogger("root")
@@ -41,28 +42,34 @@ class DBDirtyMachineHandler(object):
     """
 
     @classmethod
-    def transfer_dirty_machines(cls, bk_host_ids: List[int]):
+    def transfer_hosts_to_pool(cls, operator: str, bk_host_ids: List[int], source: PoolType, target: PoolType):
         """
-        将污点主机转移待回收模块，并从资源池移除
+        将主机转移待回收/故障池模块
         @param bk_host_ids: 主机列表
+        @param operator: 操作者
+        @param source: 主机来源
+        @param target: 主机去向
         """
-        # 将主机移动到待回收模块
-        dirty_machines = DirtyMachine.objects.filter(bk_host_id__in=bk_host_ids)
-        bk_biz_id__host_ids = defaultdict(list)
-        for machine in dirty_machines:
-            bk_biz_id__host_ids[machine.bk_biz_id].append(machine.bk_host_id)
+        # 将主机按照业务分组
+        recycle_hosts = DirtyMachine.objects.filter(bk_host_id__in=bk_host_ids)
+        biz_grouped_recycle_hosts = itertools.groupby(recycle_hosts, key=lambda x: x.bk_biz_id)
 
-        for bk_biz_id, bk_host_ids in bk_biz_id__host_ids.items():
-            CcManage(int(bk_biz_id), "").recycle_host(bk_host_ids)
-
-        # 删除污点池记录，并从资源池移除(忽略删除错误，因为机器可能不来自资源池)
-        dirty_machines.delete()
-        DBResourceApi.resource_delete(params={"bk_host_ids": bk_host_ids}, raise_exception=False)
+        for bk_biz_id, hosts in biz_grouped_recycle_hosts:
+            hosts = [{"bk_host_id": host.bk_host_id} for host in hosts]
+            # 故障池 ---> 待回收
+            if source == PoolType.Recycle and target == PoolType.Recycled:
+                CcManage(bk_biz_id, "").recycle_host([h["bk_host_id"] for h in hosts])
+                MachineEvent.host_event_trigger(bk_biz_id, hosts, event=MachineEventType.Recycled, operator=operator)
+            # 待回收 ---> 回收
+            elif source == PoolType.Fault and target == PoolType.Recycle:
+                MachineEvent.host_event_trigger(bk_biz_id, hosts, event=MachineEventType.ToRecycle, operator=operator)
+            else:
+                raise PoolTransferException(_("{}--->{}转移不合法").format(source, target))
 
     @classmethod
     def query_dirty_machine_records(cls, bk_host_ids: List[int]):
         """
-        查询污点池主机信息
+        查询污点池主机信息 TODO: 污点池废弃，代码将被移除
         @param bk_host_ids: 主机列表
         """
 
@@ -165,7 +172,7 @@ def get_module_data(data):
     @classmethod
     def insert_dirty_machines(cls, bk_biz_id: int, bk_host_ids: List[Dict[str, Any]], ticket: Ticket, flow: Flow):
         """
-        将机器导入到污点池中
+        将机器导入到污点池中 TODO: 污点池废弃，代码将被移除
         @param bk_biz_id: 业务ID
         @param bk_host_ids: 主机列表
         @param ticket: 关联的单据
@@ -223,18 +230,6 @@ def insert_dirty_machines(cls, bk_biz_id: int, bk_host_ids: List[Dict[str, Any]]
             ]
         )
 
-    @classmethod
-    def remove_dirty_machines(cls, bk_host_ids: List[Dict[str, Any]]):
-        """
-        将机器从污点池挪走，一般是重试后会调用此函数。
-        这里只用删除记录，无需做其他挪模块的操作，原因如下：
-        1. 如果重试依然失败，则机器会重新回归污点池，模块不变
-        2. 如果重试成功，则机器已经由flow挪到了对应的DB模块
-        3. 如果手动处理，则机器会被挪到待回收模块
-        @param bk_host_ids: 主机列表
-        """
-        DirtyMachine.objects.filter(bk_host_id__in=bk_host_ids).delete()
-
     @classmethod
     def handle_dirty_machine(cls, ticket_id, root_id, origin_tree_status, target_tree_status):
         """处理执行失败/重试成功涉及的污点池机器"""
@@ -243,28 +238,26 @@ def handle_dirty_machine(cls, ticket_id, root_id, origin_tree_status, target_tre
 
         try:
             ticket = Ticket.objects.get(id=ticket_id)
-            flow = Flow.objects.get(flow_obj_id=root_id)
             # 如果不是部署类单据，则无需处理
             if ticket.ticket_type not in BuilderFactory.apply_ticket_type:
                 return
         except (Ticket.DoesNotExist, Flow.DoesNotExist, ValueError):
             return
 
         # 如果初始状态是失败，则证明是重试，将机器从污点池中移除
-        bk_host_ids = get_target_items_from_details(
-            obj=ticket.details, match_keys=["host_id", "bk_host_id", "bk_host_ids"]
-        )
+        hosts = fetch_apply_hosts(ticket.details)
+        bk_host_ids = [h["bk_host_id"] for h in hosts]
 
         if not bk_host_ids:
             return
 
+        # 如果是原状态失败，则证明是重试，这里只用删除记录
         if origin_tree_status in FAILED_STATES:
             logger.info(_("【污点池】主机列表:{} 将从污点池挪出").format(bk_host_ids))
-            DBDirtyMachineHandler.remove_dirty_machines(bk_host_ids)
+            DirtyMachine.objects.filter(bk_host_id__in=bk_host_ids).delete()
 
         # 如果是目标状态失败，则证明是执行失败，将机器加入污点池
         if target_tree_status in FAILED_STATES:
-            logger.info(_("【污点池】单据-{}：任务-{}执行失败，主机列表:{}挪到污点池").format(ticket_id, root_id, bk_host_ids))
-            DBDirtyMachineHandler.insert_dirty_machines(
-                bk_biz_id=ticket.bk_biz_id, bk_host_ids=bk_host_ids, ticket=ticket, flow=flow
-            )
+            logger.info(_("【污点池】主机列表:{} 移入污点池").format(bk_host_ids))
+            hosts = fetch_apply_hosts(ticket.details)
+            MachineEvent.host_event_trigger(ticket.bk_biz_id, hosts, MachineEventType.ToDirty, ticket.creator, ticket)

dbm-ui/backend/db_dirty/migrations/0003_auto_20240925_1526.py

@@ -0,0 +1,126 @@
+# Generated by Django 3.2.25 on 2024-09-25 07:26
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+from backend.db_dirty.constants import MachineEventType, PoolType
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("ticket", "0012_alter_ticket_remark"),
+        ("db_dirty", "0002_alter_dirtymachine_options"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="dirtymachine",
+            name="flow",
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="bk_cpu",
+            field=models.IntegerField(default=0, help_text="cpu"),
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="bk_disk",
+            field=models.IntegerField(default=0, help_text="磁盘"),
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="bk_mem",
+            field=models.IntegerField(default=0, help_text="内存"),
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="city",
+            field=models.CharField(blank=True, default="", help_text="城市", max_length=128, null=True),
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="device_class",
+            field=models.CharField(blank=True, default="", help_text="机架", max_length=128, null=True),
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="os_name",
+            field=models.CharField(blank=True, default="", help_text="操作系统", max_length=128, null=True),
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="pool",
+            field=models.CharField(
+                choices=PoolType.get_choices(),
+                default="dirty",
+                help_text="池类型",
+                max_length=128,
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="rack_id",
+            field=models.CharField(blank=True, default="", help_text="机架", max_length=128, null=True),
+        ),
+        migrations.AddField(
+            model_name="dirtymachine",
+            name="sub_zone",
+            field=models.CharField(blank=True, default="", help_text="园区", max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name="dirtymachine",
+            name="ticket",
+            field=models.ForeignKey(
+                blank=True,
+                help_text="关联单据",
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="ticket.ticket",
+            ),
+        ),
+        migrations.CreateModel(
+            name="MachineEvent",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("creator", models.CharField(max_length=64, verbose_name="创建人")),
+                ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")),
+                ("updater", models.CharField(max_length=64, verbose_name="修改人")),
+                ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")),
+                ("bk_biz_id", models.IntegerField(default=0, help_text="业务ID")),
+                ("ip", models.CharField(help_text="主机IP", max_length=128)),
+                ("bk_host_id", models.PositiveBigIntegerField(help_text="主机ID")),
+                (
+                    "event",
+                    models.CharField(
+                        choices=MachineEventType.get_choices(),
+                        help_text="事件类型",
+                        max_length=128,
+                    ),
+                ),
+                (
+                    "to",
+                    models.CharField(
+                        choices=PoolType.get_choices(),
+                        help_text="资源流向",
+                        max_length=128,
+                    ),
+                ),
+                (
+                    "ticket",
+                    models.ForeignKey(
+                        blank=True,
+                        help_text="关联单据",
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="ticket.ticket",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "机器事件记录",
+                "verbose_name_plural": "机器事件记录",
+            },
+        ),
+    ]