feat: support notify idle app module envs to developers

apiserver/paasng/paas_wl/bk_app/cnative/specs/crd/bk_app.py

@@ -71,7 +71,7 @@ class HTTPGetAction(BaseModel):
     host: Optional[str] = None
     path: Optional[str] = None
     httpHeaders: List[HTTPHeader] = Field(default_factory=list)
-    scheme: Optional[Literal["HTTP", "HTTPS"]] = None
+    scheme: Optional[Literal["HTTP", "HTTPS"]] = "HTTP"
 
 
 class TCPSocketAction(BaseModel):

apiserver/paasng/paasng/plat_admin/admin42/serializers/application.py

@@ -26,7 +26,7 @@
 from paasng.plat_admin.admin42.serializers.module import ModuleSLZ
 from paasng.platform.applications.constants import ApplicationType
 from paasng.platform.applications.models import Application
-from paasng.platform.evaluation.constants import CollectionTaskStatus
+from paasng.platform.evaluation.constants import BatchTaskStatus
 from paasng.platform.evaluation.models import AppOperationReport, AppOperationReportCollectionTask
 from paasng.utils.datetime import humanize_timedelta
 from paasng.utils.models import OrderByField
@@ -142,7 +142,7 @@ def get_duration(self, obj: AppOperationReportCollectionTask) -> str:
         return humanize_timedelta(obj.end_at - obj.start_at)
 
     def get_status(self, obj: AppOperationReportCollectionTask) -> str:
-        return CollectionTaskStatus.get_choice_label(obj.status)
+        return BatchTaskStatus.get_choice_label(obj.status)
 
 
 class AppOperationReportOutputSLZ(serializers.Serializer):

apiserver/paasng/paasng/plat_admin/admin42/views/applications.py

@@ -236,7 +236,7 @@ def get_sheet_data(self) -> List[List[str]]:
                     rp.latest_operator if rp.latest_operator else "--",
                     rp.latest_operated_at.strftime("%Y-%m-%d %H:%M:%S") if rp.latest_operated_at else "--",
                     str(OperationIssueType.get_choice_label(rp.issue_type)),
-                    ", ".join(rp.evaluation_result["issues"]) if rp.evaluation_result else "--",
+                    ", ".join(rp.evaluate_result["issues"]) if rp.evaluate_result else "--",
                     administrators,
                 ]
             )

apiserver/paasng/paasng/platform/bkapp_model/importer/serializers.py

@@ -196,7 +196,7 @@ class HTTPGetActionInputSLZ(serializers.Serializer):
     path = serializers.CharField(help_text="探活路径", max_length=128)
     host = serializers.CharField(help_text="主机名", required=False, allow_null=True)
     httpHeaders = serializers.ListField(help_text="HTTP 请求标头", required=False, child=HTTPHeaderInputSLZ())
-    scheme = serializers.CharField(help_text="http/https", required=False)
+    scheme = serializers.CharField(help_text="http/https", required=False, default="HTTP")
 
     def to_internal_value(self, data) -> bk_app.HTTPGetAction:
         d = super().to_internal_value(data)

apiserver/paasng/paasng/platform/bkapp_model/serializers.py

@@ -85,7 +85,7 @@ class HTTPGetProbeActionSLZ(serializers.Serializer):
     path = serializers.CharField(help_text="探活路径", max_length=128)
     host = serializers.CharField(help_text="主机名", required=False, allow_null=True)
     http_headers = serializers.ListField(help_text="HTTP 请求标头", required=False, child=HTTPHeaderSLZ())
-    scheme = serializers.CharField(help_text="http/https", required=False)
+    scheme = serializers.CharField(help_text="http/https", required=False, default="HTTP")
 
 
 class ProbeSLZ(serializers.Serializer):

apiserver/paasng/paasng/platform/declarative/deployment/validations/v2.py

@@ -85,7 +85,7 @@ class HTTPGetSLZ(serializers.Serializer):
     http_headers = serializers.ListField(
         help_text="HTTP 请求标头", required=False, child=HTTPHeaderSLZ(), source="httpHeaders"
     )
-    scheme = serializers.CharField(help_text="连接主机的方案", required=False)
+    scheme = serializers.CharField(help_text="连接主机的方案", required=False, default="HTTP")
 
 
 class TCPSocketSLZ(serializers.Serializer):

apiserver/paasng/paasng/platform/evaluation/constants.py

@@ -19,13 +19,19 @@
 from django.utils.translation import gettext_lazy as _
 
 
-class CollectionTaskStatus(str, StructuredEnum):
-    """采集任务状态"""
+class BatchTaskStatus(str, StructuredEnum):
+    """任务状态"""
 
     RUNNING = EnumField("running", label=_("运行中"))
     FINISHED = EnumField("finished", label=_("已完成"))
 
 
+class EmailNotificationType(str, StructuredEnum):
+    """邮件通知类型"""
+
+    IDLE_APP_MODULE_ENVS = EnumField("idle_app_module_envs", label=_("闲置应用模块"))
+
+
 class EmailReceiverType(str, StructuredEnum):
     """邮件接收者类型"""
 

apiserver/paasng/paasng/platform/evaluation/management/commands/send_idle_email_to_app_developer.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+
+"""Collect application resource usage report
+
+Examples:
+
+    # 仅通知指定的应用的管理员 & 开发者
+    python manage.py send_idle_email_to_app_developer --codes app-code-1 app-code-2
+
+    # 全量应用通知
+    python manage.py send_idle_email_to_app_developer --all
+
+    # 全量应用 + 异步执行
+    python manage.py send_idle_email_to_app_developer --all --async
+
+    # 仅通知指定的应用的指定管理员 & 开发者
+    python manage.py send_idle_email_to_app_developer --codes app-code-1 --only_specified_users user-1 user-2
+"""
+from django.core.management.base import BaseCommand
+
+from paasng.platform.evaluation.tasks import send_idle_email_to_app_developers
+
+
+class Command(BaseCommand):
+    help = "Send application's idle module env infos to application's developers by emails"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--codes", dest="app_codes", default=[], nargs="*", help="应用 Code 列表")
+        parser.add_argument(
+            "--only_specified_users", dest="only_specified_users", default=[], nargs="*", help="只发送给指定的用户"
+        )
+        parser.add_argument("--all", dest="notify_all", default=False, action="store_true", help="全量应用通知")
+        parser.add_argument("--async", dest="async_run", default=False, action="store_true", help="异步执行")
+
+    def handle(self, app_codes, only_specified_users, notify_all, async_run, *args, **options):
+        if not (notify_all or app_codes):
+            raise ValueError("please specify --codes or --all")
+
+        if async_run:
+            send_idle_email_to_app_developers.delay(app_codes, only_specified_users)
+        else:
+            send_idle_email_to_app_developers(app_codes, only_specified_users)

apiserver/paasng/paasng/platform/evaluation/migrations/0002_auto_20240530_1152.py

@@ -21,7 +21,7 @@ class Migration(migrations.Migration):
                 ('succeed_count', models.IntegerField(default=0, verbose_name='采集成功数')),
                 ('failed_count', models.IntegerField(default=0, verbose_name='采集失败数')),
                 ('failed_app_codes', models.JSONField(default=list, verbose_name='采集失败应用 Code 列表')),
-                ('status', models.CharField(choices=[('running', '运行中'), ('finished', '已完成')], default=paasng.platform.evaluation.constants.CollectionTaskStatus['RUNNING'], max_length=32, verbose_name='任务状态')),
+                ('status', models.CharField(choices=[('running', '运行中'), ('finished', '已完成')], default=paasng.platform.evaluation.constants.BatchTaskStatus['RUNNING'], max_length=32, verbose_name='任务状态')),
             ],
         ),
         migrations.RemoveField(

apiserver/paasng/paasng/platform/evaluation/migrations/0003_appoperationemailnotificationtask.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.25 on 2024-07-03 11:01
+
+from django.db import migrations, models
+import paasng.platform.evaluation.constants
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('evaluation', '0002_auto_20240530_1152'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AppOperationEmailNotificationTask',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('start_at', models.DateTimeField(auto_now_add=True, verbose_name='任务开始时间')),
+                ('end_at', models.DateTimeField(null=True, verbose_name='任务结束时间')),
+                ('total_count', models.IntegerField(default=0, verbose_name='应用总数')),
+                ('succeed_count', models.IntegerField(default=0, verbose_name='采集成功数')),
+                ('failed_count', models.IntegerField(default=0, verbose_name='采集失败数')),
+                ('failed_usernames', models.JSONField(default=list, verbose_name='通知失败的应用数量')),
+                ('notification_type', models.CharField(max_length=64, verbose_name='通知类型')),
+                ('status', models.CharField(choices=[('running', '运行中'), ('finished', '已完成')], default=paasng.platform.evaluation.constants.BatchTaskStatus['RUNNING'], max_length=32, verbose_name='任务状态')),
+            ],
+        ),
+    ]

apiserver/paasng/paasng/platform/evaluation/models.py

@@ -18,7 +18,7 @@
 from django.db import models
 
 from paasng.platform.applications.models import Application
-from paasng.platform.evaluation.constants import CollectionTaskStatus, OperationIssueType
+from paasng.platform.evaluation.constants import BatchTaskStatus, OperationIssueType
 
 
 class AppOperationReportCollectionTask(models.Model):
@@ -33,8 +33,8 @@ class AppOperationReportCollectionTask(models.Model):
     status = models.CharField(
         verbose_name="任务状态",
         max_length=32,
-        choices=CollectionTaskStatus.get_choices(),
-        default=CollectionTaskStatus.RUNNING,
+        choices=BatchTaskStatus.get_choices(),
+        default=BatchTaskStatus.RUNNING,
     )
 
 
@@ -68,3 +68,21 @@ class AppOperationReport(models.Model):
     issue_type = models.CharField(verbose_name="问题类型", default=OperationIssueType.NONE, max_length=32)
     evaluate_result = models.JSONField(verbose_name="评估结果", default=dict)
     collected_at = models.DateTimeField(verbose_name="采集时间")
+
+
+class AppOperationEmailNotificationTask(models.Model):
+    """应用运营报告邮件通知任务"""
+
+    start_at = models.DateTimeField(verbose_name="任务开始时间", auto_now_add=True)
+    end_at = models.DateTimeField(verbose_name="任务结束时间", null=True)
+    total_count = models.IntegerField(verbose_name="应用总数", default=0)
+    succeed_count = models.IntegerField(verbose_name="采集成功数", default=0)
+    failed_count = models.IntegerField(verbose_name="采集失败数", default=0)
+    failed_usernames = models.JSONField(verbose_name="通知失败的应用数量", default=list)
+    notification_type = models.CharField(verbose_name="通知类型", max_length=64)
+    status = models.CharField(
+        verbose_name="任务状态",
+        max_length=32,
+        choices=BatchTaskStatus.get_choices(),
+        default=BatchTaskStatus.RUNNING,
+    )

apiserver/paasng/paasng/platform/evaluation/notifiers.py

@@ -18,15 +18,18 @@
 import logging
 from typing import List
 
+from django.db.models import QuerySet
+
 from paasng.platform.evaluation.constants import EmailReceiverType
+from paasng.platform.evaluation.models import AppOperationReport
 
 logger = logging.getLogger(__name__)
 
 
 class AppOperationReportNotifier:
     """将蓝鲸应用的运营报告发送到指定对象"""
 
-    def send(self, receiver_type: EmailReceiverType, receivers: List[str]):
+    def send(self, reports: QuerySet[AppOperationReport], receiver_type: EmailReceiverType, receivers: List[str]):
         # 该版本暂时不支持发送邮件，有需要可以在 notifiers_ext 中实现同名类
         logger.warning("send operation report is unsupported")
 

apiserver/paasng/paasng/platform/evaluation/tasks.py

@@ -24,15 +24,25 @@
 from django.utils import timezone
 
 from paasng.infras.iam.helpers import fetch_role_members
+from paasng.infras.iam.permissions.resources.application import ApplicationPermission
 from paasng.misc.operations.models import Operation
 from paasng.platform.applications.constants import ApplicationRole, ApplicationType
 from paasng.platform.applications.models import Application
 from paasng.platform.engine.constants import AppEnvName
 from paasng.platform.engine.models import Deployment
 from paasng.platform.evaluation.collectors import AppDeploymentCollector, AppResQuotaCollector, AppUserVisitCollector
-from paasng.platform.evaluation.constants import CollectionTaskStatus, EmailReceiverType
+from paasng.platform.evaluation.constants import (
+    BatchTaskStatus,
+    EmailNotificationType,
+    EmailReceiverType,
+    OperationIssueType,
+)
 from paasng.platform.evaluation.evaluators import AppOperationEvaluator
-from paasng.platform.evaluation.models import AppOperationReport, AppOperationReportCollectionTask
+from paasng.platform.evaluation.models import (
+    AppOperationEmailNotificationTask,
+    AppOperationReport,
+    AppOperationReportCollectionTask,
+)
 from paasng.platform.evaluation.notifiers import AppOperationReportNotifier
 from paasng.utils.basic import get_username_by_bkpaas_user_id
 
@@ -131,11 +141,11 @@ def collect_and_update_app_operation_reports(app_codes: List[str]):
     for idx, app in enumerate(applications, start=1):
         try:
             _update_or_create_operation_report(app)
-            succeed_cnt += 1
         except Exception:
             failed_app_codes.append(app.code)
             logger.exception("failed to collect app: %s operation report", app.code)
 
+        succeed_cnt += 1
         # 完整采集完需要较长时间，因此每隔一段时间更新下进度
         if idx % 20 == 0:
             task.succeed_count = succeed_cnt
@@ -145,10 +155,69 @@ def collect_and_update_app_operation_reports(app_codes: List[str]):
     task.succeed_count = succeed_cnt
     task.failed_count = len(failed_app_codes)
     task.failed_app_codes = failed_app_codes
-    task.status = CollectionTaskStatus.FINISHED
+    task.status = BatchTaskStatus.FINISHED
     task.end_at = timezone.now()
     task.save(update_fields=["succeed_count", "failed_count", "failed_app_codes", "status", "end_at"])
 
     # 根据配置判断是否发送报告邮件给到平台管理员
     if settings.ENABLE_SEND_OPERATION_REPORT_EMAIL_TO_PLAT_MANAGE:
-        AppOperationReportNotifier().send(EmailReceiverType.PLAT_ADMIN, settings.BKPAAS_PLATFORM_MANAGERS)
+        reports = AppOperationReport.objects.exclude(issue_type=OperationIssueType.NONE)
+        AppOperationReportNotifier().send(reports, EmailReceiverType.PLAT_ADMIN, settings.BKPAAS_PLATFORM_MANAGERS)
+
+
+@shared_task
+def send_idle_email_to_app_developers(app_codes: List[str], only_specified_users: List[str]):
+    """发送应用闲置模块邮件给应用管理员/开发者"""
+    reports = AppOperationReport.objects.filter(issue_type=OperationIssueType.IDLE)
+    if app_codes:
+        reports = reports.filter(app__code__in=app_codes)
+
+    if not reports.exists():
+        logger.info("no idle app reports, skip current notification task")
+        return
+
+    waiting_notify_usernames = set()
+    for r in reports:
+        waiting_notify_usernames.update(r.administrators)
+        waiting_notify_usernames.update(r.developers)
+
+    # 如果特殊指定用户，只发送给指定的用户
+    if only_specified_users:
+        waiting_notify_usernames &= set(only_specified_users)
+
+    total_cnt, succeed_cnt = len(waiting_notify_usernames), 0
+    failed_usernames = []
+
+    task = AppOperationEmailNotificationTask.objects.create(
+        total_count=total_cnt, notification_type=EmailNotificationType.IDLE_APP_MODULE_ENVS
+    )
+    for idx, username in enumerate(waiting_notify_usernames):
+        filters = ApplicationPermission().gen_develop_app_filters(username)
+        app_codes = Application.objects.filter(filters).values_list("code", flat=True)
+        user_idle_app_reports = reports.filter(app__code__in=app_codes)
+
+        if not user_idle_app_reports.exists():
+            total_cnt -= 1
+            logger.info("no idle app reports, skip notification to %s", username)
+            continue
+
+        try:
+            AppOperationReportNotifier().send(user_idle_app_reports, EmailReceiverType.APP_ADMIN, [username])
+        except Exception:
+            failed_usernames.append(username)
+            logger.exception("failed to send idle module envs email to %s", username)
+
+        succeed_cnt += 1
+        # 通知完所有用户需要较长时间，因此每隔一段时间更新下进度
+        if idx % 20 == 0:
+            task.succeed_count = succeed_cnt
+            task.failed_count = len(failed_usernames)
+            task.save(update_fields=["succeed_count", "failed_count"])
+
+    task.total_count = total_cnt
+    task.succeed_count = succeed_cnt
+    task.failed_count = len(failed_usernames)
+    task.failed_usernames = failed_usernames
+    task.status = BatchTaskStatus.FINISHED
+    task.end_at = timezone.now()
+    task.save(update_fields=["total_count", "succeed_count", "failed_count", "failed_usernames", "status", "end_at"])