Disable auto quality updates

changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md

@@ -0,0 +1,4 @@
+### Removed
+
+- Automatic quality report updates in tasks
+  (<https://github.com/cvat-ai/cvat/pull/8790>)

cvat/apps/analytics_report/report/create.py

@@ -70,15 +70,6 @@ def _make_queue_job_id_base(self, obj) -> str:
         elif isinstance(obj, Job):
             return f"{self._QUEUE_JOB_PREFIX_JOB}{obj.id}"
 
-    @classmethod
-    def _get_last_report_time(cls, obj):
-        try:
-            report = obj.analytics_report
-            if report:
-                return report.created_date
-        except ObjectDoesNotExist:
-            return None
-
     def schedule_analytics_check_job(self, *, job=None, task=None, project=None, user_id):
         rq_id = self._make_queue_job_id_base(job or task or project)
 

cvat/apps/quality_control/apps.py

@@ -10,14 +10,6 @@ class QualityControlConfig(AppConfig):
     name = "cvat.apps.quality_control"
 
     def ready(self) -> None:
-        from django.conf import settings
-
-        from . import default_settings
-
-        for key in dir(default_settings):
-            if key.isupper() and not hasattr(settings, key):
-                setattr(settings, key, getattr(default_settings, key))
-
         from cvat.apps.iam.permissions import load_app_permissions
 
         load_app_permissions(self)

cvat/apps/quality_control/quality_reports.py

@@ -2180,38 +2180,23 @@ def generate_report(self) -> ComparisonReport:
 
 
 class QualityReportUpdateManager:
-    _QUEUE_AUTOUPDATE_JOB_PREFIX = "update-quality-metrics-"
     _QUEUE_CUSTOM_JOB_PREFIX = "quality-check-"
     _RQ_CUSTOM_QUALITY_CHECK_JOB_TYPE = "custom_quality_check"
     _JOB_RESULT_TTL = 120
 
-    @classmethod
-    def _get_quality_check_job_delay(cls) -> timedelta:
-        return timedelta(seconds=settings.QUALITY_CHECK_JOB_DELAY)
-
     def _get_scheduler(self) -> RqScheduler:
         return django_rq.get_scheduler(settings.CVAT_QUEUES.QUALITY_REPORTS.value)
 
     def _get_queue(self) -> RqQueue:
         return django_rq.get_queue(settings.CVAT_QUEUES.QUALITY_REPORTS.value)
 
-    def _make_queue_job_id_base(self, task: Task) -> str:
-        return f"{self._QUEUE_AUTOUPDATE_JOB_PREFIX}task-{task.id}"
-
     def _make_custom_quality_check_job_id(self, task_id: int, user_id: int) -> str:
         # FUTURE-TODO: it looks like job ID template should not include user_id because:
         # 1. There is no need to compute quality reports several times for different users
         # 2. Each user (not only rq job owner) that has permission to access a task should
         # be able to check the status of the computation process
         return f"{self._QUEUE_CUSTOM_JOB_PREFIX}task-{task_id}-user-{user_id}"
 
-    @classmethod
-    def _get_last_report_time(cls, task: Task) -> Optional[timezone.datetime]:
-        report = models.QualityReport.objects.filter(task=task).order_by("-created_date").first()
-        if report:
-            return report.created_date
-        return None
-
     class QualityReportsNotAvailable(Exception):
         pass
 
@@ -2229,33 +2214,6 @@ def _check_quality_reporting_available(self, task: Task):
                 f"and in the {StatusChoice.COMPLETED} state"
             )
 
-    def _should_update(self, task: Task) -> bool:
-        try:
-            self._check_quality_reporting_available(task)
-            return True
-        except self.QualityReportsNotAvailable:
-            return False
-
-    def schedule_quality_autoupdate_job(self, task: Task):
-        """
-        This function schedules a quality report autoupdate job
-        """
-
-        if not self._should_update(task):
-            return
-
-        now = timezone.now()
-        delay = self._get_quality_check_job_delay()
-        next_job_time = now.utcnow() + delay
-
-        schedule_job_with_throttling(
-            settings.CVAT_QUEUES.QUALITY_REPORTS.value,
-            self._make_queue_job_id_base(task),
-            next_job_time,
-            self._check_task_quality,
-            task_id=task.id,
-        )
-
     class JobAlreadyExists(QualityReportsNotAvailable):
         def __str__(self):
             return "Quality computation job for this task already enqueued"
@@ -2405,15 +2363,6 @@ def _compute_reports(self, task_id: int) -> int:
             except Task.DoesNotExist:
                 return
 
-            last_report_time = self._get_last_report_time(task)
-            if not self.is_custom_quality_check_job(self._get_current_job()) and (
-                last_report_time
-                and timezone.now() < last_report_time + self._get_quality_check_job_delay()
-            ):
-                # Discard this report as it has probably been computed in parallel
-                # with another one
-                return
-
             job_quality_reports = {}
             for job in jobs:
                 job_comparison_report = job_comparison_reports[job.id]

cvat/apps/quality_control/signals.py

@@ -1,54 +1,14 @@
-# Copyright (C) 2023 CVAT.ai Corporation
+# Copyright (C) 2023-2024 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
-from django.db import transaction
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 
-from cvat.apps.engine.models import Annotation, Job, Project, Task
-from cvat.apps.quality_control import quality_reports as qc
+from cvat.apps.engine.models import Job, Task
 from cvat.apps.quality_control.models import QualitySettings
 
 
-@receiver(post_save, sender=Job, dispatch_uid=__name__ + ".save_job-update_quality_metrics")
-@receiver(post_save, sender=Task, dispatch_uid=__name__ + ".save_task-update_quality_metrics")
-@receiver(post_save, sender=Project, dispatch_uid=__name__ + ".save_project-update_quality_metrics")
-@receiver(
-    post_save, sender=Annotation, dispatch_uid=__name__ + ".save_annotation-update_quality_metrics"
-)
-@receiver(
-    post_save,
-    sender=QualitySettings,
-    dispatch_uid=__name__ + ".save_settings-update_quality_metrics",
-)
-def __save_job__update_quality_metrics(instance, created, **kwargs):
-    tasks = []
-
-    if isinstance(instance, Project):
-        tasks += list(instance.tasks.all())
-    elif isinstance(instance, Task):
-        tasks.append(instance)
-    elif isinstance(instance, Job):
-        tasks.append(instance.segment.task)
-    elif isinstance(instance, Annotation):
-        tasks.append(instance.job.segment.task)
-    elif isinstance(instance, QualitySettings):
-        tasks.append(instance.task)
-    else:
-        assert False
-
-    def schedule_autoupdate_jobs():
-        for task in tasks:
-            if task.id is None:
-                # The task may have been deleted after the on_commit call.
-                continue
-
-            qc.QualityReportUpdateManager().schedule_quality_autoupdate_job(task)
-
-    transaction.on_commit(schedule_autoupdate_jobs, robust=True)
-
-
 @receiver(post_save, sender=Task, dispatch_uid=__name__ + ".save_task-initialize_quality_settings")
 @receiver(post_save, sender=Job, dispatch_uid=__name__ + ".save_job-initialize_quality_settings")
 def __save_task__initialize_quality_settings(instance, created, **kwargs):

cvat/settings/development.py

@@ -64,6 +64,4 @@
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
 DATABASES['default']['HOST'] = os.getenv('CVAT_POSTGRES_HOST', 'localhost')
 
-QUALITY_CHECK_JOB_DELAY = 5
-
 SMOKESCREEN_ENABLED = False

cvat/settings/testing_rest.py

@@ -11,11 +11,6 @@
     "django.contrib.auth.hashers.MD5PasswordHasher",
 ]
 
-# Avoid quality updates during test runs.
-# Note that DB initialization triggers server signals,
-# so quality report updates are scheduled for applicable jobs.
-QUALITY_CHECK_JOB_DELAY = 10000
-
 IMPORT_CACHE_CLEAN_DELAY = timedelta(seconds=30)
 
 # The tests should not fail due to high disk utilization of CI infrastructure that we have no control over

site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md

@@ -493,9 +493,12 @@ Once the quality estimation is [enabled in a task](#configuring-quality-estimati
 and the Ground Truth job is configured, quality analytics becomes available
 for the task and its jobs.
 
-By default, CVAT computes quality metrics automatically at regular intervals.
+When you open the quality analytics page, you will see quality metrics from the last
+quality estimation. The first time the page is opened, there is no quality report to
+be displayed yet. You can find the last computation date next to the report downloading
+button.
 
-If you want to refresh quality metrics (e.g. after the settings were changed),
+If you want to request an update quality metrics in a task (e.g. after the settings were changed),
 you can do this by pressing the **Refresh** button on the
 task **Quality Management** > **Analytics** page.