From 3dae8745d1b929ee5078022d57224266123d9eeb Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 7 Dec 2024 02:43:13 +0200 Subject: [PATCH 1/6] Update docs --- .../manual/advanced/analytics-and-monitoring/auto-qa.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md b/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md index 19166c79aea5..1e93fce0b2cc 100644 --- a/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md +++ b/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md @@ -493,9 +493,12 @@ Once the quality estimation is [enabled in a task](#configuring-quality-estimati and the Ground Truth job is configured, quality analytics becomes available for the task and its jobs. -By default, CVAT computes quality metrics automatically at regular intervals. +When you open the quality analytics page, you will see quality metrics from the last +quality estimation. The first time the page is opened, there is no quality report to +be displayed yet. You can find the last computation date next to the report downloading +button. -If you want to refresh quality metrics (e.g. after the settings were changed), +If you want to request an update quality metrics in a task (e.g. after the settings were changed), you can do this by pressing the **Refresh** button on the task **Quality Management** > **Analytics** page. From b1627f714c34d5327084a454423f28b513daf022 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 7 Dec 2024 02:48:03 +0200 Subject: [PATCH 2/6] Disable automatic quality report updates --- cvat/apps/analytics_report/report/create.py | 9 ---- cvat/apps/quality_control/apps.py | 8 --- cvat/apps/quality_control/default_settings.py | 8 --- cvat/apps/quality_control/quality_reports.py | 51 ------------------- cvat/apps/quality_control/signals.py | 44 +--------------- cvat/settings/development.py | 2 - cvat/settings/testing_rest.py | 5 -- 7 files changed, 2 insertions(+), 125 deletions(-) delete mode 100644 cvat/apps/quality_control/default_settings.py diff --git a/cvat/apps/analytics_report/report/create.py b/cvat/apps/analytics_report/report/create.py index e028bb590ed1..8606b7cdc1f3 100644 --- a/cvat/apps/analytics_report/report/create.py +++ b/cvat/apps/analytics_report/report/create.py @@ -70,15 +70,6 @@ def _make_queue_job_id_base(self, obj) -> str: elif isinstance(obj, Job): return f"{self._QUEUE_JOB_PREFIX_JOB}{obj.id}" - @classmethod - def _get_last_report_time(cls, obj): - try: - report = obj.analytics_report - if report: - return report.created_date - except ObjectDoesNotExist: - return None - def schedule_analytics_check_job(self, *, job=None, task=None, project=None, user_id): rq_id = self._make_queue_job_id_base(job or task or project) diff --git a/cvat/apps/quality_control/apps.py b/cvat/apps/quality_control/apps.py index b1fd560de705..780613bb085c 100644 --- a/cvat/apps/quality_control/apps.py +++ b/cvat/apps/quality_control/apps.py @@ -10,14 +10,6 @@ class QualityControlConfig(AppConfig): name = "cvat.apps.quality_control" def ready(self) -> None: - from django.conf import settings - - from . import default_settings - - for key in dir(default_settings): - if key.isupper() and not hasattr(settings, key): - setattr(settings, key, getattr(default_settings, key)) - from cvat.apps.iam.permissions import load_app_permissions load_app_permissions(self) diff --git a/cvat/apps/quality_control/default_settings.py b/cvat/apps/quality_control/default_settings.py deleted file mode 100644 index 06f5aadaffc3..000000000000 --- a/cvat/apps/quality_control/default_settings.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) 2023 Intel Corporation -# -# SPDX-License-Identifier: MIT - -import os - -QUALITY_CHECK_JOB_DELAY = int(os.getenv("CVAT_QUALITY_CHECK_JOB_DELAY", 15 * 60)) -"The delay before the next quality check job is queued, in seconds" diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 627c4dc7b978..1677c0fe6b3f 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -2180,24 +2180,16 @@ def generate_report(self) -> ComparisonReport: class QualityReportUpdateManager: - _QUEUE_AUTOUPDATE_JOB_PREFIX = "update-quality-metrics-" _QUEUE_CUSTOM_JOB_PREFIX = "quality-check-" _RQ_CUSTOM_QUALITY_CHECK_JOB_TYPE = "custom_quality_check" _JOB_RESULT_TTL = 120 - @classmethod - def _get_quality_check_job_delay(cls) -> timedelta: - return timedelta(seconds=settings.QUALITY_CHECK_JOB_DELAY) - def _get_scheduler(self) -> RqScheduler: return django_rq.get_scheduler(settings.CVAT_QUEUES.QUALITY_REPORTS.value) def _get_queue(self) -> RqQueue: return django_rq.get_queue(settings.CVAT_QUEUES.QUALITY_REPORTS.value) - def _make_queue_job_id_base(self, task: Task) -> str: - return f"{self._QUEUE_AUTOUPDATE_JOB_PREFIX}task-{task.id}" - def _make_custom_quality_check_job_id(self, task_id: int, user_id: int) -> str: # FUTURE-TODO: it looks like job ID template should not include user_id because: # 1. There is no need to compute quality reports several times for different users @@ -2205,13 +2197,6 @@ def _make_custom_quality_check_job_id(self, task_id: int, user_id: int) -> str: # be able to check the status of the computation process return f"{self._QUEUE_CUSTOM_JOB_PREFIX}task-{task_id}-user-{user_id}" - @classmethod - def _get_last_report_time(cls, task: Task) -> Optional[timezone.datetime]: - report = models.QualityReport.objects.filter(task=task).order_by("-created_date").first() - if report: - return report.created_date - return None - class QualityReportsNotAvailable(Exception): pass @@ -2229,33 +2214,6 @@ def _check_quality_reporting_available(self, task: Task): f"and in the {StatusChoice.COMPLETED} state" ) - def _should_update(self, task: Task) -> bool: - try: - self._check_quality_reporting_available(task) - return True - except self.QualityReportsNotAvailable: - return False - - def schedule_quality_autoupdate_job(self, task: Task): - """ - This function schedules a quality report autoupdate job - """ - - if not self._should_update(task): - return - - now = timezone.now() - delay = self._get_quality_check_job_delay() - next_job_time = now.utcnow() + delay - - schedule_job_with_throttling( - settings.CVAT_QUEUES.QUALITY_REPORTS.value, - self._make_queue_job_id_base(task), - next_job_time, - self._check_task_quality, - task_id=task.id, - ) - class JobAlreadyExists(QualityReportsNotAvailable): def __str__(self): return "Quality computation job for this task already enqueued" @@ -2405,15 +2363,6 @@ def _compute_reports(self, task_id: int) -> int: except Task.DoesNotExist: return - last_report_time = self._get_last_report_time(task) - if not self.is_custom_quality_check_job(self._get_current_job()) and ( - last_report_time - and timezone.now() < last_report_time + self._get_quality_check_job_delay() - ): - # Discard this report as it has probably been computed in parallel - # with another one - return - job_quality_reports = {} for job in jobs: job_comparison_report = job_comparison_reports[job.id] diff --git a/cvat/apps/quality_control/signals.py b/cvat/apps/quality_control/signals.py index 7371608c3ce9..b56d70e99109 100644 --- a/cvat/apps/quality_control/signals.py +++ b/cvat/apps/quality_control/signals.py @@ -1,54 +1,14 @@ -# Copyright (C) 2023 CVAT.ai Corporation +# Copyright (C) 2023-2024 CVAT.ai Corporation # # SPDX-License-Identifier: MIT -from django.db import transaction from django.db.models.signals import post_save from django.dispatch import receiver -from cvat.apps.engine.models import Annotation, Job, Project, Task -from cvat.apps.quality_control import quality_reports as qc +from cvat.apps.engine.models import Job, Task from cvat.apps.quality_control.models import QualitySettings -@receiver(post_save, sender=Job, dispatch_uid=__name__ + ".save_job-update_quality_metrics") -@receiver(post_save, sender=Task, dispatch_uid=__name__ + ".save_task-update_quality_metrics") -@receiver(post_save, sender=Project, dispatch_uid=__name__ + ".save_project-update_quality_metrics") -@receiver( - post_save, sender=Annotation, dispatch_uid=__name__ + ".save_annotation-update_quality_metrics" -) -@receiver( - post_save, - sender=QualitySettings, - dispatch_uid=__name__ + ".save_settings-update_quality_metrics", -) -def __save_job__update_quality_metrics(instance, created, **kwargs): - tasks = [] - - if isinstance(instance, Project): - tasks += list(instance.tasks.all()) - elif isinstance(instance, Task): - tasks.append(instance) - elif isinstance(instance, Job): - tasks.append(instance.segment.task) - elif isinstance(instance, Annotation): - tasks.append(instance.job.segment.task) - elif isinstance(instance, QualitySettings): - tasks.append(instance.task) - else: - assert False - - def schedule_autoupdate_jobs(): - for task in tasks: - if task.id is None: - # The task may have been deleted after the on_commit call. - continue - - qc.QualityReportUpdateManager().schedule_quality_autoupdate_job(task) - - transaction.on_commit(schedule_autoupdate_jobs, robust=True) - - @receiver(post_save, sender=Task, dispatch_uid=__name__ + ".save_task-initialize_quality_settings") @receiver(post_save, sender=Job, dispatch_uid=__name__ + ".save_job-initialize_quality_settings") def __save_task__initialize_quality_settings(instance, created, **kwargs): diff --git a/cvat/settings/development.py b/cvat/settings/development.py index 2898fc88628f..ece4e544d2e6 100644 --- a/cvat/settings/development.py +++ b/cvat/settings/development.py @@ -64,6 +64,4 @@ # https://docs.djangoproject.com/en/3.2/ref/settings/#databases DATABASES['default']['HOST'] = os.getenv('CVAT_POSTGRES_HOST', 'localhost') -QUALITY_CHECK_JOB_DELAY = 5 - SMOKESCREEN_ENABLED = False diff --git a/cvat/settings/testing_rest.py b/cvat/settings/testing_rest.py index addda985492a..6203421bb7a0 100644 --- a/cvat/settings/testing_rest.py +++ b/cvat/settings/testing_rest.py @@ -11,11 +11,6 @@ "django.contrib.auth.hashers.MD5PasswordHasher", ] -# Avoid quality updates during test runs. -# Note that DB initialization triggers server signals, -# so quality report updates are scheduled for applicable jobs. -QUALITY_CHECK_JOB_DELAY = 10000 - IMPORT_CACHE_CLEAN_DELAY = timedelta(seconds=30) # The tests should not fail due to high disk utilization of CI infrastructure that we have no control over From bca9050a7d3fad89774ba7eb63d95281eac6b910 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 7 Dec 2024 02:50:19 +0200 Subject: [PATCH 3/6] Update changelog --- .../20241207_024832_mzhiltso_disable_auto_quality_updates.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md diff --git a/changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md b/changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md new file mode 100644 index 000000000000..51bdb20ca0f0 --- /dev/null +++ b/changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md @@ -0,0 +1,4 @@ +### Removed + +- Automatic quality report updates in tasks + () From 5afbd761d04e36404e86846d51558eac481264cc Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 7 Dec 2024 02:57:03 +0200 Subject: [PATCH 4/6] Fix imports --- cvat/apps/quality_control/quality_reports.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 1677c0fe6b3f..25b5c962dc26 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -9,7 +9,6 @@ from collections import Counter from collections.abc import Hashable, Sequence from copy import deepcopy -from datetime import timedelta from functools import cached_property, partial from typing import Any, Callable, Optional, Union, cast @@ -22,7 +21,6 @@ from datumaro.util import dump_json, parse_json from django.conf import settings from django.db import transaction -from django.utils import timezone from django_rq.queues import DjangoRQ as RqQueue from rest_framework.request import Request from rq.job import Job as RqJob @@ -61,7 +59,6 @@ AnnotationConflictType, AnnotationType, ) -from cvat.utils.background_jobs import schedule_job_with_throttling class _Serializable: From d291ab7a17cc7ef632bb50263c0ee8d17c02f921 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 7 Dec 2024 02:57:14 +0200 Subject: [PATCH 5/6] Remove unused function --- cvat/utils/background_jobs.py | 45 ----------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 cvat/utils/background_jobs.py diff --git a/cvat/utils/background_jobs.py b/cvat/utils/background_jobs.py deleted file mode 100644 index 72c93eaeaf8e..000000000000 --- a/cvat/utils/background_jobs.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (C) 2024 CVAT.ai Corporation -# -# SPDX-License-Identifier: MIT - -from collections.abc import Callable -from datetime import datetime - -import django_rq - - -def schedule_job_with_throttling( - queue_name: str, job_id_base: str, scheduled_time: datetime, func: Callable, **func_kwargs -) -> None: - """ - This function schedules an RQ job to run at `scheduled_time`, - unless it had already been used to schedule a job to run at some future time - with the same values of `queue_name` and `job_id_base`, - in which case it does nothing. - - The scheduled job will have an ID beginning with `job_id_base`, - and will execute `func(**func_kwargs)`. - """ - with django_rq.get_connection(queue_name) as connection: - # The blocker key is used to implement the throttling. - # The first time this function is called for a given tuple of - # (queue_name, job_id_base), we schedule the job and create a blocker - # that expires at the same time as the job is supposed to start. - # Until the blocker expires, we don't schedule any more jobs - # with the same tuple. - blocker_key = f"cvat:utils:scheduling-blocker:{queue_name}:{job_id_base}" - if connection.exists(blocker_key): - return - - queue_job_id = f"{job_id_base}-{scheduled_time.timestamp()}" - - # TODO: reuse the Redis connection if Django-RQ allows it. - # See . - django_rq.get_scheduler(queue_name).enqueue_at( - scheduled_time, - func, - **func_kwargs, - job_id=queue_job_id, - ) - - connection.set(blocker_key, queue_job_id, exat=scheduled_time) From 6932ed5ffcc73118096d9db21cae9c33e25a6fd6 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 11 Dec 2024 13:48:36 +0300 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Boris Sekachev --- ...41207_024832_mzhiltso_disable_auto_quality_updates.md | 2 +- .../manual/advanced/analytics-and-monitoring/auto-qa.md | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md b/changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md index 51bdb20ca0f0..4ec74a986a78 100644 --- a/changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md +++ b/changelog.d/20241207_024832_mzhiltso_disable_auto_quality_updates.md @@ -1,4 +1,4 @@ ### Removed -- Automatic quality report updates in tasks +- Automatic calculation of quality reports in tasks () diff --git a/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md b/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md index 1e93fce0b2cc..21ebd2d99087 100644 --- a/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md +++ b/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md @@ -493,12 +493,11 @@ Once the quality estimation is [enabled in a task](#configuring-quality-estimati and the Ground Truth job is configured, quality analytics becomes available for the task and its jobs. -When you open the quality analytics page, you will see quality metrics from the last -quality estimation. The first time the page is opened, there is no quality report to -be displayed yet. You can find the last computation date next to the report downloading -button. +When you open the Quality Analytics page, it displays quality metrics from the most recent quality estimation. +If it's your first time accessing the page, no quality report will be available yet. +The date of the last computation is shown next to the report download button. -If you want to request an update quality metrics in a task (e.g. after the settings were changed), +If you want to request updating of quality metrics in a task (e.g. after the settings were changed), you can do this by pressing the **Refresh** button on the task **Quality Management** > **Analytics** page.