diff --git a/changelog.d/20240617_144838_mzhiltso_fix_invalid_export_cache_calls.md b/changelog.d/20240617_144838_mzhiltso_fix_invalid_export_cache_calls.md new file mode 100644 index 000000000000..c5b9707adf2b --- /dev/null +++ b/changelog.d/20240617_144838_mzhiltso_fix_invalid_export_cache_calls.md @@ -0,0 +1,4 @@ +### Fixed + +- Invalid server cache cleanup for backups and events (after #7864) + () diff --git a/cvat/apps/engine/backup.py b/cvat/apps/engine/backup.py index 1a316a14cfc5..3ad2d1ff01d5 100644 --- a/cvat/apps/engine/backup.py +++ b/cvat/apps/engine/backup.py @@ -1,8 +1,9 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022-2023 CVAT.ai Corporation +# Copyright (C) 2022-2024 CVAT.ai Corporation # # SPDX-License-Identifier: MIT +from logging import Logger import io import os from enum import Enum @@ -46,7 +47,7 @@ from cvat.apps.engine.cloud_provider import import_resource_from_cloud_storage, export_resource_to_cloud_storage from cvat.apps.engine.location import StorageType, get_location_configuration from cvat.apps.engine.view_utils import get_cloud_storage_for_import_or_export -from cvat.apps.dataset_manager.views import TASK_CACHE_TTL, PROJECT_CACHE_TTL, get_export_cache_dir, clear_export_cache, log_exception +from cvat.apps.dataset_manager.views import TASK_CACHE_TTL, PROJECT_CACHE_TTL, get_export_cache_dir, log_exception from cvat.apps.dataset_manager.bindings import CvatImportError slogger = ServerLogManager(__name__) @@ -904,7 +905,7 @@ def _create_backup(db_instance, Exporter, output_path, logger, cache_ttl): archive_ctime = os.path.getctime(output_path) scheduler = django_rq.get_scheduler(settings.CVAT_QUEUES.IMPORT_DATA.value) cleaning_job = scheduler.enqueue_in(time_delta=cache_ttl, - func=clear_export_cache, + func=_clear_export_cache, file_path=output_path, file_ctime=archive_ctime, logger=logger) @@ -1189,3 +1190,15 @@ def import_task(request, queue_name, filename=None): location_conf=location_conf, filename=filename ) + +def _clear_export_cache(file_path: str, file_ctime: float, logger: Logger) -> None: + try: + if os.path.exists(file_path) and os.path.getctime(file_path) == file_ctime: + os.remove(file_path) + + logger.info( + "Export cache file '{}' successfully removed" \ + .format(file_path)) + except Exception: + log_exception(logger) + raise diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index aa4b682283c7..d4a7d602f564 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -1,9 +1,10 @@ # Copyright (C) 2020-2022 Intel Corporation -# Copyright (C) 2023 CVAT.ai Corporation +# Copyright (C) 2023-2024 CVAT.ai Corporation # # SPDX-License-Identifier: MIT from contextlib import ExitStack +from datetime import timedelta import io from itertools import product import os @@ -24,6 +25,7 @@ import json import av +import django_rq import numpy as np from pdf2image import convert_from_bytes from pyunpack import Archive @@ -3032,6 +3034,33 @@ def test_api_v2_tasks_id_export_somebody(self): def test_api_v2_tasks_id_export_no_auth(self): self._run_api_v2_tasks_id_export_import(None) + def test_can_remove_export_cache_automatically_after_successful_export(self): + self._create_tasks() + task_id = self.tasks[0]["id"] + user = self.admin + + with mock.patch('cvat.apps.engine.backup.TASK_CACHE_TTL', new=timedelta(hours=10)): + response = self._run_api_v2_tasks_id_export(task_id, user) + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + response = self._run_api_v2_tasks_id_export(task_id, user) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + scheduler = django_rq.get_scheduler(settings.CVAT_QUEUES.IMPORT_DATA.value) + scheduled_jobs = list(scheduler.get_jobs()) + cleanup_job = next( + j for j in scheduled_jobs if j.func_name.endswith('.engine.backup._clear_export_cache') + ) + + export_path = cleanup_job.kwargs['file_path'] + self.assertTrue(os.path.isfile(export_path)) + + from cvat.apps.engine.backup import _clear_export_cache + _clear_export_cache(**cleanup_job.kwargs) + + self.assertFalse(os.path.isfile(export_path)) + + def generate_random_image_file(filename): gen = random.SystemRandom() width = gen.randint(100, 800) diff --git a/cvat/apps/events/export.py b/cvat/apps/events/export.py index 0b97775e938c..da248db4d292 100644 --- a/cvat/apps/events/export.py +++ b/cvat/apps/events/export.py @@ -1,7 +1,8 @@ -# Copyright (C) 2023 CVAT.ai Corporation +# Copyright (C) 2023-2024 CVAT.ai Corporation # # SPDX-License-Identifier: MIT +from logging import Logger import os import csv from datetime import datetime, timedelta, timezone @@ -16,7 +17,7 @@ from rest_framework import serializers, status from rest_framework.response import Response -from cvat.apps.dataset_manager.views import clear_export_cache, log_exception +from cvat.apps.dataset_manager.views import log_exception from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.utils import sendfile @@ -72,7 +73,7 @@ def _create_csv(query_params, output_filename, cache_ttl): archive_ctime = os.path.getctime(output_filename) scheduler = django_rq.get_scheduler(settings.CVAT_QUEUES.EXPORT_DATA.value) cleaning_job = scheduler.enqueue_in(time_delta=cache_ttl, - func=clear_export_cache, + func=_clear_export_cache, file_path=output_filename, file_ctime=archive_ctime, logger=slogger.glob, @@ -168,3 +169,15 @@ def export(request, filter_query, queue_name): result_ttl=ttl, failure_ttl=ttl) return Response(data=response_data, status=status.HTTP_202_ACCEPTED) + +def _clear_export_cache(file_path: str, file_ctime: float, logger: Logger) -> None: + try: + if os.path.exists(file_path) and os.path.getctime(file_path) == file_ctime: + os.remove(file_path) + + logger.info( + "Export cache file '{}' successfully removed" \ + .format(file_path)) + except Exception: + log_exception(logger) + raise