Fix server cache cleanup for backups and events (#8040)

In #7864 the cache cleanup function was updated. The function was not
supposed to be called for anything except datasets, but it was called
for backups and events. This PR changes these clients to use their own
functions.

- Fixed `ValueError: Couldn't parse filename components in
'c71eba87-0914-4ccb-b883-a1bf1612fbf8.csv'` errors

changelog.d/20240617_144838_mzhiltso_fix_invalid_export_cache_calls.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Invalid server cache cleanup for backups and events (after #7864)
+  (<https://github.com/cvat-ai/cvat/pull/8040>)

cvat/apps/engine/backup.py

@@ -1,8 +1,9 @@
 # Copyright (C) 2021-2022 Intel Corporation
-# Copyright (C) 2022-2023 CVAT.ai Corporation
+# Copyright (C) 2022-2024 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
+from logging import Logger
 import io
 import os
 from enum import Enum
@@ -46,7 +47,7 @@
 from cvat.apps.engine.cloud_provider import import_resource_from_cloud_storage, export_resource_to_cloud_storage
 from cvat.apps.engine.location import StorageType, get_location_configuration
 from cvat.apps.engine.view_utils import get_cloud_storage_for_import_or_export
-from cvat.apps.dataset_manager.views import TASK_CACHE_TTL, PROJECT_CACHE_TTL, get_export_cache_dir, clear_export_cache, log_exception
+from cvat.apps.dataset_manager.views import TASK_CACHE_TTL, PROJECT_CACHE_TTL, get_export_cache_dir, log_exception
 from cvat.apps.dataset_manager.bindings import CvatImportError
 
 slogger = ServerLogManager(__name__)
@@ -904,7 +905,7 @@ def _create_backup(db_instance, Exporter, output_path, logger, cache_ttl):
             archive_ctime = os.path.getctime(output_path)
             scheduler = django_rq.get_scheduler(settings.CVAT_QUEUES.IMPORT_DATA.value)
             cleaning_job = scheduler.enqueue_in(time_delta=cache_ttl,
-                func=clear_export_cache,
+                func=_clear_export_cache,
                 file_path=output_path,
                 file_ctime=archive_ctime,
                 logger=logger)
@@ -1189,3 +1190,15 @@ def import_task(request, queue_name, filename=None):
         location_conf=location_conf,
         filename=filename
     )
+
+def _clear_export_cache(file_path: str, file_ctime: float, logger: Logger) -> None:
+    try:
+        if os.path.exists(file_path) and os.path.getctime(file_path) == file_ctime:
+            os.remove(file_path)
+
+            logger.info(
+                "Export cache file '{}' successfully removed" \
+                .format(file_path))
+    except Exception:
+        log_exception(logger)
+        raise

cvat/apps/engine/tests/test_rest_api.py

@@ -1,9 +1,10 @@
 # Copyright (C) 2020-2022 Intel Corporation
-# Copyright (C) 2023 CVAT.ai Corporation
+# Copyright (C) 2023-2024 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
 from contextlib import ExitStack
+from datetime import timedelta
 import io
 from itertools import product
 import os
@@ -24,6 +25,7 @@
 import json
 
 import av
+import django_rq
 import numpy as np
 from pdf2image import convert_from_bytes
 from pyunpack import Archive
@@ -3032,6 +3034,33 @@ def test_api_v2_tasks_id_export_somebody(self):
     def test_api_v2_tasks_id_export_no_auth(self):
         self._run_api_v2_tasks_id_export_import(None)
 
+    def test_can_remove_export_cache_automatically_after_successful_export(self):
+        self._create_tasks()
+        task_id = self.tasks[0]["id"]
+        user = self.admin
+
+        with mock.patch('cvat.apps.engine.backup.TASK_CACHE_TTL', new=timedelta(hours=10)):
+            response = self._run_api_v2_tasks_id_export(task_id, user)
+            self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
+
+            response = self._run_api_v2_tasks_id_export(task_id, user)
+            self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        scheduler = django_rq.get_scheduler(settings.CVAT_QUEUES.IMPORT_DATA.value)
+        scheduled_jobs = list(scheduler.get_jobs())
+        cleanup_job = next(
+            j for j in scheduled_jobs if j.func_name.endswith('.engine.backup._clear_export_cache')
+        )
+
+        export_path = cleanup_job.kwargs['file_path']
+        self.assertTrue(os.path.isfile(export_path))
+
+        from cvat.apps.engine.backup import _clear_export_cache
+        _clear_export_cache(**cleanup_job.kwargs)
+
+        self.assertFalse(os.path.isfile(export_path))
+
+
 def generate_random_image_file(filename):
     gen = random.SystemRandom()
     width = gen.randint(100, 800)

cvat/apps/events/export.py

@@ -1,7 +1,8 @@
-# Copyright (C) 2023 CVAT.ai Corporation
+# Copyright (C) 2023-2024 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 
+from logging import Logger
 import os
 import csv
 from datetime import datetime, timedelta, timezone
@@ -16,7 +17,7 @@
 from rest_framework import serializers, status
 from rest_framework.response import Response
 
-from cvat.apps.dataset_manager.views import clear_export_cache, log_exception
+from cvat.apps.dataset_manager.views import log_exception
 from cvat.apps.engine.log import ServerLogManager
 from cvat.apps.engine.utils import sendfile
 
@@ -72,7 +73,7 @@ def _create_csv(query_params, output_filename, cache_ttl):
         archive_ctime = os.path.getctime(output_filename)
         scheduler = django_rq.get_scheduler(settings.CVAT_QUEUES.EXPORT_DATA.value)
         cleaning_job = scheduler.enqueue_in(time_delta=cache_ttl,
-            func=clear_export_cache,
+            func=_clear_export_cache,
             file_path=output_filename,
             file_ctime=archive_ctime,
             logger=slogger.glob,
@@ -168,3 +169,15 @@ def export(request, filter_query, queue_name):
         result_ttl=ttl, failure_ttl=ttl)
 
     return Response(data=response_data, status=status.HTTP_202_ACCEPTED)
+
+def _clear_export_cache(file_path: str, file_ctime: float, logger: Logger) -> None:
+    try:
+        if os.path.exists(file_path) and os.path.getctime(file_path) == file_ctime:
+            os.remove(file_path)
+
+            logger.info(
+                "Export cache file '{}' successfully removed" \
+                .format(file_path))
+    except Exception:
+        log_exception(logger)
+        raise