Rework chunk cache (#6773)

<!-- Raise an issue to propose your change
(https://github.com/opencv/cvat/issues).
It helps to avoid duplication of efforts from multiple independent
contributors.
Discuss your ideas with maintainers to be sure that changes will be
approved and merged.
Read the [Contribution
guide](https://opencv.github.io/cvat/docs/contributing/). -->

<!-- Provide a general summary of your changes in the Title above -->

### Motivation and context
<!-- Why is this change required? What problem does it solve? If it
fixes an open
issue, please link to the issue here. Describe your changes in detail,
add
screenshots. -->

### How has this been tested?
<!-- Please describe in detail how you tested your changes.
Include details of your testing environment, and the tests you ran to
see how your change affects other areas of the code, etc. -->

### Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply.
If an item isn't applicable for some reason, then ~~explicitly
strikethrough~~ the whole
line. If you don't do that, GitHub will show incorrect progress for the
pull request.
If you're unsure about any of these, don't hesitate to ask. We're here
to help! -->
- [x] I submit my changes into the `develop` branch
- [x] I have added a description of my changes into the
[CHANGELOG](https://github.com/opencv/cvat/blob/develop/CHANGELOG.md)
file
- [ ] I have updated the documentation accordingly
- [ ] I have added tests to cover my changes
- [ ] I have linked related issues (see [GitHub docs](

https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword))
- [ ] I have increased versions of npm packages if it is necessary

([cvat-canvas](https://github.com/opencv/cvat/tree/develop/cvat-canvas#versioning),

[cvat-core](https://github.com/opencv/cvat/tree/develop/cvat-core#versioning),

[cvat-data](https://github.com/opencv/cvat/tree/develop/cvat-data#versioning)
and

[cvat-ui](https://github.com/opencv/cvat/tree/develop/cvat-ui#versioning))

### License

- [x] I submit _my code changes_ under the same [MIT License](
https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the
project.
  Feel free to contact the maintainers if that's a concern.

---------

Co-authored-by: Roman Donchenko <roman@cvat.ai>

CHANGELOG.md

@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (<https://github.com/opencv/cvat/pull/6780>)
 - Repeated clicks on left sidebar icons now allow to close/reopen corresponding popovers
  (<https://github.com/opencv/cvat/pull/6817>)
+- Using KeyDB with FLASH to cache data chunk instead of diskcache (<https://github.com/opencv/cvat/pull/6773>)
 
 ### Deprecated
 - TDB

cvat/requirements/base.in

@@ -40,7 +40,6 @@ tensorflow==2.11.1 # Optional requirement of Datumaro. Use tensorflow-macos==2.8
 # The package is used by pyunpack as a command line tool to support multiple
 # archives. Don't use as a python module because it has GPL license.
 patool==1.12
-diskcache==5.4.0
 boto3==1.17.61
 azure-storage-blob==12.13.0
 google-cloud-storage==1.42.0

cvat/requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:d39b91ed6d2e90285edb9a70e4f64406dd01a96b
+# SHA1:39ecb07f2a59817ea1c9492587f393bdaadb4468
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -14,14 +14,14 @@ asgiref==3.7.2
     # via django
 astunparse==1.6.3
     # via tensorflow
-async-timeout==4.0.2
+async-timeout==4.0.3
     # via redis
 attrs==21.4.0
     # via
     #   -r cvat/requirements/base.in
     #   datumaro
     #   jsonschema
-azure-core==1.28.0
+azure-core==1.29.3
     # via
     #   azure-storage-blob
     #   msrest
@@ -42,9 +42,9 @@ certifi==2023.7.22
     #   requests
 cffi==1.15.1
     # via cryptography
-charset-normalizer==2.0.12
+charset-normalizer==3.2.0
     # via requests
-click==8.1.3
+click==8.1.7
     # via rq
 clickhouse-connect==0.6.8
     # via -r cvat/requirements/base.in
@@ -70,13 +70,11 @@ defusedxml==0.7.1
     #   python3-openid
 deprecated==1.2.14
     # via limits
-diskcache==5.4.0
-    # via -r cvat/requirements/base.in
 dj-pagination==2.5.0
     # via -r cvat/requirements/base.in
 dj-rest-auth[with_social]==2.2.7
     # via -r cvat/requirements/base.in
-django==4.2.3
+django==4.2.4
     # via
     #   -r cvat/requirements/base.in
     #   dj-rest-auth
@@ -122,13 +120,13 @@ dnspython==2.2.0
     # via -r cvat/requirements/base.in
 drf-spectacular==0.26.2
     # via -r cvat/requirements/base.in
-easyprocess==0.3
+easyprocess==1.1
     # via pyunpack
 entrypoint2==1.1
     # via pyunpack
 flatbuffers==23.5.26
     # via tensorflow
-fonttools==4.41.1
+fonttools==4.42.1
     # via matplotlib
 furl==2.1.0
     # via -r cvat/requirements/base.in
@@ -161,21 +159,21 @@ google-pasta==0.2.0
     # via tensorflow
 google-resumable-media==2.5.0
     # via google-cloud-storage
-googleapis-common-protos==1.59.1
+googleapis-common-protos==1.60.0
     # via google-api-core
-grpcio==1.56.2
+grpcio==1.57.0
     # via
     #   tensorboard
     #   tensorflow
-h5py==3.6.0
+h5py==3.9.0
     # via
     #   datumaro
     #   tensorflow
 idna==3.4
     # via requests
 importlib-metadata==6.8.0
     # via clickhouse-connect
-importlib-resources==6.0.0
+importlib-resources==6.0.1
     # via limits
 inflection==0.5.1
     # via drf-spectacular
@@ -193,7 +191,7 @@ jsonschema==4.17.3
     # via drf-spectacular
 keras==2.11.0
     # via tensorflow
-kiwisolver==1.4.4
+kiwisolver==1.4.5
     # via matplotlib
 libclang==16.0.6
     # via tensorflow
@@ -203,7 +201,7 @@ lxml==4.9.3
     # via datumaro
 lz4==4.3.2
     # via clickhouse-connect
-markdown==3.2.2
+markdown==3.4.4
     # via tensorboard
 markupsafe==2.1.3
     # via
@@ -225,7 +223,7 @@ opt-einsum==3.3.0
     # via tensorflow
 orderedmultidict==1.0.1
     # via furl
-orjson==3.9.2
+orjson==3.9.5
     # via datumaro
 packaging==23.1
     # via
@@ -260,13 +258,13 @@ pyasn1-modules==0.3.0
     # via
     #   google-auth
     #   python-ldap
-pycocotools==2.0.6
+pycocotools==2.0.7
     # via datumaro
 pycparser==2.21
     # via cffi
 pyjwt[crypto]==2.8.0
     # via django-allauth
-pylogbeat==2.0.0
+pylogbeat==2.0.1
     # via python-logstash-async
 pyparsing==3.0.9
     # via matplotlib
@@ -342,7 +340,7 @@ rules==3.3
     # via -r cvat/requirements/base.in
 s3transfer==0.4.2
     # via boto3
-scipy==1.11.1
+scipy==1.11.2
     # via datumaro
 shapely==1.7.1
     # via -r cvat/requirements/base.in
@@ -373,7 +371,7 @@ tensorflow==2.11.1
     # via -r cvat/requirements/base.in
 tensorflow-estimator==2.11.0
     # via tensorflow
-tensorflow-io-gcs-filesystem==0.32.0
+tensorflow-io-gcs-filesystem==0.33.0
     # via tensorflow
 termcolor==2.3.0
     # via tensorflow
@@ -396,9 +394,9 @@ urllib3==1.26.16
     #   clickhouse-connect
     #   google-auth
     #   requests
-werkzeug==2.3.6
+werkzeug==2.3.7
     # via tensorboard
-wheel==0.41.0
+wheel==0.41.2
     # via
     #   astunparse
     #   tensorboard

cvat/requirements/development.txt

@@ -8,7 +8,7 @@
 -r base.txt
 astroid==2.11.7
     # via pylint
-autopep8==2.0.2
+autopep8==2.0.4
     # via django-silk
 black==23.3.0
     # via -r cvat/requirements/development.in
@@ -30,11 +30,11 @@ mypy-extensions==1.0.0
     # via black
 pathspec==0.11.2
     # via black
-platformdirs==3.9.1
+platformdirs==3.10.0
     # via
     #   black
     #   pylint
-pycodestyle==2.10.0
+pycodestyle==2.11.0
     # via autopep8
 pylint==2.14.5
     # via

cvat/requirements/production.txt

@@ -6,11 +6,11 @@
 #    pip-compile-multi
 #
 -r base.txt
-anyio==3.7.1
+anyio==4.0.0
     # via watchfiles
 coverage==7.2.3
     # via -r cvat/requirements/production.in
-exceptiongroup==1.1.2
+exceptiongroup==1.1.3
     # via anyio
 h11==0.14.0
     # via uvicorn
@@ -24,7 +24,7 @@ uvicorn[standard]==0.22.0
     # via -r cvat/requirements/production.in
 uvloop==0.17.0
     # via uvicorn
-watchfiles==0.19.0
+watchfiles==0.20.0
     # via uvicorn
 websockets==11.0.3
     # via uvicorn

cvat/settings/base.py

@@ -25,6 +25,7 @@
 from datetime import timedelta
 from distutils.util import strtobool
 from enum import Enum
+import urllib
 
 from corsheaders.defaults import default_headers
 from logstash_async.constants import constants as logstash_async_constants
@@ -41,6 +42,10 @@
 ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', 'localhost,127.0.0.1').split(',')
 INTERNAL_IPS = ['127.0.0.1']
 
+redis_host = os.getenv('CVAT_REDIS_HOST', 'localhost')
+redis_port = os.getenv('CVAT_REDIS_PORT', 6379)
+redis_password = os.getenv('CVAT_REDIS_PASSWORD', '')
+
 def generate_secret_key():
     """
     Creates secret_key.py in such a way that multiple processes calling
@@ -325,52 +330,60 @@ class CVAT_QUEUES(Enum):
 
 RQ_QUEUES = {
     CVAT_QUEUES.IMPORT_DATA.value: {
-        'HOST': 'localhost',
-        'PORT': 6379,
+        'HOST': redis_host,
+        'PORT': redis_port,
         'DB': 0,
-        'DEFAULT_TIMEOUT': '4h'
+        'DEFAULT_TIMEOUT': '4h',
+        'PASSWORD': urllib.parse.quote(redis_password),
     },
     CVAT_QUEUES.EXPORT_DATA.value: {
-        'HOST': 'localhost',
-        'PORT': 6379,
+        'HOST': redis_host,
+        'PORT': redis_port,
         'DB': 0,
-        'DEFAULT_TIMEOUT': '4h'
+        'DEFAULT_TIMEOUT': '4h',
+        'PASSWORD': urllib.parse.quote(redis_password),
     },
     CVAT_QUEUES.AUTO_ANNOTATION.value: {
-        'HOST': 'localhost',
-        'PORT': 6379,
+        'HOST': redis_host,
+        'PORT': redis_port,
         'DB': 0,
-        'DEFAULT_TIMEOUT': '24h'
+        'DEFAULT_TIMEOUT': '24h',
+        'PASSWORD': urllib.parse.quote(redis_password),
     },
     CVAT_QUEUES.WEBHOOKS.value: {
-        'HOST': 'localhost',
-        'PORT': 6379,
+        'HOST': redis_host,
+        'PORT': redis_port,
         'DB': 0,
-        'DEFAULT_TIMEOUT': '1h'
+        'DEFAULT_TIMEOUT': '1h',
+        'PASSWORD': urllib.parse.quote(redis_password),
     },
     CVAT_QUEUES.NOTIFICATIONS.value: {
-        'HOST': 'localhost',
-        'PORT': 6379,
+        'HOST': redis_host,
+        'PORT': redis_port,
         'DB': 0,
-        'DEFAULT_TIMEOUT': '1h'
+        'DEFAULT_TIMEOUT': '1h',
+        'PASSWORD': urllib.parse.quote(redis_password),
     },
     CVAT_QUEUES.QUALITY_REPORTS.value: {
-        'HOST': 'localhost',
-        'PORT': 6379,
+        'HOST': redis_host,
+        'PORT': redis_port,
         'DB': 0,
         'DEFAULT_TIMEOUT': '1h',
+        'PASSWORD': urllib.parse.quote(redis_password),
     },
     CVAT_QUEUES.ANALYTICS_REPORTS.value: {
-        'HOST': 'localhost',
-        'PORT': 6379,
+        'HOST': redis_host,
+        'PORT': redis_port,
         'DB': 0,
-        'DEFAULT_TIMEOUT': '1h'
+        'DEFAULT_TIMEOUT': '1h',
+        'PASSWORD': urllib.parse.quote(redis_password),
     },
     CVAT_QUEUES.CLEANING.value: {
-        'HOST': 'localhost',
-        'PORT': 6379,
+        'HOST': redis_host,
+        'PORT': redis_port,
         'DB': 0,
-        'DEFAULT_TIMEOUT': '1h'
+        'DEFAULT_TIMEOUT': '1h',
+        'PASSWORD': urllib.parse.quote(redis_password),
     },
 }
 
@@ -568,19 +581,14 @@ class CVAT_QUEUES(Enum):
     'analytics_visibility': True,
 }
 
-# http://www.grantjenks.com/docs/diskcache/tutorial.html#djangocache
 CACHES = {
    'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
     },
    'media' : {
-       'BACKEND' : 'diskcache.DjangoCache',
-       'LOCATION' : CACHE_ROOT,
-       'TIMEOUT' : None,
-       'SHARDS': 32,
-       'OPTIONS' : {
-            'size_limit' : 2 ** 40, # 1 Tb
-       }
+       'BACKEND' : 'django.core.cache.backends.redis.RedisCache',
+       "LOCATION": f"redis://:{urllib.parse.quote(redis_password)}@{redis_host}:{redis_port}",
+       'TIMEOUT' : 3600 * 24, # 1 day
    }
 }
 

cvat/settings/production.py

@@ -7,9 +7,6 @@
 DEBUG = False
 
 NUCLIO['HOST'] = os.getenv('CVAT_NUCLIO_HOST', 'nuclio')
-for key in RQ_QUEUES:
-    RQ_QUEUES[key]['HOST'] = os.getenv('CVAT_REDIS_HOST', 'cvat_redis')
-    RQ_QUEUES[key]['PASSWORD'] = os.getenv('CVAT_REDIS_PASSWORD', '')
 
 # Django-sendfile:
 # https://github.com/moggers87/django-sendfile2

cvat/settings/testing.py

@@ -62,8 +62,6 @@
 
 LOGGING["handlers"]["server_file"] = LOGGING["handlers"]["console"]
 
-CACHES["media"]["LOCATION"] = CACHE_ROOT
-
 PASSWORD_HASHERS = (
     'django.contrib.auth.hashers.MD5PasswordHasher',
 )

docker-compose.ci.yml

@@ -23,3 +23,8 @@ services:
       GITHUB_RUN_ID:
     volumes:
       - ${HOST_COVERAGE_DATA_DIR}:${CONTAINER_COVERAGE_DATA_DIR}
+
+  cvat_redis:
+    ports:
+      - 6379:6379
+