Rework chunk cache

CHANGELOG.md

@@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - TDB
 
 ### Changed
-- TDB
+- Using KeyDB with FLASH to cache data chunk instead of diskcache (<https://github.com/opencv/cvat/pull/6773>)
 
 ### Deprecated
 - TDB

cvat/requirements/base.in

@@ -40,7 +40,6 @@ tensorflow==2.11.1 # Optional requirement of Datumaro. Use tensorflow-macos==2.8
 # The package is used by pyunpack as a command line tool to support multiple
 # archives. Don't use as a python module because it has GPL license.
 patool==1.12
-diskcache==5.4.0
 boto3==1.17.61
 azure-storage-blob==12.13.0
 google-cloud-storage==1.42.0

cvat/requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:d39b91ed6d2e90285edb9a70e4f64406dd01a96b
+# SHA1:39ecb07f2a59817ea1c9492587f393bdaadb4468
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -14,14 +14,14 @@ asgiref==3.7.2
     # via django
 astunparse==1.6.3
     # via tensorflow
-async-timeout==4.0.2
+async-timeout==4.0.3
     # via redis
 attrs==21.4.0
     # via
     #   -r cvat/requirements/base.in
     #   datumaro
     #   jsonschema
-azure-core==1.28.0
+azure-core==1.29.3
     # via
     #   azure-storage-blob
     #   msrest
@@ -42,9 +42,9 @@ certifi==2023.7.22
     #   requests
 cffi==1.15.1
     # via cryptography
-charset-normalizer==2.0.12
+charset-normalizer==3.2.0
     # via requests
-click==8.1.3
+click==8.1.7
     # via rq
 clickhouse-connect==0.6.8
     # via -r cvat/requirements/base.in
@@ -70,13 +70,11 @@ defusedxml==0.7.1
     #   python3-openid
 deprecated==1.2.14
     # via limits
-diskcache==5.4.0
-    # via -r cvat/requirements/base.in
 dj-pagination==2.5.0
     # via -r cvat/requirements/base.in
 dj-rest-auth[with_social]==2.2.7
     # via -r cvat/requirements/base.in
-django==4.2.3
+django==4.2.4
     # via
     #   -r cvat/requirements/base.in
     #   dj-rest-auth
@@ -122,13 +120,13 @@ dnspython==2.2.0
     # via -r cvat/requirements/base.in
 drf-spectacular==0.26.2
     # via -r cvat/requirements/base.in
-easyprocess==0.3
+easyprocess==1.1
     # via pyunpack
 entrypoint2==1.1
     # via pyunpack
 flatbuffers==23.5.26
     # via tensorflow
-fonttools==4.41.1
+fonttools==4.42.1
     # via matplotlib
 furl==2.1.0
     # via -r cvat/requirements/base.in
@@ -161,21 +159,21 @@ google-pasta==0.2.0
     # via tensorflow
 google-resumable-media==2.5.0
     # via google-cloud-storage
-googleapis-common-protos==1.59.1
+googleapis-common-protos==1.60.0
     # via google-api-core
-grpcio==1.56.2
+grpcio==1.57.0
     # via
     #   tensorboard
     #   tensorflow
-h5py==3.6.0
+h5py==3.9.0
     # via
     #   datumaro
     #   tensorflow
 idna==3.4
     # via requests
 importlib-metadata==6.8.0
     # via clickhouse-connect
-importlib-resources==6.0.0
+importlib-resources==6.0.1
     # via limits
 inflection==0.5.1
     # via drf-spectacular
@@ -193,7 +191,7 @@ jsonschema==4.17.3
     # via drf-spectacular
 keras==2.11.0
     # via tensorflow
-kiwisolver==1.4.4
+kiwisolver==1.4.5
     # via matplotlib
 libclang==16.0.6
     # via tensorflow
@@ -203,7 +201,7 @@ lxml==4.9.3
     # via datumaro
 lz4==4.3.2
     # via clickhouse-connect
-markdown==3.2.2
+markdown==3.4.4
     # via tensorboard
 markupsafe==2.1.3
     # via
@@ -225,7 +223,7 @@ opt-einsum==3.3.0
     # via tensorflow
 orderedmultidict==1.0.1
     # via furl
-orjson==3.9.2
+orjson==3.9.5
     # via datumaro
 packaging==23.1
     # via
@@ -260,13 +258,13 @@ pyasn1-modules==0.3.0
     # via
     #   google-auth
     #   python-ldap
-pycocotools==2.0.6
+pycocotools==2.0.7
     # via datumaro
 pycparser==2.21
     # via cffi
 pyjwt[crypto]==2.8.0
     # via django-allauth
-pylogbeat==2.0.0
+pylogbeat==2.0.1
     # via python-logstash-async
 pyparsing==3.0.9
     # via matplotlib
@@ -342,7 +340,7 @@ rules==3.3
     # via -r cvat/requirements/base.in
 s3transfer==0.4.2
     # via boto3
-scipy==1.11.1
+scipy==1.11.2
     # via datumaro
 shapely==1.7.1
     # via -r cvat/requirements/base.in
@@ -373,7 +371,7 @@ tensorflow==2.11.1
     # via -r cvat/requirements/base.in
 tensorflow-estimator==2.11.0
     # via tensorflow
-tensorflow-io-gcs-filesystem==0.32.0
+tensorflow-io-gcs-filesystem==0.33.0
     # via tensorflow
 termcolor==2.3.0
     # via tensorflow
@@ -396,9 +394,9 @@ urllib3==1.26.16
     #   clickhouse-connect
     #   google-auth
     #   requests
-werkzeug==2.3.6
+werkzeug==2.3.7
     # via tensorboard
-wheel==0.41.0
+wheel==0.41.2
     # via
     #   astunparse
     #   tensorboard

cvat/requirements/development.txt

@@ -8,7 +8,7 @@
 -r base.txt
 astroid==2.11.7
     # via pylint
-autopep8==2.0.2
+autopep8==2.0.4
     # via django-silk
 black==23.3.0
     # via -r cvat/requirements/development.in
@@ -30,11 +30,11 @@ mypy-extensions==1.0.0
     # via black
 pathspec==0.11.2
     # via black
-platformdirs==3.9.1
+platformdirs==3.10.0
     # via
     #   black
     #   pylint
-pycodestyle==2.10.0
+pycodestyle==2.11.0
     # via autopep8
 pylint==2.14.5
     # via

cvat/requirements/production.txt

@@ -6,11 +6,11 @@
 #    pip-compile-multi
 #
 -r base.txt
-anyio==3.7.1
+anyio==4.0.0
     # via watchfiles
 coverage==7.2.3
     # via -r cvat/requirements/production.in
-exceptiongroup==1.1.2
+exceptiongroup==1.1.3
     # via anyio
 h11==0.14.0
     # via uvicorn
@@ -24,7 +24,7 @@ uvicorn[standard]==0.22.0
     # via -r cvat/requirements/production.in
 uvloop==0.17.0
     # via uvicorn
-watchfiles==0.19.0
+watchfiles==0.20.0
     # via uvicorn
 websockets==11.0.3
     # via uvicorn

cvat/settings/base.py

@@ -549,13 +549,9 @@ class CVAT_QUEUES(Enum):
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
     },
    'media' : {
-       'BACKEND' : 'diskcache.DjangoCache',
-       'LOCATION' : CACHE_ROOT,
-       'TIMEOUT' : None,
-       'SHARDS': 32,
-       'OPTIONS' : {
-            'size_limit' : 2 ** 40, # 1 Tb
-       }
+       'BACKEND' : 'django.core.cache.backends.redis.RedisCache',
+       "LOCATION": "redis://localhost:6379",
+       'TIMEOUT' : 3600 * 24, # 1 day
    }
 }
 

cvat/settings/production.py

@@ -15,3 +15,5 @@
 # https://github.com/moggers87/django-sendfile2
 SENDFILE_BACKEND = 'django_sendfile.backends.nginx'
 SENDFILE_URL = '/'
+
+CACHES['media']['LOCATION'] = f"redis://:{os.getenv('CVAT_REDIS_PASSWORD', '')}@{os.getenv('CVAT_REDIS_HOST', 'cvat_redis')}:6379"

cvat/settings/testing.py

@@ -62,6 +62,7 @@
 
 LOGGING["handlers"]["server_file"] = LOGGING["handlers"]["console"]
 
+CACHES["media"]["BACKEND"] = "django.core.cache.backends.filebased.FileBasedCache"
 CACHES["media"]["LOCATION"] = CACHE_ROOT
 
 PASSWORD_HASHERS = (

cvat/settings/testing_rest.py

@@ -24,3 +24,5 @@
 HEALTH_CHECK = {
     'DISK_USAGE_MAX': 100,  # percent
 }
+
+CACHES["media"]["TIMEOUT"] = 1

docker-compose.yml

@@ -18,8 +18,16 @@ services:
 
   cvat_redis:
     container_name: cvat_redis
-    image: redis:7.0-alpine
+    image: eqalpha/keydb:x86_64_v6.3.2
     restart: always
+    command: [
+      "keydb-server",
+      "/etc/keydb/keydb.conf",
+      "--storage-provider", "flash", "/data/flash",
+      "--maxmemory", "5G",
+      "--maxmemory-policy", "allkeys-lfu"]
+    volumes:
+      - cvat_cache_db:/data
     networks:
       - cvat
 
@@ -380,6 +388,7 @@ volumes:
   cvat_keys:
   cvat_logs:
   cvat_events_db:
+  cvat_cache_db:
 
 networks:
   cvat:

helm-chart/Chart.yaml

@@ -25,11 +25,6 @@ appVersion: latest
 # We dont use it, so you can override it using values.override.yaml
 
 dependencies:
-  - name: redis
-    version: "17.3.*"
-    repository: https://charts.bitnami.com/bitnami
-    condition: redis.enabled
-
   - name: postgresql
     version: "12.1.*"
     repository: https://charts.bitnami.com/bitnami
@@ -59,3 +54,8 @@ dependencies:
     version: 10.24.0
     repository: https://helm.traefik.io/traefik
     condition: traefik.enabled
+
+  - name: keydb
+    version: 0.48.0
+    repository: https://enapter.github.io/charts/
+    condition: keydb.enabled

helm-chart/templates/_helpers.tpl

@@ -62,20 +62,18 @@ Create the name of the service account to use
 {{- end }}
 
 {{- define "cvat.sharedBackendEnv" }}
-{{- if .Values.redis.enabled }}
+{{- if .Values.keydb.enabled }}
 - name: CVAT_REDIS_HOST
-  value: "{{ .Release.Name }}-redis-master"
+  value: "{{ .Release.Name }}-keydb"
 {{- else }}
 - name: CVAT_REDIS_HOST
-  value: "{{ .Values.redis.external.host }}"
+  value: "{{ .Values.keydb.external.host }}"
 {{- end }}
-
 - name: CVAT_REDIS_PASSWORD
   valueFrom:
     secretKeyRef:
-      name: "{{ tpl (.Values.redis.secret.name) . }}"
-      key: redis-password
-
+      name: "{{ tpl (.Values.keydb.secret.name) . }}"
+      key: password
 {{- if .Values.postgresql.enabled }}
 - name: CVAT_POSTGRES_HOST
   value: "{{ .Release.Name }}-postgresql"
@@ -87,7 +85,6 @@ Create the name of the service account to use
 - name: CVAT_POSTGRES_PORT
   value: "{{ .Values.postgresql.external.port }}"
 {{- end }}
-
 - name: CVAT_POSTGRES_USER
   valueFrom:
     secretKeyRef:

helm-chart/templates/cvat-redis-secret.yml → helm-chart/templates/cvat-keydb-secret.yml

@@ -1,12 +1,12 @@
-{{- if .Values.redis.secret.create }}
+{{- if .Values.keydb.secret.create }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: "{{ tpl (.Values.redis.secret.name) . }}"
+  name: "{{ tpl (.Values.keydb.secret.name) . }}"
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "cvat.labels" . | nindent 4 }}
 type: generic
 stringData:
-  redis-password: {{ .Values.redis.secret.password | toString | quote }}
+  password: {{ .Values.keydb.secret.password | toString | quote }}
 {{- end }}

helm-chart/values.yaml

@@ -226,19 +226,23 @@ postgresql:
     replication_password: cvat_postgresql_replica
 
 
-# See https://github.com/bitnami/charts/blob/master/bitnami/redis/ for more info
-redis:
+# https://artifacthub.io/packages/helm/enapter/keydb
+keydb:
   enabled: true
   external:
-      host: 127.0.0.1
-  auth:
-    existingSecret: "{{ .Release.Name }}-redis-secret"
+    host: 127.0.0.1
+  existingSecret: "cvat-keydb-secret"
   secret:
     create: true
-    name: "{{ .Release.Name }}-redis-secret"
-    password: cvat_redis
-  cluster:
-    enabled: false
+    name: cvat-keydb-secret
+    password: cvat_keydb
+  multiMaster: false
+  activeReplicas: false
+  nodes: 1
+  configExtraArgs:
+  - storage-provider: ["flash", "/data/flash"]
+  - maxmemory: "5G"
+  - maxmemory-policy: "allkeys-lfu"
 
 nuclio:
   enabled: false