Set autorestart=true for all workers (#6945)

Helm: set memory request for keydb SUpervisord configs: - Added `autorestart=true` option for all workers - Unified program names to use dashes as delimiter instead of mixed '_' and '-' - Minor improvements to supervisor configurations
cvat-ai · Oct 12, 2023 · 1b77005 · 1b77005
1 parent 2802442
commit 1b77005
Show file tree

Hide file tree

Showing 12 changed files with 45 additions and 21 deletions.
diff --git a/changelog.d/20231012_181301_andrey_fix_worker_restart.md b/changelog.d/20231012_181301_andrey_fix_worker_restart.md
@@ -0,0 +1,8 @@
+### Changed
+
+- Helm: set memory request for keydb
+- Supervisord:
+  - added `autorestart=true` option for all workers
+  - unified program names to use dashes as delimiter instead of mixed '_' and '-'
+  - minor improvements to supervisor configurations
+  (<https://github.com/opencv/cvat/pull/6945>)
diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.10.0
+version: 0.10.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm-chart/test.values.yaml b/helm-chart/test.values.yaml
@@ -27,9 +27,14 @@ cvat:
   frontend:
     imagePullPolicy: Never
 
+keydb:
+  resources:
+    requests:
+
 traefik:
   logs:
     general:
       level: DEBUG
     access:
       enabled: true
+
diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml
@@ -252,6 +252,9 @@ keydb:
   - storage-provider: ["flash", "/data/flash"]
   - maxmemory: "5G"
   - maxmemory-policy: "allkeys-lfu"
+  resources:
+    requests:
+      memory: "7G"
 
 nuclio:
   enabled: false

diff --git a/supervisord/server.conf b/supervisord/server.conf
@@ -28,15 +28,15 @@ autostart=true
 autorestart=true
 startretries=5
 numprocs=1
-process_name=%(program_name)s-%(process_num)s
+process_name=%(program_name)s-%(process_num)d
 
 [fcgi-program:uvicorn]
 socket=unix:///tmp/uvicorn.sock
 command=python3 -m uvicorn --fd 0 --forwarded-allow-ips='*' cvat.asgi:application
 autorestart=true
 environment=CVAT_EVENTS_LOCAL_DB_FILENAME="events_%(process_num)03d.db"
 numprocs=%(ENV_NUMPROCS)s
-process_name=%(program_name)s-%(process_num)s
+process_name=%(program_name)s-%(process_num)d
 
 [program:smokescreen]
 command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s
diff --git a/supervisord/utils.conf b/supervisord/utils.conf
@@ -18,24 +18,25 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
 childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqscheduler]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c \
     "python3 ~/rqscheduler.py --host %(ENV_CVAT_REDIS_HOST)s --password '%(ENV_CVAT_REDIS_PASSWORD)s' -i 30 --path '%(ENV_HOME)s'"
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=1
 
 [program:rqworker-notifications]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \
     exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 notifications \
         --worker-class cvat.rqworker.DefaultWorker \
     "
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=1
 
-[program:rqworker_cleaning]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
+[program:rqworker-cleaning]
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \
     exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 cleaning \
         --worker-class cvat.rqworker.DefaultWorker \
     "
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=%(ENV_NUMPROCS)s
-process_name=rqworker_cleaning_%(process_num)s
+process_name=%(program_name)s-%(process_num)d
+autorestart=true
diff --git a/supervisord/worker.analytics_reports.conf b/supervisord/worker.analytics_reports.conf
@@ -17,11 +17,12 @@ loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
 childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
-[program:rqworker_analytics_reports]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
+[program:rqworker-analytics-reports]
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \
     exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 analytics_reports \
         --worker-class cvat.rqworker.DefaultWorker \
     "
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=%(ENV_NUMPROCS)s
-process_name=%(program_name)s-%(process_num)s
+process_name=%(program_name)s-%(process_num)d
+autorestart=true
diff --git a/supervisord/worker.annotation.conf b/supervisord/worker.annotation.conf
@@ -18,9 +18,11 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
 childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-annotation]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \
     exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 annotation \
         --worker-class cvat.rqworker.DefaultWorker \
     "
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=%(ENV_NUMPROCS)s
+process_name=%(program_name)s-%(process_num)d
+autorestart=true
diff --git a/supervisord/worker.export.conf b/supervisord/worker.export.conf
@@ -18,10 +18,11 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
 childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-export]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \
     exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 export \
         --worker-class cvat.rqworker.DefaultWorker \
     "
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=%(ENV_NUMPROCS)s
-process_name=%(program_name)s-%(process_num)s
+process_name=%(program_name)s-%(process_num)d
+autorestart=true
diff --git a/supervisord/worker.import.conf b/supervisord/worker.import.conf
@@ -18,13 +18,14 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
 childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-import]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \
     exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 import \
         --worker-class cvat.rqworker.DefaultWorker \
     "
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=%(ENV_NUMPROCS)s
-process_name=%(program_name)s-%(process_num)s
+process_name=%(program_name)s-%(process_num)d
+autorestart=true
 
 
 [program:clamav-update]

diff --git a/supervisord/worker.quality_reports.conf b/supervisord/worker.quality_reports.conf
@@ -17,11 +17,12 @@ loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
 childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
-[program:rqworker_quality_reports]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic " \
+[program:rqworker-quality-reports]
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c " \
     exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 quality_reports \
         --worker-class cvat.rqworker.DefaultWorker \
     "
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=%(ENV_NUMPROCS)s
-process_name=rqworker_quality_reports_%(process_num)s
+process_name=%(program_name)s-%(process_num)d
+autorestart=true
diff --git a/supervisord/worker.webhooks.conf b/supervisord/worker.webhooks.conf
@@ -17,13 +17,14 @@ loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
 childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
-[program:rqworker_webhooks]
-command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -ic \
+[program:rqworker-webhooks]
+command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -c \
     "exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 webhooks \
         --worker-class cvat.rqworker.DefaultWorker \
     "
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler"
 numprocs=%(ENV_NUMPROCS)s
+process_name=%(program_name)s-%(process_num)d
 
 [program:smokescreen]
 command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s