From 4fa8e45c9222f05cabef543c8fd33f737826ebe3 Mon Sep 17 00:00:00 2001 From: Bowrna Date: Mon, 29 Jan 2024 00:47:32 +0530 Subject: [PATCH] prevent celery command autoscale misconfig (#36576) --- airflow/providers/celery/provider.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index fd63a5685ac4..947dfcb6b1a0 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -119,9 +119,10 @@ config: default: "16" worker_autoscale: description: | - The maximum and minimum concurrency that will be used when starting workers with the - ``airflow celery worker`` command (always keep minimum processes, but grow - to maximum if necessary). Note the value should be max_concurrency,min_concurrency + The maximum and minimum number of pool processes that will be used to dynamically resize + the pool based on load.Enable autoscaling by providing max_concurrency,min_concurrency + with the ``airflow celery worker`` command (always keep minimum processes, + but grow to maximum if necessary). Pick these numbers based on resources on worker box and the nature of the task. If autoscale option is available, worker_concurrency will be ignored. https://docs.celeryq.dev/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale