From 0882e7f33b45000b7e42967f5e769a62f31607ea Mon Sep 17 00:00:00 2001 From: Bowrna Date: Thu, 4 Jan 2024 11:20:46 +0530 Subject: [PATCH] prevent celery command autoscale misconfig --- airflow/providers/celery/provider.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index fd63a5685ac4..947dfcb6b1a0 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -119,9 +119,10 @@ config: default: "16" worker_autoscale: description: | - The maximum and minimum concurrency that will be used when starting workers with the - ``airflow celery worker`` command (always keep minimum processes, but grow - to maximum if necessary). Note the value should be max_concurrency,min_concurrency + The maximum and minimum number of pool processes that will be used to dynamically resize + the pool based on load.Enable autoscaling by providing max_concurrency,min_concurrency + with the ``airflow celery worker`` command (always keep minimum processes, + but grow to maximum if necessary). Pick these numbers based on resources on worker box and the nature of the task. If autoscale option is available, worker_concurrency will be ignored. https://docs.celeryq.dev/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale