Move worker_log_server_port option to the logging section (#17621)

UPDATING.md

@@ -73,6 +73,10 @@ https://developers.google.com/style/inclusive-documentation
 
 -->
 
+### `worker_log_server_port` configuration has been moved to the ``logging`` section.
+
+The `worker_log_server_port` configuration option has been moved from `[celery]` section to `[logging]` section to allow for re-use between different executors.
+
 ### `pandas` is now an optional dependency
 
 Previously `pandas` was a core requirement so when you run `pip install apache-airflow` it looked for `pandas`

airflow/config_templates/config.yml

@@ -611,6 +611,17 @@
       type: string
       example: "connexion,sqlalchemy"
       default: ""
+    - name: worker_log_server_port
+      description: |
+        When you start an airflow worker, airflow starts a tiny web server
+        subprocess to serve the workers local log files to the airflow main
+        web server, who then builds pages and sends them to users. This defines
+        the port on which the logs are served. It needs to be unused, and open
+        visible from the main web server to connect into the workers.
+      version_added: "2.3.0"
+      type: string
+      example: ~
+      default: "8793"
 - name: metrics
   description: |
     StatsD (https://github.com/etsy/statsd) integration settings.
@@ -1476,17 +1487,6 @@
       type: integer
       example: "1"
       default: ~
-    - name: worker_log_server_port
-      description: |
-        When you start an airflow worker, airflow starts a tiny web server
-        subprocess to serve the workers local log files to the airflow main
-        web server, who then builds pages and sends them to users. This defines
-        the port on which the logs are served. It needs to be unused, and open
-        visible from the main web server to connect into the workers.
-      version_added: ~
-      type: string
-      example: ~
-      default: "8793"
     - name: worker_umask
       description: |
         Umask that will be used when starting workers with the ``airflow celery worker``

airflow/config_templates/default_airflow.cfg

@@ -338,6 +338,13 @@ task_log_reader = task
 # Example: extra_loggers = connexion,sqlalchemy
 extra_loggers =
 
+# When you start an airflow worker, airflow starts a tiny web server
+# subprocess to serve the workers local log files to the airflow main
+# web server, who then builds pages and sends them to users. This defines
+# the port on which the logs are served. It needs to be unused, and open
+# visible from the main web server to connect into the workers.
+worker_log_server_port = 8793
+
 [metrics]
 
 # StatsD (https://github.com/etsy/statsd) integration settings.
@@ -739,13 +746,6 @@ worker_concurrency = 16
 # Example: worker_prefetch_multiplier = 1
 # worker_prefetch_multiplier =
 
-# When you start an airflow worker, airflow starts a tiny web server
-# subprocess to serve the workers local log files to the airflow main
-# web server, who then builds pages and sends them to users. This defines
-# the port on which the logs are served. It needs to be unused, and open
-# visible from the main web server to connect into the workers.
-worker_log_server_port = 8793
-
 # Umask that will be used when starting workers with the ``airflow celery worker``
 # in daemon mode. This control the file-creation mode mask which determines the initial
 # value of file permission bits for newly created files.

airflow/config_templates/default_test.cfg

@@ -54,6 +54,7 @@ fab_logging_level = WARN
 log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log
 log_processor_filename_template = {{{{ filename }}}}.log
 dag_processor_manager_log_location = {AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log
+worker_log_server_port = 8793
 
 [cli]
 api_client = airflow.api.client.local_client
@@ -94,7 +95,6 @@ smtp_timeout = 30
 [celery]
 celery_app_name = airflow.executors.celery_executor
 worker_concurrency = 16
-worker_log_server_port = 8793
 broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow
 result_backend = db+mysql://airflow:airflow@localhost:3306/airflow
 flower_host = 0.0.0.0

airflow/configuration.py

@@ -163,6 +163,7 @@ class AirflowConfigParser(ConfigParser):
         ('core', 'sensitive_var_conn_names'): ('admin', 'sensitive_variable_fields', '2.1.0'),
         ('core', 'default_pool_task_slot_count'): ('core', 'non_pooled_task_slot_count', '1.10.4'),
         ('core', 'max_active_tasks_per_dag'): ('core', 'dag_concurrency', '2.2.0'),
+        ('logging', 'worker_log_server_port'): ('celery', 'worker_log_server_port', '2.3.0'),
     }
 
     # A mapping of old default values that we want to change and warn the user

airflow/utils/log/file_task_handler.py

@@ -162,7 +162,7 @@ def _read(self, ti, try_number, metadata=None):
                 log += f'*** Unable to fetch logs from worker pod {ti.hostname} ***\n{str(f)}\n\n'
         else:
             url = os.path.join("http://{ti.hostname}:{worker_log_server_port}/log", log_relative_path).format(
-                ti=ti, worker_log_server_port=conf.get('celery', 'WORKER_LOG_SERVER_PORT')
+                ti=ti, worker_log_server_port=conf.get('logging', 'WORKER_LOG_SERVER_PORT')
             )
             log += f"*** Log file does not exist: {location}\n"
             log += f"*** Fetching from: {url}\n"

airflow/utils/serve_logs.py

@@ -103,7 +103,7 @@ def serve_logs():
     setproctitle("airflow serve-logs")
     wsgi_app = create_app()
 
-    worker_log_server_port = conf.getint('celery', 'WORKER_LOG_SERVER_PORT')
+    worker_log_server_port = conf.getint('logging', 'WORKER_LOG_SERVER_PORT')
     options = {
         'bind': f"0.0.0.0:{worker_log_server_port}",
         'workers': 2,

docs/apache-airflow/logging-monitoring/logging-tasks.rst

@@ -116,7 +116,7 @@ Most task handlers send logs upon completion of a task. In order to view logs in
 - If ``SchedulerExecutor`` or ``LocalExecutor`` is used, then when ``airflow scheduler`` is running.
 - If ``CeleryExecutor`` is used, then when ``airflow worker`` is running.
 
-The server is running on the port specified by ``worker_log_server_port`` option in ``celery`` section. By default, it is ``8793``.
-Communication between the webserver and the worker is signed with the key specified by ``secret_key`` option  in ``webserver`` section. You must ensure that the key matches so that communication can take place without problems.
+The server is running on the port specified by ``worker_log_server_port`` option in ``[logging]`` section. By default, it is ``8793``.
+Communication between the webserver and the worker is signed with the key specified by ``secret_key`` option  in ``[webserver]`` section. You must ensure that the key matches so that communication can take place without problems.
 
 We are using `Gunicorm <https://gunicorn.org/>`__ as a WSGI server. Its configuration options can be overridden with the ``GUNICORN_CMD_ARGS`` env variable. For details, see `Gunicorn settings <https://docs.gunicorn.org/en/latest/settings.html#settings>`__.