Use the regular OTel environment variables for configuring traces and metrics

airflow-core/docs/administration-and-deployment/logging-monitoring/metrics.rst

@@ -79,10 +79,20 @@ Add the Collector details to your configuration file e.g. ``airflow.cfg``
 
 .. note::
 
-    To support the OpenTelemetry exporter standard, the ``metrics`` configurations are transparently overridden by use of standard OpenTelemetry SDK environment variables.
+    **The following config keys have been deprecated and will be removed in the future**
 
-    - ``OTEL_EXPORTER_OTLP_ENDPOINT`` and ``OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`` supersede ``otel_host``, ``otel_port`` and ``otel_ssl_active``
-    - ``OTEL_METRIC_EXPORT_INTERVAL`` supersedes ``otel_interval_milliseconds``
+        .. code-block:: ini
+
+            [metrics]
+            otel_host = localhost
+            otel_port = 8889
+            otel_interval_milliseconds = 30000
+            otel_debugging_on = False
+            otel_service = Airflow
+            otel_ssl_active = False
+
+    The OpenTelemetry SDK should be configured using standard OpenTelemetry environment variables
+    such as ``OTEL_EXPORTER_OTLP_ENDPOINT``, ``OTEL_EXPORTER_OTLP_PROTOCOL``, etc.
 
     See the OpenTelemetry `exporter protocol specification <https://opentelemetry.io/docs/specs/otel/protocol/exporter/#configuration-options>`_  and
     `SDK environment variable documentation <https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#periodic-exporting-metricreader>`_ for more information.

airflow-core/docs/administration-and-deployment/logging-monitoring/traces.rst

@@ -45,9 +45,19 @@ Add the following lines to your configuration file e.g. ``airflow.cfg``
 
 .. note::
 
-    To support the OpenTelemetry exporter standard, the ``traces`` configurations are transparently superseded by use of standard OpenTelemetry SDK environment variables.
+    **The following config keys have been deprecated and will be removed in the future**
 
-    - ``OTEL_EXPORTER_OTLP_ENDPOINT`` and ``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`` overridden ``otel_host``, ``otel_port`` and ``otel_ssl_active``
+        .. code-block:: ini
+
+            [traces]
+            otel_host = localhost
+            otel_port = 8889
+            otel_debugging_on = False
+            otel_service = Airflow
+            otel_ssl_active = False
+
+    The OpenTelemetry SDK should be configured using standard OpenTelemetry environment variables
+    such as ``OTEL_EXPORTER_OTLP_ENDPOINT``, ``OTEL_EXPORTER_OTLP_PROTOCOL``, etc.
 
     See the OpenTelemetry `exporter protocol specification <https://opentelemetry.io/docs/specs/otel/protocol/exporter/#configuration-options>`_  and
     `SDK environment variable documentation <https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#periodic-exporting-metricreader>`_ for more information.

airflow-core/src/airflow/config_templates/config.yml

@@ -1209,13 +1209,33 @@ metrics:
         Specifies the hostname or IP address of the OpenTelemetry Collector to which Airflow sends
         metrics and traces.
       version_added: 2.6.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: string
       example: ~
       default: "localhost"
     otel_port:
       description: |
         Specifies the port of the OpenTelemetry Collector that is listening to.
       version_added: 2.6.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: integer
       example: ~
       default: "8889"
@@ -1231,20 +1251,50 @@ metrics:
         Defines the interval, in milliseconds, at which Airflow sends batches of metrics and traces
         to the configured OpenTelemetry Collector.
       version_added: 2.6.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: integer
       example: ~
       default: "60000"
     otel_debugging_on:
       description: |
         If ``True``, all metrics are also emitted to the console. Defaults to ``False``.
       version_added: 2.7.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: boolean
       example: ~
       default: "False"
     otel_service:
       description: |
         The default service name of traces.
       version_added: 2.10.3
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: string
       example: ~
       default: "Airflow"
@@ -1255,6 +1305,16 @@ metrics:
         you need to configure the SSL certificate and key within the OpenTelemetry collector's
         ``config.yml`` file.
       version_added: 2.7.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: boolean
       example: ~
       default: "False"
@@ -1274,27 +1334,67 @@ traces:
         Specifies the hostname or IP address of the OpenTelemetry Collector to which Airflow sends
         traces.
       version_added: 2.10.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: string
       example: ~
       default: "localhost"
     otel_port:
       description: |
         Specifies the port of the OpenTelemetry Collector that is listening to.
       version_added: 2.10.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: integer
       example: ~
       default: "8889"
     otel_service:
       description: |
         The default service name of traces.
       version_added: 2.10.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: string
       example: ~
       default: "Airflow"
     otel_debugging_on:
       description: |
         If True, all traces are also emitted to the console. Defaults to False.
       version_added: 2.10.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: boolean
       example: ~
       default: "False"
@@ -1305,6 +1405,16 @@ traces:
         you need to configure the SSL certificate and key within the OpenTelemetry collector's
         config.yml file.
       version_added: 2.10.0
+      version_deprecated: 3.2.0
+      deprecation_reason: |
+        According to the OpenTelemetry specification, configuration is expected to happen through
+        the standard OpenTelemetry environment variables rather than project-specific settings.
+
+        This option has been deprecated to ensure consistent behavior across different environments
+        and deployments that use OpenTelemetry.
+
+        OpenTelemetry should be configured exclusively using the standard OpenTelemetry environment variables
+        such as 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_PROTOCOL', 'OTEL_SERVICE_NAME', etc.
       type: boolean
       example: ~
       default: "False"

airflow-core/src/airflow/observability/metrics/otel_logger.py

@@ -26,17 +26,28 @@
 
 
 def get_otel_logger() -> SafeOtelLogger:
+    # The config values have been deprecated and therefore,
+    # if the user hasn't added them to the config, the default values won't be used.
+    # A fallback is needed to avoid an exception.
+    port = None
+    if conf.has_option("metrics", "otel_port"):
+        port = conf.getint("metrics", "otel_port")
+
+    conf_interval = None
+    if conf.has_option("metrics", "otel_interval_milliseconds"):
+        conf_interval = conf.getfloat("metrics", "otel_interval_milliseconds")
+
     return otel_logger.get_otel_logger(
-        host=conf.get("metrics", "otel_host"),  # ex: "breeze-otel-collector"
-        port=conf.getint("metrics", "otel_port"),  # ex: 4318
-        prefix=conf.get("metrics", "otel_prefix"),  # ex: "airflow"
-        ssl_active=conf.getboolean("metrics", "otel_ssl_active"),
+        host=conf.get("metrics", "otel_host", fallback=None),  # ex: "breeze-otel-collector"
+        port=port,  # ex: 4318
+        prefix=conf.get("metrics", "otel_prefix", fallback=None),  # ex: "airflow"
+        ssl_active=conf.getboolean("metrics", "otel_ssl_active", fallback=False),
         # PeriodicExportingMetricReader will default to an interval of 60000 millis.
-        conf_interval=conf.getfloat("metrics", "otel_interval_milliseconds", fallback=None),  # ex: 30000
-        debug=conf.getboolean("metrics", "otel_debugging_on"),
-        service_name=conf.get("metrics", "otel_service"),
+        conf_interval=conf_interval,  # ex: 30000
+        debug=conf.getboolean("metrics", "otel_debugging_on", fallback=False),
+        service_name=conf.get("metrics", "otel_service", fallback=None),
         metrics_allow_list=conf.get("metrics", "metrics_allow_list", fallback=None),
         metrics_block_list=conf.get("metrics", "metrics_block_list", fallback=None),
-        stat_name_handler=conf.getimport("metrics", "stat_name_handler"),
+        stat_name_handler=conf.getimport("metrics", "stat_name_handler", fallback=None),
         statsd_influxdb_enabled=conf.getboolean("metrics", "statsd_influxdb_enabled", fallback=False),
     )

airflow-core/src/airflow/observability/traces/otel_tracer.py

@@ -27,14 +27,21 @@
 
 
 def get_otel_tracer(cls, use_simple_processor: bool = False) -> OtelTrace:
+    # The config values have been deprecated and therefore,
+    # if the user hasn't added them to the config, the default values won't be used.
+    # A fallback is needed to avoid an exception.
+    port = None
+    if conf.has_option("traces", "otel_port"):
+        port = conf.getint("traces", "otel_port")
+
     return otel_tracer.get_otel_tracer(
         cls,
         use_simple_processor,
-        host=conf.get("traces", "otel_host"),
-        port=conf.getint("traces", "otel_port"),
-        ssl_active=conf.getboolean("traces", "otel_ssl_active"),
-        otel_service=conf.get("traces", "otel_service"),
-        debug=conf.getboolean("traces", "otel_debugging_on"),
+        host=conf.get("traces", "otel_host", fallback=None),
+        port=port,
+        ssl_active=conf.getboolean("traces", "otel_ssl_active", fallback=False),
+        otel_service=conf.get("traces", "otel_service", fallback=None),
+        debug=conf.getboolean("traces", "otel_debugging_on", fallback=False),
     )
 
 

airflow-core/tests/integration/otel/test_otel.py

@@ -695,10 +695,10 @@ def setup_class(cls):
         wait_for_otel_collector(otel_host, otel_port)
 
         os.environ["AIRFLOW__TRACES__OTEL_ON"] = "True"
-        os.environ["AIRFLOW__TRACES__OTEL_HOST"] = otel_host
-        os.environ["AIRFLOW__TRACES__OTEL_PORT"] = str(otel_port)
+        os.environ["OTEL_EXPORTER_OTLP_PROTOCOL"] = "http/protobuf"
+        os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = "http://breeze-otel-collector:4318/v1/traces"
         if cls.use_otel != "true":
-            os.environ["AIRFLOW__TRACES__OTEL_DEBUGGING_ON"] = "True"
+            os.environ["OTEL_TRACES_EXPORTER"] = "console"
 
         os.environ["AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR"] = "False"
         os.environ["AIRFLOW__SCHEDULER__PROCESSOR_POLL_INTERVAL"] = "2"
@@ -824,15 +824,14 @@ def test_export_legacy_metric_names(
     ):
         # Metrics.
         os.environ["AIRFLOW__METRICS__OTEL_ON"] = "True"
-        os.environ["AIRFLOW__METRICS__OTEL_HOST"] = "breeze-otel-collector"
-        os.environ["AIRFLOW__METRICS__OTEL_PORT"] = "4318"
-        os.environ["AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS"] = "5000"
+        os.environ["OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"] = "http://breeze-otel-collector:4318/v1/metrics"
+        os.environ["OTEL_METRIC_EXPORT_INTERVAL"] = "5000"
 
         assert isinstance(legacy_names_on_bool, bool)
         os.environ["AIRFLOW__METRICS__LEGACY_NAMES_ON"] = str(legacy_names_on_bool)
 
         if self.use_otel != "true":
-            os.environ["AIRFLOW__METRICS__OTEL_DEBUGGING_ON"] = "True"
+            os.environ["OTEL_METRICS_EXPORTER"] = "console"
 
         celery_worker_process = None
         scheduler_process = None