chore(telemetry): report telemetry for all config sources (#14170)

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: mabdinur

ddtrace/internal/telemetry/__init__.py

@@ -25,20 +25,6 @@
 __all__ = ["telemetry_writer"]
 
 
-def report_config_telemetry(effective_env, val, source, otel_env, config_id):
-    if effective_env == otel_env:
-        # We only report the raw value for OpenTelemetry configurations, we should make this consistent
-        raw_val = os.environ.get(effective_env, "").lower()
-        telemetry_writer.add_configuration(effective_env, raw_val, source)
-    else:
-        if otel_env is not None and otel_env in os.environ:
-            if source in ("fleet_stable_config", "env_var"):
-                _hiding_otel_config(otel_env, effective_env)
-            else:
-                _invalid_otel_config(otel_env)
-        telemetry_writer.add_configuration(effective_env, val, source, config_id)
-
-
 def get_config(
     envs: t.Union[str, t.List[str]],
     default: t.Any = None,
@@ -47,63 +33,75 @@ def get_config(
     report_telemetry=True,
 ) -> t.Any:
     """Retrieve a configuration value in order of precedence:
-    1. Fleet stable config
+    1. Fleet stable config (highest)
     2. Datadog env vars
     3. OpenTelemetry env vars
     4. Local stable config
-    5. Default value
+    5. Default value (lowest)
+
+    Reports telemetry for every detected configuration source.
     """
     if isinstance(envs, str):
         envs = [envs]
-    source = ""
-    effective_env = ""
-    val = None
-    config_id = None
-    # Get configurations from fleet stable config
+
+    effective_val = default
+    telemetry_name = envs[0]
+    if report_telemetry:
+        telemetry_writer.add_configuration(telemetry_name, default, "default")
+
+    for env in envs:
+        if env in LOCAL_CONFIG:
+            val = LOCAL_CONFIG[env]
+            if modifier:
+                val = modifier(val)
+
+            if report_telemetry:
+                telemetry_writer.add_configuration(telemetry_name, val, "local_stable_config")
+            effective_val = val
+            break
+
+    if otel_env is not None and otel_env in os.environ:
+        raw_val, parsed_val = parse_otel_env(otel_env)
+        if parsed_val is not None:
+            val = parsed_val
+            if modifier:
+                val = modifier(val)
+
+            if report_telemetry:
+                # OpenTelemetry configurations always report the raw value
+                telemetry_writer.add_configuration(telemetry_name, raw_val, "otel_env_var")
+            effective_val = val
+        else:
+            _invalid_otel_config(otel_env)
+
+    for env in envs:
+        if env in os.environ:
+            val = os.environ[env]
+            if modifier:
+                val = modifier(val)
+
+            if report_telemetry:
+                telemetry_writer.add_configuration(telemetry_name, val, "env_var")
+                if otel_env is not None and otel_env in os.environ:
+                    _hiding_otel_config(otel_env, env)
+            effective_val = val
+            break
+
     for env in envs:
         if env in FLEET_CONFIG:
-            source = "fleet_stable_config"
-            effective_env = env
             val = FLEET_CONFIG[env]
             config_id = FLEET_CONFIG_IDS.get(env)
+            if modifier:
+                val = modifier(val)
+
+            if report_telemetry:
+                telemetry_writer.add_configuration(telemetry_name, val, "fleet_stable_config", config_id)
+                if otel_env is not None and otel_env in os.environ:
+                    _hiding_otel_config(otel_env, env)
+            effective_val = val
             break
-    # Get configurations from datadog env vars
-    if val is None:
-        for env in envs:
-            if env in os.environ:
-                source = "env_var"
-                effective_env = env
-                val = os.environ[env]
-                break
-    # Get configurations from otel env vars
-    if val is None:
-        if otel_env is not None and otel_env in os.environ:
-            parsed_val = parse_otel_env(otel_env)
-            if parsed_val is not None:
-                source = "env_var"
-                effective_env = otel_env
-                val = parsed_val
-    # Get configurations from local stable config
-    if val is None:
-        for env in envs:
-            if env in LOCAL_CONFIG:
-                source = "local_stable_config"
-                effective_env = env
-                val = LOCAL_CONFIG[env]
-                break
-    # Convert the raw value to expected format, if a modifier is provided
-    if val is not None and modifier:
-        val = modifier(val)
-    # If no value is found, use the default
-    if val is None:
-        effective_env = envs[0]
-        val = default
-        source = "default"
-    # Report telemetry
-    if report_telemetry:
-        report_config_telemetry(effective_env, val, source, otel_env, config_id)
 
-    return val
+    return effective_val
 
 
 telemetry_enabled = get_config("DD_INSTRUMENTATION_TELEMETRY_ENABLED", True, asbool, report_telemetry=False)

ddtrace/settings/_core.py

@@ -20,6 +20,7 @@ class ValueSource(str, Enum):
     CODE = "code"
     DEFAULT = "default"
     UNKNOWN = "unknown"
+    OTEL_ENV_VAR = "otel_env_var"
 
 
 class DDConfig(Env):
@@ -57,6 +58,8 @@ def __init__(
 
             if env_name in self.fleet_source:
                 value_source = ValueSource.FLEET_STABLE_CONFIG
+            elif env_name in self.env_source and env_name.upper().startswith("OTEL_"):
+                value_source = ValueSource.OTEL_ENV_VAR
             elif env_name in self.env_source:
                 value_source = ValueSource.ENV_VAR
             elif env_name in self.local_source:
@@ -75,5 +78,5 @@ def __init__(
             else:
                 self.config_id = None
 
-    def value_source(self, env_name: str) -> ValueSource:
+    def value_source(self, env_name: str) -> str:
         return self._value_source.get(env_name, ValueSource.UNKNOWN)

ddtrace/settings/_otel_remapper.py

@@ -155,13 +155,13 @@ def _remap_default(otel_value: str) -> Optional[str]:
 }
 
 
-def parse_otel_env(otel_env: str) -> Optional[str]:
+def parse_otel_env(otel_env: str) -> Tuple[str, Optional[str]]:
     _, otel_config_validator = ENV_VAR_MAPPINGS[otel_env]
     raw_value = os.environ.get(otel_env, "")
     if otel_env not in ("OTEL_RESOURCE_ATTRIBUTES", "OTEL_SERVICE_NAME"):
         # Resource attributes and service name are case-insensitive
         raw_value = raw_value.lower()
     mapped_value = otel_config_validator(raw_value)
     if mapped_value is None:
-        return None
-    return mapped_value
+        return "", None
+    return raw_value, mapped_value

tests/conftest.py

@@ -638,20 +638,21 @@ def get_configurations(self, name=None, ignores=None, remove_seq_id=False, effec
         configurations = []
         events_with_configs = self.get_events("app-started") + self.get_events("app-client-configuration-change")
         for event in events_with_configs:
-            for c in event["payload"]["configuration"]:
-                config = c.copy()
-                if remove_seq_id:
-                    config.pop("seq_id")
+            for config in event["payload"]["configuration"]:
                 if config["name"] == name or (name is None and config["name"] not in ignores):
                     configurations.append(config)
 
+        configurations.sort(key=lambda x: x["seq_id"])
         if effective:
             config_map = {}
             for c in configurations:
                 config_map[c["name"]] = c
             configurations = list(config_map.values())
 
-        configurations.sort(key=lambda x: x["name"])
+        if remove_seq_id:
+            for c in configurations:
+                c.pop("seq_id")
+
         return configurations
 
 

tests/opentelemetry/test_config.py

@@ -37,7 +37,9 @@ def _global_sampling_rule():
         "OTEL_SDK_DISABLED": "True",
         "DD_TRACE_OTEL_ENABLED": "True",
     },
-    err=b"Setting OTEL_LOGS_EXPORTER to warning is not supported by ddtrace, this configuration will be ignored.\n",
+    err=b"Setting OTEL_LOGS_EXPORTER to warning is not supported by ddtrace, this configuration "
+    b"will be ignored.\nTrace sampler set from always_off to parentbased_always_off; only parent based "
+    b"sampling is supported.\nFollowing style not supported by ddtrace: jaegar.\n",
 )
 def test_dd_otel_mixed_env_configuration():
     from ddtrace import config
@@ -69,8 +71,9 @@ def test_dd_otel_mixed_env_configuration():
         "service.version=1.0,testtag1=random1,testtag2=random2,testtag3=random3,testtag4=random4",
         "OTEL_SDK_DISABLED": "False",
     },
-    err=b"Setting OTEL_LOGS_EXPORTER to warning is not supported by ddtrace, "
-    b"this configuration will be ignored.\nFollowing style not supported by ddtrace: jaegar.\n",
+    err=b"Setting OTEL_LOGS_EXPORTER to warning is not supported by ddtrace, this configuration will be ignored.\n"
+    b"Trace sampler set from always_off to parentbased_always_off; only parent based sampling is supported.\n"
+    b"Following style not supported by ddtrace: jaegar.\n",
 )
 def test_dd_otel_missing_dd_env_configuration():
     from ddtrace import config

tests/telemetry/test_telemetry.py

@@ -353,3 +353,33 @@ def test_installed_excepthook():
     assert telemetry_writer._enabled is True
     telemetry_writer.uninstall_excepthook()
     assert sys.excepthook.__name__ != "_telemetry_excepthook"
+
+
+def test_telemetry_multiple_sources(test_agent_session, run_python_code_in_subprocess):
+    """Test that a config is submitted for multiple sources with increasing seq_id"""
+
+    env = os.environ.copy()
+    env["OTEL_TRACES_EXPORTER"] = "none"
+    env["DD_TRACE_ENABLED"] = "false"
+    env["_DD_INSTRUMENTATION_TELEMETRY_TESTS_FORCE_APP_STARTED"] = "true"
+
+    _, err, status, _ = run_python_code_in_subprocess(
+        "from ddtrace import config; config._tracing_enabled = True", env=env
+    )
+    assert status == 0, err
+
+    configs = test_agent_session.get_configurations(name="DD_TRACE_ENABLED", remove_seq_id=False, effective=False)
+    assert len(configs) == 4, configs
+
+    sorted_configs = sorted(configs, key=lambda x: x["seq_id"])
+    assert sorted_configs[0]["value"] is True
+    assert sorted_configs[0]["origin"] == "default"
+
+    assert sorted_configs[1]["value"] == "none"
+    assert sorted_configs[1]["origin"] == "otel_env_var"
+
+    assert sorted_configs[2]["value"] is False
+    assert sorted_configs[2]["origin"] == "env_var"
+
+    assert sorted_configs[3]["value"] is True
+    assert sorted_configs[3]["origin"] == "code"

tests/telemetry/test_writer.py

@@ -79,7 +79,7 @@ def test_app_started_event_configuration_override_asm(
     _, stderr, status, _ = run_python_code_in_subprocess("import ddtrace.auto", env=env)
     assert status == 0, stderr
 
-    configuration = test_agent_session.get_configurations(name=env_var, remove_seq_id=True)
+    configuration = test_agent_session.get_configurations(name=env_var, remove_seq_id=True, effective=True)
     assert len(configuration) == 1, configuration
     assert configuration[0] == {"name": env_var, "origin": "env_var", "value": expected_value}
 
@@ -300,6 +300,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
         ignores=["DD_TRACE_AGENT_URL", "DD_AGENT_PORT", "DD_TRACE_AGENT_PORT"], remove_seq_id=True, effective=True
     )
     assert configurations
+    configurations.sort(key=lambda x: x["name"])
 
     expected = [
         {"name": "DD_AGENT_HOST", "origin": "default", "value": None},
@@ -994,32 +995,40 @@ def test_otel_config_telemetry(test_agent_session, run_python_code_in_subprocess
     _, stderr, status, _ = run_python_code_in_subprocess("import ddtrace", env=env)
     assert status == 0, stderr
 
-    configurations = {c["name"]: c for c in test_agent_session.get_configurations(remove_seq_id=True)}
+    configurations = {c["name"]: c for c in test_agent_session.get_configurations(remove_seq_id=True, effective=True)}
 
     assert configurations["DD_SERVICE"] == {"name": "DD_SERVICE", "origin": "env_var", "value": "dd_service"}
-    assert configurations["OTEL_LOG_LEVEL"] == {"name": "OTEL_LOG_LEVEL", "origin": "env_var", "value": "debug"}
-    assert configurations["OTEL_PROPAGATORS"] == {
-        "name": "OTEL_PROPAGATORS",
-        "origin": "env_var",
+    assert configurations["DD_TRACE_DEBUG"] == {"name": "DD_TRACE_DEBUG", "origin": "otel_env_var", "value": "debug"}
+    assert configurations["DD_TRACE_PROPAGATION_STYLE_INJECT"] == {
+        "name": "DD_TRACE_PROPAGATION_STYLE_INJECT",
+        "origin": "otel_env_var",
         "value": "tracecontext",
     }
-    assert configurations["OTEL_TRACES_SAMPLER"] == {
-        "name": "OTEL_TRACES_SAMPLER",
-        "origin": "env_var",
+    assert configurations["DD_TRACE_PROPAGATION_STYLE_EXTRACT"] == {
+        "name": "DD_TRACE_PROPAGATION_STYLE_EXTRACT",
+        "origin": "otel_env_var",
+        "value": "tracecontext",
+    }
+    assert configurations["DD_TRACE_SAMPLING_RULES"] == {
+        "name": "DD_TRACE_SAMPLING_RULES",
+        "origin": "otel_env_var",
         "value": "always_on",
     }
-    assert configurations["OTEL_TRACES_EXPORTER"] == {
-        "name": "OTEL_TRACES_EXPORTER",
-        "origin": "env_var",
+    assert configurations["DD_TRACE_ENABLED"] == {
+        "name": "DD_TRACE_ENABLED",
+        "origin": "otel_env_var",
         "value": "none",
     }
-    assert configurations["OTEL_LOGS_EXPORTER"] == {"name": "OTEL_LOGS_EXPORTER", "origin": "env_var", "value": "otlp"}
-    assert configurations["OTEL_RESOURCE_ATTRIBUTES"] == {
-        "name": "OTEL_RESOURCE_ATTRIBUTES",
-        "origin": "env_var",
+    assert configurations["DD_TAGS"] == {
+        "name": "DD_TAGS",
+        "origin": "otel_env_var",
         "value": "team=apm,component=web",
     }
-    assert configurations["OTEL_SDK_DISABLED"] == {"name": "OTEL_SDK_DISABLED", "origin": "env_var", "value": "true"}
+    assert configurations["DD_TRACE_OTEL_ENABLED"] == {
+        "name": "DD_TRACE_OTEL_ENABLED",
+        "origin": "otel_env_var",
+        "value": "true",
+    }
 
     env_hiding_metrics = test_agent_session.get_metrics("otel.env.hiding")
     tags = [m["tags"] for m in env_hiding_metrics]
@@ -1093,3 +1102,44 @@ def test_redact_filename(filename, is_redacted):
     """Test file redaction logic"""
     writer = TelemetryWriter(is_periodic=False)
     assert writer._should_redact(filename) == is_redacted
+
+
+def test_telemetry_writer_multiple_sources_config(telemetry_writer, test_agent_session):
+    """Test that telemetry data is submitted for multiple sources with increasing seq_id"""
+
+    telemetry_writer.add_configuration("DD_SERVICE", "unamed_python_service", "default")
+    telemetry_writer.add_configuration("DD_SERVICE", "otel_service", "otel_env_var")
+    telemetry_writer.add_configuration("DD_SERVICE", "dd_service", "env_var")
+    telemetry_writer.add_configuration("DD_SERVICE", "monkey", "code")
+    telemetry_writer.add_configuration("DD_SERVICE", "baboon", "remote_config")
+    telemetry_writer.add_configuration("DD_SERVICE", "baboon", "fleet_stable_config")
+
+    telemetry_writer.periodic(force_flush=True)
+
+    configs = test_agent_session.get_configurations(name="DD_SERVICE", remove_seq_id=False, effective=False)
+    assert len(configs) == 6, configs
+
+    sorted_configs = sorted(configs, key=lambda x: x["seq_id"])
+    assert sorted_configs[0]["value"] == "unamed_python_service"
+    assert sorted_configs[0]["origin"] == "default"
+    assert sorted_configs[0]["seq_id"] == 1
+
+    assert sorted_configs[1]["value"] == "otel_service"
+    assert sorted_configs[1]["origin"] == "otel_env_var"
+    assert sorted_configs[1]["seq_id"] == 2
+
+    assert sorted_configs[2]["value"] == "dd_service"
+    assert sorted_configs[2]["origin"] == "env_var"
+    assert sorted_configs[2]["seq_id"] == 3
+
+    assert sorted_configs[3]["value"] == "monkey"
+    assert sorted_configs[3]["origin"] == "code"
+    assert sorted_configs[3]["seq_id"] == 4
+
+    assert sorted_configs[4]["value"] == "baboon"
+    assert sorted_configs[4]["origin"] == "remote_config"
+    assert sorted_configs[4]["seq_id"] == 5
+
+    assert sorted_configs[5]["value"] == "baboon"
+    assert sorted_configs[5]["origin"] == "fleet_stable_config"
+    assert sorted_configs[5]["seq_id"] == 6