chore(aap): move iast and api sec logic out of the tracer (#14292)

Following #14244, continue to
remove appsec specific code out of the tracer.
- IAST Processor is now enabled in product
- API Security Manager is now enabled in product or RC layer

APPSEC-57505

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: christophe-papazian

ddtrace/_trace/tracer.py

@@ -74,27 +74,6 @@ def _default_span_processors_factory(
     span_processors: List[SpanProcessor] = []
     span_processors += [TopLevelSpanProcessor()]
 
-    if asm_config._asm_libddwaf_available:
-        if asm_config._asm_enabled:
-            if asm_config._api_security_enabled:
-                from ddtrace.appsec._api_security.api_manager import APIManager
-
-                APIManager.enable()
-
-        else:
-            # api_security_active will keep track of the service status of APIManager
-            # we don't want to import the module if it was not started before due to
-            # one click activation of ASM via Remote Config
-            if asm_config._api_security_active:
-                from ddtrace.appsec._api_security.api_manager import APIManager
-
-                APIManager.disable()
-
-    if asm_config._iast_enabled:
-        from ddtrace.appsec._iast.processor import AppSecIastSpanProcessor
-
-        span_processors.append(AppSecIastSpanProcessor())
-
     # When using the NativeWriter stats are computed by the native code.
     if config._trace_compute_stats and not config._trace_writer_native:
         # Inline the import to avoid pulling in ddsketch or protobuf

ddtrace/appsec/_iast/processor.py

@@ -1,4 +1,6 @@
 from dataclasses import dataclass
+from typing import ClassVar
+from typing import Optional
 
 from ddtrace._trace.processor import SpanProcessor
 from ddtrace._trace.span import Span
@@ -14,6 +16,22 @@
 
 @dataclass(eq=False)
 class AppSecIastSpanProcessor(SpanProcessor):
+    _instance: ClassVar[Optional["AppSecIastSpanProcessor"]] = None
+
+    @classmethod
+    def enable(cls) -> None:
+        """Enable the IAST span processor."""
+        if cls._instance is None:
+            instance = cls._instance = cls()
+            instance.register()
+
+    @classmethod
+    def disable(cls) -> None:
+        """Disable the IAST span processor."""
+        if cls._instance is not None:
+            cls._instance.unregister()
+            cls._instance = None
+
     def __post_init__(self) -> None:
         from . import load_iast
 

ddtrace/appsec/_listeners.py

@@ -35,6 +35,10 @@ def load_appsec() -> None:
         from ddtrace.appsec._processor import AppSecSpanProcessor
 
         AppSecSpanProcessor.enable()
+        if asm_config._api_security_enabled and not asm_config._api_security_active:
+            from ddtrace.appsec._api_security.api_manager import APIManager
+
+            APIManager.enable()
 
 
 def load_common_appsec_modules():

ddtrace/appsec/_remoteconfiguration.py

@@ -167,14 +167,23 @@ def disable_asm(local_tracer: Tracer):
         AppSecSpanProcessor.disable()
 
         asm_config._asm_enabled = False
+        if asm_config._api_security_active:
+            from ddtrace.appsec._api_security.api_manager import APIManager
+
+            APIManager.disable()
+
         local_tracer.configure(appsec_enabled=False)
 
 
 def enable_asm(local_tracer: Tracer):
-    if not asm_config._asm_enabled:
+    if asm_config._asm_can_be_enabled and not asm_config._asm_enabled:
         from ddtrace.appsec._listeners import load_appsec
 
         asm_config._asm_enabled = True
+        if asm_config._api_security_enabled:
+            from ddtrace.appsec._api_security.api_manager import APIManager
+
+            APIManager.enable()
         load_appsec()
         local_tracer.configure(appsec_enabled=True, appsec_enabled_origin=APPSEC.ENABLED_ORIGIN_RC)
 

ddtrace/appsec/_utils.py

@@ -52,6 +52,9 @@ def set_container_depth(self, depth: int) -> None:
         else:
             self.container_depth = max(self.container_depth, depth)
 
+    def __repr__(self) -> str:
+        return f"_observator(length={self.string_length}, size={self.container_size}, depth={self.container_depth})"
+
 
 class DDWaf_result:
     __slots__ = [

ddtrace/internal/iast/product.py

@@ -87,10 +87,11 @@ def post_preload():
 def start():
     """
     Start the IAST product.
-
-    Currently a no-op as all initialization happens in post_preload().
     """
-    pass
+    if asm_config._iast_enabled:
+        from ddtrace.appsec._iast.processor import AppSecIastSpanProcessor
+
+        AppSecIastSpanProcessor.enable()
 
 
 def restart(join=False):

riotfile.py

@@ -3520,6 +3520,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                 "DD_TRACE_AGENT_URL": "http://testagent:9126",
                 "AGENT_VERSION": "testagent",
                 "DD_REMOTE_CONFIGURATION_ENABLED": "true",
+                "DD_API_SECURITY_SAMPLE_DELAY": "0",
             },
             venvs=[
                 Venv(
@@ -3572,6 +3573,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                 "DD_TRACE_AGENT_URL": "http://testagent:9126",
                 "AGENT_VERSION": "testagent",
                 "DD_REMOTE_CONFIGURATION_ENABLED": "true",
+                "DD_API_SECURITY_SAMPLE_DELAY": "0",
             },
             venvs=[
                 Venv(
@@ -3621,6 +3623,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
                 "AGENT_VERSION": "testagent",
                 "DD_REMOTE_CONFIGURATION_ENABLED": "true",
                 "DD_IAST_DEDUPLICATION_ENABLED": "false",
+                "DD_API_SECURITY_SAMPLE_DELAY": "0",
             },
             venvs=[
                 Venv(

tests/appsec/appsec/test_telemetry.py

@@ -279,21 +279,30 @@ def test_log_metric_error_ddwaf_update_deduplication_timelapse(telemetry_writer)
         ({}, True, False, 1, APPSEC.ENABLED_ORIGIN_UNKNOWN),
         ({}, True, True, 1, APPSEC.ENABLED_ORIGIN_UNKNOWN),
         ({}, False, True, 1, APPSEC.ENABLED_ORIGIN_RC),
-        ({APPSEC_ENV: "true"}, False, True, 1, APPSEC.ENABLED_ORIGIN_ENV),
+        ({APPSEC_ENV: "true"}, True, True, 1, APPSEC.ENABLED_ORIGIN_ENV),
+        (
+            {APPSEC_ENV: "true"},
+            False,
+            True,
+            0,
+            APPSEC.ENABLED_ORIGIN_ENV,
+        ),  # 0 because RC should not change the value if env var is set
     ),
 )
 def test_appsec_enabled_metric(
     environment, appsec_enabled, rc_enabled, expected_result, expected_origin, telemetry_writer, tracer
 ):
     """Test that an internal error is logged when the WAF returns an internal error."""
-    # Restore defaults.
-    tracer.configure(appsec_enabled=appsec_enabled, appsec_enabled_origin=APPSEC.ENABLED_ORIGIN_UNKNOWN)
+    # Restore defaults and enabling telemetry appsec service
+    with override_global_config({"_asm_enabled": True}):
+        tracer.configure(appsec_enabled=appsec_enabled, appsec_enabled_origin=APPSEC.ENABLED_ORIGIN_UNKNOWN)
     telemetry_writer._flush_configuration_queue()
 
     # Start the test
-    with override_env(environment), override_global_config(dict(_asm_enabled=appsec_enabled)):
+    with override_env(environment), override_global_config(
+        dict(_asm_enabled=appsec_enabled, _remote_config_enabled=rc_enabled)
+    ):
         tracer.configure(appsec_enabled=appsec_enabled)
-        AppSecSpanProcessor()
         if rc_enabled:
             enable_asm(tracer)
 

tests/appsec/contrib_appsec/db.sqlite3

@@ -322,7 +322,7 @@ def test_truncation_telemetry(self, interface: Interface, get_entry_span_metric)
                         ("rate_limited", "false"),
                     ),
                 ),
-            ]
+            ], args_list
 
     @pytest.mark.parametrize("asm_enabled", [True, False])
     @pytest.mark.parametrize(
@@ -1358,8 +1358,14 @@ def test_api_security_scanners(
     @pytest.mark.parametrize("priority", ["keep", "drop"])
     @pytest.mark.parametrize("delay", [0.0, 120.0])
     def test_api_security_sampling(self, interface: Interface, get_entry_span_tag, apisec_enabled, priority, delay):
+        from ddtrace.appsec._api_security.api_manager import APIManager
         from ddtrace.ext import http
 
+        # clear the hashtable to avoid collisions with previous tests
+        if apisec_enabled:
+            assert APIManager._instance
+            APIManager._instance._hashtable.clear()
+
         payload = {"mastercard": "5123456789123456"}
         with override_global_config(
             dict(_asm_enabled=True, _api_security_enabled=apisec_enabled, _api_security_sample_delay=delay)