Merge branch 'main' into christophe-papazian/asm_standalone_fixed

.github/workflows/run-end-to-end.yml

@@ -106,6 +106,9 @@ jobs:
     - name: Run IAST_STANDALONE scenario
       if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"IAST_STANDALONE"')
       run: ./run.sh IAST_STANDALONE
+    - name: Run SCA_STANDALONE scenario
+      if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"SCA_STANDALONE"')
+      run: ./run.sh SCA_STANDALONE
     - name: Run IAST_DEDUPLICATION scenario
       if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"IAST_DEDUPLICATION"')
       run: ./run.sh IAST_DEDUPLICATION

docs/edit/format.md

@@ -1,4 +1,4 @@
-System tests code is in python, and is linted/formated using [black](https://black.readthedocs.io/en/stable/) and [pylint](https://pylint.readthedocs.io/en/latest/).
+System tests code is in python, and is linted/formated using [mypy](https://mypy.readthedocs.io/en/stable/), [black](https://black.readthedocs.io/en/stable/) and [pylint](https://pylint.readthedocs.io/en/latest/).
 
 Ensure you meet the other pre-reqs in [README.md](../../README.md#requirements)
 Then, run the linter with:

docs/scenarios/parametric.md

@@ -12,11 +12,11 @@ from utils.parametric.spec.trace import find_span, find_trace, find_span_in_trac
 @pytest.mark.parametrize("library_env", [{"DD_ENV": "prod"}])
 def test_datadog_spans(library_env, test_library, test_agent):
     with test_library:
-        with test_library.start_span("operation") as s1:
-            with test_library.start_span("operation1", service="hello", parent_id=s1.span_id) as s2:
+        with test_library.dd_start_span("operation") as s1:
+            with test_library.dd_start_span("operation1", service="hello", parent_id=s1.span_id) as s2:
                 pass
 
-        with test_library.start_span("otel_rocks") as os1:
+        with test_library.dd_start_span("otel_rocks") as os1:
             pass
 
     # Waits for 2 traces to be captured and avoids sorting the received spans by start time

docs/weblog/README.md

@@ -512,6 +512,23 @@ By default, the generated event has the following specification:
 
 Values can be changed with the query params called `event_name`.
 
+### GET '/inferred-proxy/span-creation'
+
+This endpoint is supposed to be hit with the necessary headers that are used to create inferred proxy
+spans for routers such as AWS API Gateway. Not including the headers means a span will not be created by the tracer
+if the feature exists.
+
+The endpoint supports the following query parameters:
+ - `status_code`: str containing status code to used in API response
+
+The headers necessary to create a span with example values:
+  `x-dd-proxy-request-time-ms`: start time in milliseconds
+  `x-dd-proxy-path`: "/api/data",
+  `x-dd-proxy-httpmethod`: "GET",
+  `x-dd-proxy-domain-name`: "system-tests-api-gateway.com",
+  `x-dd-proxy-stage`: "staging",
+  `x-dd-proxy`: "aws-apigateway",
+
 ### GET /users
 
 This endpoint calls the appsec blocking SDK functions used for blocking users. If the expected parameter matches one of

format.sh

@@ -38,11 +38,22 @@ source venv/bin/activate
 
 echo "Checking Python files..."
 if [ "$COMMAND" == "fix" ]; then
-  black .
+  black --quiet .
 else
   black --check --diff .
 fi
-pylint utils  # pylint does not have a fix mode
+
+echo "Running mypy type checks..."
+if ! mypy --config pyproject.toml; then
+  echo "Mypy type checks failed. Please fix the errors above. 💥 💔 💥"
+  exit 1
+fi
+
+echo "Running pylint checks..."
+if ! pylint utils; then
+  echo "Pylint checks failed. Please fix the errors above. 💥 💔 💥"
+  exit 1
+fi
 
 echo "Checking trailing whitespaces..."
 INCLUDE_PATTERN='.*\.(md|yml|yaml|sh|cs|Dockerfile|java|sql|ts|js|php)$'

manifests/cpp.yml

@@ -137,6 +137,8 @@ tests/:
       Test_DsmSQS: missing_feature
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   parametric/:

manifests/dotnet.yml

@@ -242,6 +242,7 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: v2.55.0
       Test_IastStandalone_UpstreamPropagation: v2.55.0
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: irrelevant (was v2.53.0 but will be replaced by V2)
       Test_Login_Events_Extended: irrelevant (was v2.53.0 but will be replaced by V2)
@@ -360,6 +361,8 @@ tests/:
       Test_DsmSQS: v2.48.0
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:

manifests/golang.yml

@@ -297,6 +297,7 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: missing_feature
       Test_IastStandalone_UpstreamPropagation: missing_feature
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: missing_feature
       Test_Login_Events_Extended: missing_feature
@@ -482,6 +483,8 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         "*": irrelevant
         net-http: missing_feature (Endpoint not implemented)
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   parametric/:

manifests/java.yml

@@ -907,6 +907,7 @@ tests/:
       Test_IastStandalone_UpstreamPropagation:
         '*': v1.36.0
         spring-boot-3-native: missing_feature (GraalVM. Tracing support only)
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: irrelevant (was v1.36.0 but will be replaced by V2)
       Test_Login_Events_Extended: irrelevant (was v1.36.0 but will be replaced by V2)
@@ -1426,6 +1427,8 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         "*": irrelevant
         spring-boot: bug (AIDM-325)
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_mongo.py:
       Test_Mongo: bug (APMAPI-729)
     test_otel_drop_in.py:

manifests/nodejs.yml

@@ -39,6 +39,7 @@ refs:
   - &ref_5_24_0 '>=5.24.0 || ^4.48.0'
   - &ref_5_25_0 '>=5.25.0 || ^4.49.0'
   - &ref_5_26_0 '>=5.26.0 || ^4.50.0'
+  - &ref_5_27_0 '>=5.27.0 || ^4.51.0'
 
 tests/:
   apm_tracing_e2e/:
@@ -427,6 +428,9 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: *ref_5_18_0
       Test_IastStandalone_UpstreamPropagation: missing_feature # was supposed to be released in 5.18.0
+      Test_SCAStandalone_Telemetry:
+        '*': *ref_5_18_0
+        nextjs: missing_feature
     test_automated_login_events.py:
       Test_Login_Events:
         '*': *ref_4_4_0
@@ -628,6 +632,10 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         '*': irrelevant
         express4: *ref_5_20_0
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation:
+          '*': irrelevant
+          express4: *ref_5_26_0
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:
@@ -640,7 +648,8 @@ tests/:
       Test_Config_TraceEnabled: *ref_4_3_0
       Test_Config_TraceLogDirectory: missing_feature
       Test_Config_UnifiedServiceTagging: *ref_5_25_0
-    test_crashtracking.py: missing_feature
+    test_crashtracking.py:
+      Test_Crashtracking: *ref_5_27_0
     test_dynamic_configuration.py:
       TestDynamicConfigSamplingRules: *ref_5_16_0
       TestDynamicConfigTracingEnabled: *ref_5_4_0

manifests/php.yml

@@ -201,6 +201,7 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: missing_feature
       Test_IastStandalone_UpstreamPropagation: missing_feature
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: irrelevant (was v0.89.0 but will be replaced by V2)
       Test_Login_Events_Extended: irrelevant (was v0.89.0 but will be replaced by V2)
@@ -296,6 +297,8 @@ tests/:
       Test_DsmSQS: missing_feature
       Test_Dsm_Manual_Checkpoint_Inter_Process: missing_feature
       Test_Dsm_Manual_Checkpoint_Intra_Process: missing_feature
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   parametric/:

manifests/python.yml

@@ -424,6 +424,7 @@ tests/:
         '*': v2.12.3
         uwsgi-poc: v2.17.1
       Test_IastStandalone_UpstreamPropagation: missing_feature
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: irrelevant (was v2.10.0 but will be replaced by V2)
       Test_Login_Events_Extended: irrelevant (was v2.10.0 but will be replaced by V2)
@@ -723,6 +724,8 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         '*': irrelevant
         flask-poc: v2.8.0
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:

manifests/ruby.yml

@@ -189,6 +189,7 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: v2.4.1-dev
       Test_IastStandalone_UpstreamPropagation: missing_feature
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events:
         '*': v1.13.0
@@ -364,6 +365,8 @@ tests/:
       Test_Dsm_Manual_Checkpoint_Intra_Process:
         '*': irrelevant
         rails70: missing_feature (Endpoint not implemented)
+    test_inferred_proxy.py:
+      Test_AWS_API_Gateway_Inferred_Span_Creation: missing_feature
     test_otel_drop_in.py:
       Test_Otel_Drop_In: missing_feature
   k8s_lib_injection/:

pyproject.toml

@@ -102,6 +102,13 @@ allow_no_jira_ticket_for_bugs = [
     "tests/parametric/test_config_consistency.py::Test_Config_TraceLogDirectory",
 ]
 
+[tool.mypy]
+files = ["utils/parametric", "tests/parametric"]
+ignore_missing_imports = true
+disable_error_code = ["no-redef"]
+exclude = 'utils/parametric/_library_client\.py|^(?!utils/parametric|tests/parametric).*$'
+follow_imports = "skip"
+
 [tool.pylint]
 init-hook='import sys; sys.path.append(".")'
 max-line-length = 120

requirements.txt

@@ -11,6 +11,7 @@ pylint==3.0.4
 python-dateutil==2.8.2
 msgpack==1.0.4
 watchdog==3.0.0
+mypy==1.0.0
 
 aiohttp==3.9.0
 yarl==1.9.4

tests/appsec/test_asm_standalone.py

@@ -2,7 +2,8 @@
 
 from requests.structures import CaseInsensitiveDict
 
-from utils import weblog, interfaces, scenarios, features, rfc, bug, flaky
+from utils.telemetry_utils import TelemetryUtils
+from utils import context, weblog, interfaces, scenarios, features, rfc, bug, flaky
 
 
 class AsmStandalone_UpstreamPropagation_Base:
@@ -667,3 +668,65 @@ def test_no_appsec_upstream__no_asm_event__is_kept_with_priority_1__from_1(self)
     @bug(library="java", weblog_variant="play", reason="APPSEC-55552")
     def test_no_appsec_upstream__no_asm_event__is_kept_with_priority_1__from_2(self):
         super().test_no_appsec_upstream__no_asm_event__is_kept_with_priority_1__from_2()
+
+
+@rfc("https://docs.google.com/document/d/12NBx-nD-IoQEMiCRnJXneq4Be7cbtSc6pJLOFUWTpNE/edit")
+@features.sca_standalone
+@scenarios.sca_standalone
+class Test_SCAStandalone_Telemetry:
+    """Tracer correctly propagates SCA telemetry in distributing tracing."""
+
+    def assert_standalone_is_enabled(self, request):
+        # test standalone is enabled and dropping traces
+        for data, _trace, span in interfaces.library.get_spans(request):
+            assert span["metrics"]["_sampling_priority_v1"] <= 0
+            assert span["metrics"]["_dd.apm.enabled"] == 0
+
+    def setup_telemetry_sca_enabled_propagated(self):
+        self.r = weblog.get("/")
+
+    def test_telemetry_sca_enabled_propagated(self):
+        self.assert_standalone_is_enabled(self.r)
+
+        for data in interfaces.library.get_telemetry_data():
+            content = data["request"]["content"]
+            if content.get("request_type") != "app-started":
+                continue
+            configuration = content["payload"]["configuration"]
+
+            configuration_by_name = {item["name"]: item for item in configuration}
+
+        assert configuration_by_name
+
+        DD_APPSEC_SCA_ENABLED = TelemetryUtils.get_dd_appsec_sca_enabled_str(context.library)
+
+        cfg_appsec_enabled = configuration_by_name.get(DD_APPSEC_SCA_ENABLED)
+        assert cfg_appsec_enabled is not None, "Missing telemetry config item for '{}'".format(DD_APPSEC_SCA_ENABLED)
+
+        outcome_value = True
+        if context.library == "java":
+            outcome_value = str(outcome_value).lower()
+        assert cfg_appsec_enabled.get("value") == outcome_value
+
+    def setup_app_dependencies_loaded(self):
+        self.r = weblog.get("/load_dependency")
+
+    def test_app_dependencies_loaded(self):
+        self.assert_standalone_is_enabled(self.r)
+
+        seen_loaded_dependencies = TelemetryUtils.get_loaded_dependency(context.library.library)
+
+        for data in interfaces.library.get_telemetry_data():
+            content = data["request"]["content"]
+            if content.get("request_type") != "app-dependencies-loaded":
+                continue
+
+            for dependency in content["payload"]["dependencies"]:
+                dependency_id = dependency["name"]  # +dependency["version"]
+
+                if dependency_id in seen_loaded_dependencies:
+                    seen_loaded_dependencies[dependency_id] = True
+
+        for dependency, seen in seen_loaded_dependencies.items():
+            if not seen:
+                raise Exception(dependency + " not received in app-dependencies-loaded message")

tests/debugger/probes/pii_line.json

@@ -0,0 +1,18 @@
+[
+    {
+        "language": "",
+        "pii": "",
+        "id": "log170aa-acda-4453-9111-1478a600line",
+        "where": {
+           "typeName": null,
+           "sourceFile": "ACTUAL_SOURCE_FILE",
+            "lines": [
+                "33"
+            ]
+        },
+        "captureSnapshot": true,
+        "capture": {
+            "maxFieldCount": 200
+        }
+    }
+]