Merge branch 'main' into mtoff/query-string-consistency

docs/scenarios/parametric.md

@@ -122,7 +122,7 @@ Clone the repo:
 git clone git@github.com:DataDog/dd-trace-java.git
 cd dd-trace-java
 ```
-By default you will be on the `master` branch, but if you'd like to run system-tests on the changes you made to your local branch, `gitc checkout` to that branch.
+By default you will be on the `master` branch, but if you'd like to run system-tests on the changes you made to your local branch, `git checkout` to that branch before proceeding.
 
 2. Build Java Tracer artifacts
 ```

pyproject.toml

@@ -52,14 +52,8 @@ allow_no_feature_nodes = [
 
 allow_no_jira_ticket_for_bugs = [
  "tests/apm_tracing_e2e/test_otel.py::Test_Otel_Span.test_datadog_otel_span",
- "tests/appsec/iast/sink/test_insecure_cookie.py::TestInsecureCookie.test_secure",
- "tests/appsec/iast/sink/test_no_httponly_cookie.py::TestNoHttponlyCookie.test_secure",
- "tests/appsec/iast/sink/test_no_samesite_cookie.py::TestNoSamesiteCookie.test_secure",
  "tests/appsec/iast/sink/test_sql_injection.py::TestSqlInjection.test_insecure",
- "tests/appsec/iast/sink/test_ssrf.py::TestSSRF.test_insecure",
  "tests/appsec/iast/source/test_body.py::TestRequestBody.test_source_reported",
- "tests/appsec/iast/source/test_body.py::TestRequestBody.test_telemetry_metric_instrumented_source",
- "tests/appsec/iast/source/test_cookie_name.py::TestCookieName.test_telemetry_metric_instrumented_source",
  "tests/appsec/iast/source/test_parameter_name.py::TestParameterName.test_source_get_reported",
  "tests/appsec/iast/source/test_parameter_name.py::TestParameterName.test_source_post_reported",
  "tests/appsec/iast/source/test_parameter_name.py::TestParameterName.test_source_reported",
@@ -89,30 +83,21 @@ allow_no_jira_ticket_for_bugs = [
  "tests/appsec/test_shell_execution.py::Test_ShellExecution.test_truncate_1st_argument",
  "tests/appsec/test_shell_execution.py::Test_ShellExecution.test_truncate_blank_2nd_argument",
  "tests/appsec/test_traces.py::Test_AppSecEventSpanTags.test_header_collection",
- "tests/appsec/test_traces.py::Test_AppSecEventSpanTags.test_root_span_coherence",
  "tests/appsec/test_traces.py::Test_RetainTraces",
  "tests/appsec/test_user_blocking_full_denylist.py::Test_UserBlocking_FullDenylist.test_blocking_test",
  "tests/appsec/waf/test_addresses.py::Test_BodyJson",
  "tests/appsec/waf/test_addresses.py::Test_BodyUrlEncoded",
  "tests/appsec/waf/test_addresses.py::Test_BodyXml",
  "tests/appsec/waf/test_addresses.py::Test_BodyXml.test_xml_attr_value",
  "tests/appsec/waf/test_addresses.py::Test_BodyXml.test_xml_content",
- "tests/appsec/waf/test_addresses.py::Test_Cookies.test_cookies_with_special_chars2",
- "tests/appsec/waf/test_addresses.py::Test_Cookies.test_cookies_with_special_chars2_custom_rules",
  "tests/appsec/waf/test_blocking.py::Test_Blocking.test_accept_all",
  "tests/appsec/waf/test_blocking.py::Test_Blocking.test_accept_full_json",
  "tests/appsec/waf/test_blocking.py::Test_Blocking.test_accept_partial_json",
- "tests/appsec/waf/test_blocking.py::Test_Blocking.test_no_accept",
  "tests/appsec/waf/test_exclusions.py::Test_Exclusions.test_input_exclusion_negative_test",
  "tests/appsec/waf/test_exclusions.py::Test_Exclusions.test_rule_exclusion_positive_test",
  "tests/appsec/waf/test_miscs.py::Test_404",
- "tests/appsec/waf/test_rules.py::Test_DiscoveryScan.test_security_scan",
- "tests/appsec/waf/test_rules.py::Test_HttpProtocol.test_http_protocol",
- "tests/appsec/waf/test_rules.py::Test_LFI.test_lfi_in_path",
  "tests/appsec/waf/test_rules.py::Test_SQLI.test_sqli2",
  "tests/appsec/waf/test_rules.py::Test_SQLI.test_sqli3",
- "tests/appsec/waf/test_telemetry.py::Test_TelemetryMetrics.test_headers_are_correct",
- "tests/appsec/waf/test_telemetry.py::Test_TelemetryMetrics.test_metric_waf_requests",
  "tests/auto_inject/test_auto_inject_install.py::TestContainerAutoInjectInstallScript.test_install",
  "tests/auto_inject/test_auto_inject_install.py::TestInstallerAutoInjectManual.test_install_uninstall",
  "tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManual.test_install",
@@ -165,9 +150,7 @@ allow_no_jira_ticket_for_bugs = [
  "tests/parametric/test_trace_sampling.py::Test_Trace_Sampling_Tags_Feb2024_Revision.test_globs_different_casing",
  "tests/parametric/test_trace_sampling.py::Test_Trace_Sampling_Tags_Feb2024_Revision.test_metric_existence",
  "tests/parametric/test_trace_sampling.py::Test_Trace_Sampling_Tags_Feb2024_Revision.test_metric_matching",
- "tests/remote_config/test_remote_configuration.py::Test_RemoteConfigurationUpdateSequenceASMDD.test_tracer_update_sequence",
  "tests/remote_config/test_remote_configuration.py::Test_RemoteConfigurationUpdateSequenceFeatures.test_tracer_update_sequence",
- "tests/remote_config/test_remote_configuration.py::Test_RemoteConfigurationUpdateSequenceLiveDebugging.test_tracer_update_sequence",
  "tests/stats/test_miscs.py::Test_Miscs.test_request_headers",
  "tests/test_data_integrity.py::Test_TraceHeaders.test_trace_header_container_tags",
  "tests/test_data_integrity.py::Test_TraceHeaders.test_traces_header_present",

tests/appsec/iast/sink/test_insecure_cookie.py

@@ -17,7 +17,7 @@ class TestInsecureCookie(BaseSinkTest):
  data = {}
  location_map = {"nodejs": {"express4": "iast/index.js", "express4-typescript": "iast.ts"}}
 
- @bug(context.library < "java@1.18.3", reason="Incorrect handling of HttpOnly flag")
+ @bug(context.library < "java@1.18.3", reason="APMRP-360")
  def test_secure(self):
  super().test_secure()
 

tests/appsec/iast/sink/test_no_httponly_cookie.py

@@ -17,7 +17,7 @@ class TestNoHttponlyCookie(BaseSinkTest):
  data = {}
  location_map = {"nodejs": {"express4": "iast/index.js", "express4-typescript": "iast.ts"}}
 
- @bug(context.library < "java@1.18.3", reason="Incorrect handling of HttpOnly flag")
+ @bug(context.library < "java@1.18.3", reason="APMRP-360")
  def test_secure(self):
  super().test_secure()
 

tests/appsec/iast/sink/test_no_samesite_cookie.py

@@ -17,7 +17,7 @@ class TestNoSamesiteCookie(BaseSinkTest):
  data = {}
  location_map = {"nodejs": {"express4": "iast/index.js", "express4-typescript": "iast.ts"}}
 
- @bug(context.library < "java@1.18.3", reason="Incorrect handling of HttpOnly flag")
+ @bug(context.library < "java@1.18.3", reason="APMRP-360")
  def test_secure(self):
  super().test_secure()
 

tests/appsec/iast/sink/test_ssrf.py

@@ -21,7 +21,7 @@ class TestSSRF(BaseSinkTest):
  "python": {"flask-poc": "app.py", "django-poc": "app/urls.py"},
  }
 
- @bug(context.library < "java@1.14.0", reason="https://github.com/DataDog/dd-trace-java/pull/5172")
+ @bug(context.library < "java@1.14.0", reason="APMRP-360")
  def test_insecure(self):
  super().test_insecure()
 

tests/appsec/iast/source/test_body.py

@@ -25,7 +25,7 @@ def test_source_reported(self):
  context.library < "java@1.22.0" and "spring-boot" not in context.weblog_variant,
  reason="Metrics not implemented",
  )
- @bug(context.library >= "java@1.13.0" and context.library < "java@1.17.0", reason="Not reported")
+ @bug(context.library >= "java@1.13.0" and context.library < "java@1.17.0", reason="APMRP-360")
  @missing_feature(library="dotnet", reason="Not implemented yet")
  def test_telemetry_metric_instrumented_source(self):
  super().test_telemetry_metric_instrumented_source()

tests/appsec/iast/source/test_cookie_name.py

@@ -22,7 +22,7 @@ class TestCookieName(BaseSourceTest):
  context.library < "java@1.22.0" and "spring-boot" not in context.weblog_variant,
  reason="Metrics not implemented",
  )
- @bug(context.library >= "java@1.16.0" and context.library < "java@1.22.0", reason="Not working as expected")
+ @bug(context.library >= "java@1.16.0" and context.library < "java@1.22.0", reason="APMRP-360")
  @missing_feature(weblog_variant="akka-http", reason="Not working as expected")
  def test_telemetry_metric_instrumented_source(self):
  super().test_telemetry_metric_instrumented_source()

tests/appsec/test_traces.py

@@ -98,7 +98,7 @@ def test_header_collection(self):
  missing_response_headers = set(required_response_headers) - set(span.get("meta", {}).keys())
  assert not missing_response_headers, f"Missing response headers: {missing_response_headers}"
 
- @bug(context.library < "java@0.93.0")
+ @bug(context.library < "java@0.93.0", reason="APMRP-360")
  def test_root_span_coherence(self):
  """Appsec tags are not on span where type is not web, http or rpc"""
  valid_appsec_span_types = ["web", "http", "rpc"]

tests/appsec/waf/test_addresses.py

@@ -181,7 +181,7 @@ def setup_cookies_with_special_chars2(self):
 
  @irrelevant(library="golang", reason="not handled by the Go standard cookie parser")
  @irrelevant(library="dotnet", reason="Quotation marks cause kestrel to erase the whole value")
- @bug(context.library < "java@0.96.0")
+ @bug(context.library < "java@0.96.0", reason="APMRP-360")
  @irrelevant(context.appsec_rules_version >= "1.2.7", reason="cookies were disabled for the time being")
  def test_cookies_with_special_chars2(self):
  """Other cookies patterns"""
@@ -225,7 +225,7 @@ def setup_cookies_with_special_chars2_custom_rules(self):
 
  @irrelevant(library="golang", reason="Not handled by the Go standard cookie parser")
  @irrelevant(library="dotnet", reason="Quotation marks cause kestrel to erase the whole value")
- @bug(context.library < "java@0.96.0")
+ @bug(context.library < "java@0.96.0", reason="APMRP-360")
  @scenarios.appsec_custom_rules
  def test_cookies_with_special_chars2_custom_rules(self):
  """Other cookies patterns"""

tests/appsec/waf/test_blocking.py

@@ -52,10 +52,10 @@ class Test_Blocking:
  def setup_no_accept(self):
  self.r_na = weblog.get("/waf/", headers={"User-Agent": "Arachni/v1"})
 
- @bug(context.library < "java@0.115.0" and context.weblog_variant == "spring-boot-undertow", reason="npe")
- @bug(context.library < "java@0.115.0" and context.weblog_variant == "spring-boot-wildfly", reason="npe")
- @bug(context.library < "python@1.16.1", reason="Bug, minify and remove new line characters")
- @bug(context.library < "ruby@1.12.1", reason="wrong default content-type")
+ @bug(context.library < "java@0.115.0" and context.weblog_variant == "spring-boot-undertow", reason="APMRP-360")
+ @bug(context.library < "java@0.115.0" and context.weblog_variant == "spring-boot-wildfly", reason="APMRP-360")
+ @bug(context.library < "python@1.16.1", reason="APMRP-360")
+ @bug(context.library < "ruby@1.12.1", reason="APMRP-360")
  def test_no_accept(self):
  """Blocking without an accept header"""
  assert self.r_na.status_code == 403

tests/appsec/waf/test_rules.py

@@ -30,8 +30,8 @@ class Test_HttpProtocol:
  def setup_http_protocol(self):
  self.r_1 = weblog.get("/waf/", params={"key": ".cookie;domain="})
 
- @bug(context.library < "dotnet@2.1.0")
- @bug(context.library < "java@0.98.1")
+ @bug(context.library < "dotnet@2.1.0", reason="APMRP-360")
+ @bug(context.library < "java@0.98.1", reason="APMRP-360")
  def test_http_protocol(self):
  """ AppSec catches attacks by violation of HTTP protocol in encoded cookie value"""
  interfaces.library.assert_waf_attack(self.r_1, waf_rules.http_protocol_violation.crs_943_100)
@@ -74,7 +74,7 @@ def test_lfi_percent_2f(self):
  def setup_lfi_in_path(self):
  self.r_5 = weblog.get("/waf/..")
 
- @bug(context.library < "java@0.92.0")
+ @bug(context.library < "java@0.92.0", reason="APMRP-360")
  @irrelevant(library="python", weblog_variant="django-poc")
  @irrelevant(library="dotnet", reason="lfi patterns are always filtered by the host web-server")
  @irrelevant(
@@ -322,7 +322,7 @@ def setup_security_scan(self):
  self.r10 = weblog.get("/administrator/components/component.php")
  self.r11 = weblog.get("/login.pwd")
 
- @bug(context.library < "java@0.98.0" and context.weblog_variant == "spring-boot-undertow")
+ @bug(context.library < "java@0.98.0" and context.weblog_variant == "spring-boot-undertow", reason="APMRP-360")
  @bug(library="java", weblog_variant="spring-boot-openliberty", reason="APPSEC-6583")
  def test_security_scan(self):
  """AppSec WAF catches Discovery scan"""

tests/appsec/waf/test_telemetry.py

@@ -32,7 +32,7 @@ class Test_TelemetryMetrics:
 
  setup_headers_are_correct = _setup
 
- @bug(context.library < "java@1.13.0", reason="Missing two headers")
+ @bug(context.library < "java@1.13.0", reason="APMRP-360")
  def test_headers_are_correct(self):
  """Tests that all telemetry requests have correct headers."""
  for data in interfaces.library.get_telemetry_data(flatten_message_batches=False):
@@ -77,7 +77,7 @@ def test_metric_waf_init(self):
 
  setup_metric_waf_requests = _setup
 
- @bug(context.library < "java@1.13.0", reason="Missing tags")
+ @bug(context.library < "java@1.13.0", reason="APMRP-360")
  def test_metric_waf_requests(self):
  """Test waf.requests metric."""
  expected_metric_name = "waf.requests"