feat(pii): Sanitize URLs in Span description and breadcrumbs

sentry_sdk/consts.py

@@ -44,8 +44,6 @@
 DEFAULT_QUEUE_SIZE = 100
 DEFAULT_MAX_BREADCRUMBS = 100
 
-SENSITIVE_DATA_SUBSTITUTE = "[Filtered]"
-
 
 class INSTRUMENTER:
     SENTRY = "sentry"

sentry_sdk/integrations/boto3.py

@@ -7,6 +7,7 @@
 
 from sentry_sdk._functools import partial
 from sentry_sdk._types import MYPY
+from sentry_sdk.utils import parse_url
 
 if MYPY:
     from typing import Any
@@ -66,9 +67,14 @@ def _sentry_request_created(service_id, request, operation_name, **kwargs):
         op=OP.HTTP_CLIENT,
         description=description,
     )
+
+    parsed_url = parse_url(request.url, sanitize=False)
+
     span.set_tag("aws.service_id", service_id)
     span.set_tag("aws.operation_name", operation_name)
-    span.set_data("aws.request.url", request.url)
+    span.set_data("aws.request.url", parsed_url.url)
+    span.set_data("http.query", parsed_url.query)
+    span.set_data("http.fragment", parsed_url.fragment)
 
     # We do it in order for subsequent http calls/retries be
     # attached to this span.

sentry_sdk/integrations/django/__init__.py

@@ -6,7 +6,7 @@
 import weakref
 
 from sentry_sdk._types import MYPY
-from sentry_sdk.consts import OP, SENSITIVE_DATA_SUBSTITUTE
+from sentry_sdk.consts import OP
 from sentry_sdk.hub import Hub, _should_send_default_pii
 from sentry_sdk.scope import add_global_event_processor
 from sentry_sdk.serializer import add_global_repr_processor
@@ -16,6 +16,7 @@
     AnnotatedValue,
     HAS_REAL_CONTEXTVARS,
     CONTEXTVARS_ERROR_MESSAGE,
+    SENSITIVE_DATA_SUBSTITUTE,
     logger,
     capture_internal_exceptions,
     event_from_exception,

sentry_sdk/integrations/httpx.py

@@ -1,7 +1,7 @@
 from sentry_sdk import Hub
 from sentry_sdk.consts import OP
 from sentry_sdk.integrations import Integration, DidNotEnable
-from sentry_sdk.utils import logger
+from sentry_sdk.utils import logger, parse_url
 
 from sentry_sdk._types import MYPY
 
@@ -41,11 +41,17 @@ def send(self, request, **kwargs):
         if hub.get_integration(HttpxIntegration) is None:
             return real_send(self, request, **kwargs)
 
+        parsed_url = parse_url(str(request.url), sanitize=False)
+
         with hub.start_span(
-            op=OP.HTTP_CLIENT, description="%s %s" % (request.method, request.url)
+            op=OP.HTTP_CLIENT,
+            description="%s %s" % (request.method, parsed_url.url),
         ) as span:
             span.set_data("method", request.method)
-            span.set_data("url", str(request.url))
+            span.set_data("url", parsed_url.url)
+            span.set_data("http.query", parsed_url.query)
+            span.set_data("http.fragment", parsed_url.fragment)
+
             for key, value in hub.iter_trace_propagation_headers():
                 logger.debug(
                     "[Tracing] Adding `{key}` header {value} to outgoing request to {url}.".format(
@@ -58,6 +64,7 @@ def send(self, request, **kwargs):
             span.set_data("status_code", rv.status_code)
             span.set_http_status(rv.status_code)
             span.set_data("reason", rv.reason_phrase)
+
             return rv
 
     Client.send = send
@@ -73,11 +80,17 @@ async def send(self, request, **kwargs):
         if hub.get_integration(HttpxIntegration) is None:
             return await real_send(self, request, **kwargs)
 
+        parsed_url = parse_url(str(request.url), sanitize=False)
+
         with hub.start_span(
-            op=OP.HTTP_CLIENT, description="%s %s" % (request.method, request.url)
+            op=OP.HTTP_CLIENT,
+            description="%s %s" % (request.method, parsed_url.url),
         ) as span:
             span.set_data("method", request.method)
-            span.set_data("url", str(request.url))
+            span.set_data("url", parsed_url.url)
+            span.set_data("http.query", parsed_url.query)
+            span.set_data("http.fragment", parsed_url.fragment)
+
             for key, value in hub.iter_trace_propagation_headers():
                 logger.debug(
                     "[Tracing] Adding `{key}` header {value} to outgoing request to {url}.".format(
@@ -90,6 +103,7 @@ async def send(self, request, **kwargs):
             span.set_data("status_code", rv.status_code)
             span.set_http_status(rv.status_code)
             span.set_data("reason", rv.reason_phrase)
+
             return rv
 
     AsyncClient.send = send

sentry_sdk/integrations/huey.py

@@ -6,11 +6,15 @@
 from sentry_sdk._compat import reraise
 from sentry_sdk._types import MYPY
 from sentry_sdk import Hub
-from sentry_sdk.consts import OP, SENSITIVE_DATA_SUBSTITUTE
+from sentry_sdk.consts import OP
 from sentry_sdk.hub import _should_send_default_pii
 from sentry_sdk.integrations import DidNotEnable, Integration
 from sentry_sdk.tracing import Transaction, TRANSACTION_SOURCE_TASK
-from sentry_sdk.utils import capture_internal_exceptions, event_from_exception
+from sentry_sdk.utils import (
+    capture_internal_exceptions,
+    event_from_exception,
+    SENSITIVE_DATA_SUBSTITUTE,
+)
 
 if MYPY:
     from typing import Any, Callable, Optional, Union, TypeVar

sentry_sdk/integrations/stdlib.py

@@ -8,7 +8,12 @@
 from sentry_sdk.integrations import Integration
 from sentry_sdk.scope import add_global_event_processor
 from sentry_sdk.tracing_utils import EnvironHeaders
-from sentry_sdk.utils import capture_internal_exceptions, logger, safe_repr
+from sentry_sdk.utils import (
+    capture_internal_exceptions,
+    logger,
+    safe_repr,
+    parse_url,
+)
 
 from sentry_sdk._types import MYPY
 
@@ -79,12 +84,17 @@ def putrequest(self, method, url, *args, **kwargs):
                 url,
             )
 
+        parsed_url = parse_url(real_url, sanitize=False)
+
         span = hub.start_span(
-            op=OP.HTTP_CLIENT, description="%s %s" % (method, real_url)
+            op=OP.HTTP_CLIENT,
+            description="%s %s" % (method, parsed_url.url),
         )
 
         span.set_data("method", method)
-        span.set_data("url", real_url)
+        span.set_data("url", parsed_url.url)
+        span.set_data("http.query", parsed_url.query)
+        span.set_data("http.fragment", parsed_url.fragment)
 
         rv = real_putrequest(self, method, url, *args, **kwargs)
 

sentry_sdk/utils.py

@@ -8,6 +8,25 @@
 import sys
 import threading
 import time
+from collections import namedtuple
+
+try:
+    # Python 3
+    from urllib.parse import parse_qs
+    from urllib.parse import unquote
+    from urllib.parse import urlencode
+    from urllib.parse import urlsplit
+    from urllib.parse import urlunsplit
+
+except ImportError:
+    # Python 2
+    from cgi import parse_qs  # type: ignore
+    from urllib import unquote  # type: ignore
+    from urllib import urlencode  # type: ignore
+    from urlparse import urlsplit  # type: ignore
+    from urlparse import urlunsplit  # type: ignore
+
+
 from datetime import datetime
 from functools import partial
 
@@ -43,13 +62,14 @@
 
 epoch = datetime(1970, 1, 1)
 
-
 # The logger is created here but initialized in the debug support module
 logger = logging.getLogger("sentry_sdk.errors")
 
 MAX_STRING_LENGTH = 1024
 BASE64_ALPHABET = re.compile(r"^[a-zA-Z0-9/+=]*$")
 
+SENSITIVE_DATA_SUBSTITUTE = "[Filtered]"
+
 
 def json_dumps(data):
     # type: (Any) -> bytes
@@ -374,8 +394,6 @@ def removed_because_over_size_limit(cls):
     def substituted_because_contains_sensitive_data(cls):
         # type: () -> AnnotatedValue
         """The actual value was removed because it contained sensitive information."""
-        from sentry_sdk.consts import SENSITIVE_DATA_SUBSTITUTE
-
         return AnnotatedValue(
             value=SENSITIVE_DATA_SUBSTITUTE,
             metadata={
@@ -1163,6 +1181,79 @@ def from_base64(base64_string):
     return utf8_string
 
 
+Components = namedtuple("Components", ["scheme", "netloc", "path", "query", "fragment"])
+
+
+def sanitize_url(url, remove_authority=True, remove_query_values=True):
+    # type: (str, bool, bool) -> str
+    """
+    Removes the authority and query parameter values from a given URL.
+    """
+    parsed_url = urlsplit(url)
+    query_params = parse_qs(parsed_url.query, keep_blank_values=True)
+
+    # strip username:password (netloc can be usr:pwd@example.com)
+    if remove_authority:
+        netloc_parts = parsed_url.netloc.split("@")
+        if len(netloc_parts) > 1:
+            netloc = "%s:%s@%s" % (
+                SENSITIVE_DATA_SUBSTITUTE,
+                SENSITIVE_DATA_SUBSTITUTE,
+                netloc_parts[-1],
+            )
+        else:
+            netloc = parsed_url.netloc
+    else:
+        netloc = parsed_url.netloc
+
+    # strip values from query string
+    if remove_query_values:
+        query_string = unquote(
+            urlencode({key: SENSITIVE_DATA_SUBSTITUTE for key in query_params})
+        )
+    else:
+        query_string = parsed_url.query
+
+    safe_url = urlunsplit(
+        Components(
+            scheme=parsed_url.scheme,
+            netloc=netloc,
+            query=query_string,
+            path=parsed_url.path,
+            fragment=parsed_url.fragment,
+        )
+    )
+
+    return safe_url
+
+
+ParsedUrl = namedtuple("ParsedUrl", ["url", "query", "fragment"])
+
+
+def parse_url(url, sanitize=True):
+
+    # type: (str, bool) -> ParsedUrl
+    """
+    Splits a URL into a url (including path), query and fragment. If sanitize is True, the query
+    parameters will be sanitized to remove sensitive data. The autority (username and password)
+    in the URL will always be removed.
+    """
+    url = sanitize_url(url, remove_authority=True, remove_query_values=sanitize)
+
+    parsed_url = urlsplit(url)
+    base_url = urlunsplit(
+        Components(
+            scheme=parsed_url.scheme,
+            netloc=parsed_url.netloc,
+            query="",
+            path=parsed_url.path,
+            fragment="",
+        )
+    )
+
+    return ParsedUrl(url=base_url, query=parsed_url.query, fragment=parsed_url.fragment)
+
+
 if PY37:
 
     def nanosecond_time():

tests/integrations/httpx/test_httpx.py

@@ -34,6 +34,8 @@ def before_breadcrumb(crumb, hint):
             assert crumb["data"] == {
                 "url": url,
                 "method": "GET",
+                "http.fragment": "",
+                "http.query": "",
                 "status_code": 200,
                 "reason": "OK",
                 "extra": "foo",

tests/integrations/requests/test_requests.py

@@ -20,6 +20,8 @@ def test_crumb_capture(sentry_init, capture_events):
     assert crumb["data"] == {
         "url": "https://httpbin.org/status/418",
         "method": "GET",
+        "http.fragment": "",
+        "http.query": "",
         "status_code": response.status_code,
         "reason": response.reason,
     }