Merge branch '1.x' into w3c/munir/fix-system-test-edgecases

ddtrace/_monkey.py

@@ -110,6 +110,7 @@
         "elasticsearch7",
     ),
     "pynamodb": ("pynamodb",),
+    "rq": ("rq",),
 }
 
 IAST_PATCH = {

ddtrace/contrib/aiobotocore/__init__.py

@@ -1,22 +1,44 @@
 """
 The aiobotocore integration will trace all AWS calls made with the ``aiobotocore``
-library. This integration isn't enabled when applying the default patching.
-To enable it, you must run ``patch_all(aiobotocore=True)``
+library. This integration is not enabled by default.
 
-::
+Enabling
+~~~~~~~~
 
-    import aiobotocore.session
-    from ddtrace import patch
+The aiobotocore integration is not enabled by default. Use
+:func:`patch()<ddtrace.patch>` to enable the integration::
 
-    # If not patched yet, you can patch botocore specifically
+    from ddtrace import patch
     patch(aiobotocore=True)
 
-    # This will report spans with the default instrumentation
-    aiobotocore.session.get_session()
-    lambda_client = session.create_client('lambda', region_name='us-east-1')
+Configuration
+~~~~~~~~~~~~~
+
+.. py:data:: ddtrace.config.aiobotocore['tag_no_params']
+
+    This opts out of the default behavior of adding span tags for a narrow set of API parameters.
+
+    To not collect any API parameters, ``ddtrace.config.aiobotocore.tag_no_params = True`` or by setting the environment
+    variable ``DD_AWS_TAG_NO_PARAMS=true``.
+
+
+    Default: ``False``
+
+.. py:data:: ddtrace.config.aiobotocore['tag_all_params']
+
+    **Deprecated**: This retains the deprecated behavior of adding span tags for
+    all API parameters that are not explicitly excluded by the integration.
+    These deprecated span tags will be added along with the API parameters
+    enabled by default.
+
+    This configuration is ignored if ``tag_no_parms`` (``DD_AWS_TAG_NO_PARAMS``)
+    is set to ``True``.
+
+    To collect all API parameters, ``ddtrace.config.botocore.tag_all_params =
+    True`` or by setting the environment variable ``DD_AWS_TAG_ALL_PARAMS=true``.
+
 
-    # This query generates a trace
-    lambda_client.list_functions()
+    Default: ``False``
 """
 from ...internal.utils.importlib import require_modules
 

ddtrace/contrib/aiobotocore/patch.py

@@ -1,7 +1,10 @@
+import os
+
 import aiobotocore.client
 
 from ddtrace import config
 from ddtrace.internal.utils.version import parse_version
+from ddtrace.vendor import debtcollector
 from ddtrace.vendor import wrapt
 
 from ...constants import ANALYTICS_SAMPLE_RATE_KEY
@@ -12,6 +15,7 @@
 from ...internal.compat import PYTHON_VERSION_INFO
 from ...internal.utils import ArgumentError
 from ...internal.utils import get_argument_value
+from ...internal.utils.formats import asbool
 from ...internal.utils.formats import deep_getattr
 from ...pin import Pin
 from ..trace_utils import unwrap
@@ -31,6 +35,22 @@
 TRACED_ARGS = {"params", "path", "verb"}
 
 
+if os.getenv("DD_AWS_TAG_ALL_PARAMS") is not None:
+    debtcollector.deprecate(
+        "Using environment variable 'DD_AWS_TAG_ALL_PARAMS' is deprecated",
+        message="The aiobotocore integration no longer includes all API parameters by default.",
+        removal_version="2.0.0",
+    )
+
+config._add(
+    "aiobotocore",
+    {
+        "tag_no_params": asbool(os.getenv("DD_AWS_TAG_NO_PARAMS", default=False)),
+        "tag_all_params": asbool(os.getenv("DD_AWS_TAG_ALL_PARAMS", default=False)),
+    },
+)
+
+
 def patch():
     if getattr(aiobotocore.client, "_datadog_patch", False):
         return
@@ -94,13 +114,20 @@ async def _wrapped_api_call(original_func, instance, args, kwargs):
         span.set_tag(SPAN_MEASURED_KEY)
 
         try:
+
             operation = get_argument_value(args, kwargs, 0, "operation_name")
+            params = get_argument_value(args, kwargs, 1, "params")
+
             span.resource = "{}.{}".format(endpoint_name, operation.lower())
+
+            if params and not config.aiobotocore["tag_no_params"]:
+                aws._add_api_param_span_tags(span, endpoint_name, params)
         except ArgumentError:
             operation = None
             span.resource = endpoint_name
 
-        aws.add_span_arg_tags(span, endpoint_name, args, ARGS_NAME, TRACED_ARGS)
+        if not config.aiobotocore["tag_no_params"] and config.aiobotocore["tag_all_params"]:
+            aws.add_span_arg_tags(span, endpoint_name, args, ARGS_NAME, TRACED_ARGS)
 
         region_name = deep_getattr(instance, "meta.region_name")
 

ddtrace/contrib/boto/__init__.py

@@ -1,17 +1,47 @@
 """
 Boto integration will trace all AWS calls made via boto2.
-This integration is automatically patched when using ``patch_all()``::
 
-    import boto.ec2
-    from ddtrace import patch
+Enabling
+~~~~~~~~
+
+The boto integration is enabled automatically when using
+:ref:`ddtrace-run<ddtracerun>` or :func:`patch_all()<ddtrace.patch_all>`.
 
-    # If not patched yet, you can patch boto specifically
+Or use :func:`patch()<ddtrace.patch>` to manually enable the integration::
+
+    from ddtrace import patch
     patch(boto=True)
 
-    # This will report spans with the default instrumentation
-    ec2 = boto.ec2.connect_to_region("us-west-2")
-    # Example of instrumented query
-    ec2.get_all_instances()
+Configuration
+~~~~~~~~~~~~~
+
+.. py:data:: ddtrace.config.boto['tag_no_params']
+
+    This opts out of the default behavior of collecting a narrow set of API
+    parameters as span tags.
+
+    To not collect any API parameters, ``ddtrace.config.boto.tag_no_params =
+    True`` or by setting the environment variable ``DD_AWS_TAG_NO_PARAMS=true``.
+
+
+    Default: ``False``
+
+.. py:data:: ddtrace.config.boto['tag_all_params']
+
+    **Deprecated**: This retains the deprecated behavior of adding span tags for
+    all API parameters that are not explicitly excluded by the integration.
+    These deprecated span tags will be added along with the API parameters
+    enabled by default.
+
+    This configuration is ignored if ``tag_no_parms`` (``DD_AWS_TAG_NO_PARAMS``)
+    is set to ``True``.
+
+    To collect all API parameters, ``ddtrace.config.botocore.tag_all_params =
+    True`` or by setting the environment variable ``DD_AWS_TAG_ALL_PARAMS=true``.
+
+
+    Default: ``False``
+
 """
 
 from ...internal.utils.importlib import require_modules

ddtrace/contrib/boto/patch.py

@@ -1,4 +1,5 @@
 import inspect
+import os
 
 import boto.connection
 
@@ -10,9 +11,11 @@
 from ddtrace.ext import http
 from ddtrace.internal.utils.wrappers import unwrap
 from ddtrace.pin import Pin
+from ddtrace.vendor import debtcollector
 from ddtrace.vendor import wrapt
 
 from ...internal.utils import get_argument_value
+from ...internal.utils.formats import asbool
 
 
 # Original boto client class
@@ -32,6 +35,22 @@
 AWS_AUTH_TRACED_ARGS = {"path", "data", "host"}
 
 
+if os.getenv("DD_AWS_TAG_ALL_PARAMS") is not None:
+    debtcollector.deprecate(
+        "Using environment variable 'DD_AWS_TAG_ALL_PARAMS' is deprecated",
+        message="The boto integration no longer includes all API parameters by default.",
+        removal_version="2.0.0",
+    )
+
+config._add(
+    "boto",
+    {
+        "tag_no_params": asbool(os.getenv("DD_AWS_TAG_NO_PARAMS", default=False)),
+        "tag_all_params": asbool(os.getenv("DD_AWS_TAG_ALL_PARAMS", default=False)),
+    },
+)
+
+
 def patch():
     if getattr(boto.connection, "_datadog_patch", False):
         return
@@ -72,11 +91,17 @@ def patched_query_request(original_func, instance, args, kwargs):
         operation_name = None
         if args:
             operation_name = get_argument_value(args, kwargs, 0, "action")
+            params = get_argument_value(args, kwargs, 1, "params")
+
             span.resource = "%s.%s" % (endpoint_name, operation_name.lower())
+
+            if params and not config.boto["tag_no_params"]:
+                aws._add_api_param_span_tags(span, endpoint_name, params)
         else:
             span.resource = endpoint_name
 
-        aws.add_span_arg_tags(span, endpoint_name, args, AWS_QUERY_ARGS_NAME, AWS_QUERY_TRACED_ARGS)
+        if not config.boto["tag_no_params"] and config.boto["tag_all_params"]:
+            aws.add_span_arg_tags(span, endpoint_name, args, AWS_QUERY_ARGS_NAME, AWS_QUERY_TRACED_ARGS)
 
         # Obtaining region name
         region_name = _get_instance_region_name(instance)
@@ -140,7 +165,8 @@ def patched_auth_request(original_func, instance, args, kwargs):
         else:
             span.resource = endpoint_name
 
-        aws.add_span_arg_tags(span, endpoint_name, args, AWS_AUTH_ARGS_NAME, AWS_AUTH_TRACED_ARGS)
+        if not config.boto["tag_no_params"] and config.boto["tag_all_params"]:
+            aws.add_span_arg_tags(span, endpoint_name, args, AWS_AUTH_ARGS_NAME, AWS_AUTH_TRACED_ARGS)
 
         # Obtaining region name
         region_name = _get_instance_region_name(instance)

ddtrace/contrib/botocore/__init__.py

@@ -52,6 +52,32 @@
 
     See :ref:`HTTP - Custom Error Codes<http-custom-error>` documentation for more examples.
 
+.. py:data:: ddtrace.config.botocore['tag_no_params']
+
+    This opts out of the default behavior of collecting a narrow set of API parameters as span tags.
+
+    To not collect any API parameters, ``ddtrace.config.botocore.tag_no_params = True`` or by setting the environment
+    variable ``DD_AWS_TAG_NO_PARAMS=true``.
+
+
+    Default: ``False``
+
+.. py:data:: ddtrace.config.botocore['tag_all_params']
+
+    **Deprecated**: This retains the deprecated behavior of adding span tags for
+    all API parameters that are not explicitly excluded by the integration.
+    These deprecated span tags will be added along with the API parameters
+    enabled by default.
+
+    This configuration is ignored if ``tag_no_parms`` (``DD_AWS_TAG_NO_PARAMS``)
+    is set to ``True``.
+
+    To collect all API parameters, ``ddtrace.config.botocore.tag_all_params =
+    True`` or by setting the environment variable ``DD_AWS_TAG_ALL_PARAMS=true``.
+
+
+    Default: ``False``
+
 
 Example::
 

ddtrace/contrib/botocore/patch.py

@@ -16,6 +16,7 @@
 
 from ddtrace import config
 from ddtrace.settings.config import Config
+from ddtrace.vendor import debtcollector
 from ddtrace.vendor import wrapt
 
 from ...constants import ANALYTICS_SAMPLE_RATE_KEY
@@ -47,13 +48,23 @@
 log = get_logger(__name__)
 
 
+if os.getenv("DD_AWS_TAG_ALL_PARAMS") is not None:
+    debtcollector.deprecate(
+        "Using environment variable 'DD_AWS_TAG_ALL_PARAMS' is deprecated",
+        message="The botocore integration no longer includes all API parameters by default.",
+        removal_version="2.0.0",
+    )
+
+
 # Botocore default settings
 config._add(
     "botocore",
     {
         "distributed_tracing": asbool(os.getenv("DD_BOTOCORE_DISTRIBUTED_TRACING", default=True)),
         "invoke_with_legacy_context": asbool(os.getenv("DD_BOTOCORE_INVOKE_WITH_LEGACY_CONTEXT", default=False)),
         "operations": collections.defaultdict(Config._HTTPServerConfig),
+        "tag_no_params": asbool(os.getenv("DD_AWS_TAG_NO_PARAMS", default=False)),
+        "tag_all_params": asbool(os.getenv("DD_AWS_TAG_ALL_PARAMS", default=False)),
     },
 )
 
@@ -325,10 +336,14 @@ def patched_api_call(original_func, instance, args, kwargs):
                 except Exception:
                     log.warning("Unable to inject trace context", exc_info=True)
 
+            if params and not config.botocore["tag_no_params"]:
+                aws._add_api_param_span_tags(span, endpoint_name, params)
+
         else:
             span.resource = endpoint_name
 
-        aws.add_span_arg_tags(span, endpoint_name, args, ARGS_NAME, TRACED_ARGS)
+        if not config.botocore["tag_no_params"] and config.botocore["tag_all_params"]:
+            aws.add_span_arg_tags(span, endpoint_name, args, ARGS_NAME, TRACED_ARGS)
 
         region_name = deep_getattr(instance, "meta.region_name")
 

ddtrace/debugging/_debugger.py

@@ -254,10 +254,19 @@ def _dd_debugger_hook(self, probe):
             if isinstance(probe, MetricProbe):
                 # TODO: Handle value expressions
                 assert probe.kind is not None and probe.name is not None
+
+                value = float(probe.value(sys._getframe(1).f_locals)) if probe.value is not None else 1
+
+                # TODO[perf]: We know the tags in advance so we can avoid the
+                # list comprehension.
                 if probe.kind == MetricProbeKind.COUNTER:
-                    # TODO[perf]: We know the tags in advance so we can avoid the
-                    # list comprehension.
-                    self._probe_meter.increment(probe.name, tags=probe.tags)
+                    self._probe_meter.increment(probe.name, value, probe.tags)
+                elif probe.kind == MetricProbeKind.GAUGE:
+                    self._probe_meter.gauge(probe.name, value, probe.tags)
+                elif probe.kind == MetricProbeKind.HISTOGRAM:
+                    self._probe_meter.histogram(probe.name, value, probe.tags)
+                elif probe.kind == MetricProbeKind.DISTRIBUTION:
+                    self._probe_meter.distribution(probe.name, value, probe.tags)
 
                 return
 

ddtrace/debugging/_probe/model.py

@@ -107,3 +107,4 @@ class MetricProbeKind(object):
 class MetricProbe(LineProbe):
     kind = attr.ib(type=Optional[str], default=None)
     name = attr.ib(type=Optional[str], default=None)
+    value = attr.ib(type=Optional[Callable[[Dict[str, Any]], Any]], default=None)