From 23e4f8e19ccf80b167536a5e678f5197a38e7fd8 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 25 Feb 2022 15:49:20 -0800 Subject: [PATCH 01/71] Add stub for SolarWindsFormat custom propagator --- opentelemetry_distro_solarwinds/propagator.py | 37 +++++++++++++++++++ setup.cfg | 2 + 2 files changed, 39 insertions(+) create mode 100644 opentelemetry_distro_solarwinds/propagator.py diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py new file mode 100644 index 00000000..6bf1e29a --- /dev/null +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -0,0 +1,37 @@ +import typing + +from opentelemetry.context.context import Context +from opentelemetry.propagators import textmap + +class SolarWindsFormat(textmap.TextMapPropagator): + + def extract( + self, + carrier: textmap.CarrierT, + context: typing.Optional[Context] = None, + getter: textmap.Getter = textmap.default_getter, + ) -> Context: + """ + Extracts from carrier into SpanContext + """ + if context is None: + context = Context() + return context + + def inject( + self, + carrier: textmap.CarrierT, + context: typing.Optional[Context] = None, + setter: textmap.Setter = textmap.default_setter, + ) -> None: + """ + Injects from SpanContext into carrier + """ + pass + + @property + def fields(self) -> typing.Set[str]: + """ + Returns a set with the fields set in `inject` + """ + return {'foo', 'bar'} diff --git a/setup.cfg b/setup.cfg index 430717b5..fa31670a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,3 +25,5 @@ install_requires = [options.entry_points] opentelemetry_distro = solarwinds_distro = opentelemetry_distro_solarwinds.distro:SolarWindsDistro +opentelemetry_propagator = + solarwinds = opentelemetry_distro_solarwinds.propagator:SolarWindsFormat From b591d4fbac9ec603d3ef6f79dad4e6b3ee3ca18d Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 25 Feb 2022 18:23:56 -0800 Subject: [PATCH 02/71] (WIP) tracestate sometimes gets updated with span_id --- opentelemetry_distro_solarwinds/propagator.py | 92 +++++++++++++++++-- 1 file changed, 85 insertions(+), 7 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 6bf1e29a..81d3f056 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -1,9 +1,27 @@ +import logging +import re import typing +from opentelemetry import trace from opentelemetry.context.context import Context from opentelemetry.propagators import textmap +from opentelemetry.trace.span import TraceState + +logger = logging.getLogger(__file__) class SolarWindsFormat(textmap.TextMapPropagator): + """ + Extracts and injects SolarWinds tracestate header + + See also https://www.w3.org/TR/trace-context-1/ + """ + _TRACEPARENT_HEADER_NAME = "traceparent" + _TRACESTATE_HEADER_NAME = "tracestate" + _TRACEPARENT_HEADER_FORMAT = ( + "^[ \t]*([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})" + + "(-.*)?[ \t]*$" + ) + _TRACEPARENT_HEADER_FORMAT_RE = re.compile(_TRACEPARENT_HEADER_FORMAT) def extract( self, @@ -12,11 +30,43 @@ def extract( getter: textmap.Getter = textmap.default_getter, ) -> Context: """ - Extracts from carrier into SpanContext + Extracts sw tracestate from carrier into SpanContext """ - if context is None: - context = Context() - return context + # TODO: Is this needed if this is always used + # in composite with TraceContextTextMapPropagator? + # If not, return context? + # TODO: If so, are basic validity checks needed? + + # Get span_id, trace_flags from carrier's traceparent header + traceparent_header = getter.get(carrier, self._TRACEPARENT_HEADER_NAME) + if not traceparent_header: + return context + match = re.search(self._TRACEPARENT_HEADER_FORMAT_RE, traceparent_header[0]) + if not match: + return context + version = match.group(1) + trace_id = match.group(2) + span_id = match.group(3) + trace_flags = match.group(4) + + # Prepare context with carrier's tracestate + tracestate_header = getter.get(carrier, self._TRACESTATE_HEADER_NAME) + # TODO: Should sw tracestate be added/updated here? + if tracestate_header is None: + tracestate = None + else: + tracestate = TraceState.from_header(tracestate_header) + + span_context = trace.SpanContext( + trace_id=int(trace_id, 16), + span_id=int(span_id, 16), + is_remote=True, + trace_flags=trace.TraceFlags(trace_flags), + trace_state=tracestate, + ) + return trace.set_span_in_context( + trace.NonRecordingSpan(span_context), context + ) def inject( self, @@ -25,13 +75,41 @@ def inject( setter: textmap.Setter = textmap.default_setter, ) -> None: """ - Injects from SpanContext into carrier + Injects sw tracestate from SpanContext into carrier for HTTP request + + See also: https://www.w3.org/TR/trace-context-1/#mutating-the-tracestate-field """ - pass + # TODO: Are basic validity checks necessary if this is always used + # in composite with TraceContextTextMapPropagator? + span = trace.get_current_span(context) + span_context = span.get_span_context() + span_id = span_context.span_id + trace_state = span_context.trace_state + trace_flags = span_context.trace_flags + + # TODO: This isn't working + # Prepare carrier with context's or new tracestate + if trace_state: + # Check if trace_state already contains sw KV + if "sw" in trace_state.keys(): + # If so, modify current span_id and trace_flags, and move to beginning of list + logger.debug(f"Updating trace state with {span_id}-{trace_flags}") + trace_state.update("sw", f"{span_id}-{trace_flags}") + else: + # If not, add sw KV to beginning of list + logger.debug(f"Adding KV to trace state with {span_id}-{trace_flags}") + trace_state.add("sw", f"{span_id}-{trace_flags}") + else: + logger.debug(f"Creating new trace state with {span_id}-{trace_flags}") + trace_state = TraceState([("sw", f"{span_id}-{trace_flags}")]) + + setter.set( + carrier, self._TRACESTATE_HEADER_NAME, trace_state.to_header() + ) @property def fields(self) -> typing.Set[str]: """ Returns a set with the fields set in `inject` """ - return {'foo', 'bar'} + return {self._TRACEPARENT_HEADER_NAME, self._TRACESTATE_HEADER_NAME} From 9ceb4df6bdc5bf04fa4330af1d9fe3ecdea5d219 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 28 Feb 2022 14:15:39 -0800 Subject: [PATCH 03/71] Add helpers to format span_id, trace_flags --- opentelemetry_distro_solarwinds/propagator.py | 95 ++++++++++--------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 81d3f056..1d4d7dad 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -10,8 +10,7 @@ logger = logging.getLogger(__file__) class SolarWindsFormat(textmap.TextMapPropagator): - """ - Extracts and injects SolarWinds tracestate header + """Extracts and injects SolarWinds tracestate header See also https://www.w3.org/TR/trace-context-1/ """ @@ -29,44 +28,43 @@ def extract( context: typing.Optional[Context] = None, getter: textmap.Getter = textmap.default_getter, ) -> Context: - """ - Extracts sw tracestate from carrier into SpanContext - """ - # TODO: Is this needed if this is always used + """Extracts sw tracestate from carrier into SpanContext""" + return context + + # TODO: Is the below needed if this is always used # in composite with TraceContextTextMapPropagator? - # If not, return context? + # If not, return context as currently # TODO: If so, are basic validity checks needed? - - # Get span_id, trace_flags from carrier's traceparent header - traceparent_header = getter.get(carrier, self._TRACEPARENT_HEADER_NAME) - if not traceparent_header: - return context - match = re.search(self._TRACEPARENT_HEADER_FORMAT_RE, traceparent_header[0]) - if not match: - return context - version = match.group(1) - trace_id = match.group(2) - span_id = match.group(3) - trace_flags = match.group(4) - - # Prepare context with carrier's tracestate - tracestate_header = getter.get(carrier, self._TRACESTATE_HEADER_NAME) - # TODO: Should sw tracestate be added/updated here? - if tracestate_header is None: - tracestate = None - else: - tracestate = TraceState.from_header(tracestate_header) - - span_context = trace.SpanContext( - trace_id=int(trace_id, 16), - span_id=int(span_id, 16), - is_remote=True, - trace_flags=trace.TraceFlags(trace_flags), - trace_state=tracestate, - ) - return trace.set_span_in_context( - trace.NonRecordingSpan(span_context), context - ) + + # # Get span_id, trace_flags from carrier's traceparent header + # traceparent_header = getter.get(carrier, self._TRACEPARENT_HEADER_NAME) + # if not traceparent_header: + # return context + # match = re.search(self._TRACEPARENT_HEADER_FORMAT_RE, traceparent_header[0]) + # if not match: + # return context + # version = match.group(1) + # trace_id = match.group(2) + # span_id = match.group(3) + # trace_flags = match.group(4) + + # # Prepare context with carrier's tracestate + # tracestate_header = getter.get(carrier, self._TRACESTATE_HEADER_NAME) + # if tracestate_header is None: + # tracestate = None + # else: + # tracestate = TraceState.from_header(tracestate_header) + + # span_context = trace.SpanContext( + # trace_id=int(trace_id, 16), + # span_id=int(span_id, 16), + # is_remote=True, + # trace_flags=trace.TraceFlags(trace_flags), + # trace_state=tracestate, + # ) + # return trace.set_span_in_context( + # trace.NonRecordingSpan(span_context), context + # ) def inject( self, @@ -74,8 +72,7 @@ def inject( context: typing.Optional[Context] = None, setter: textmap.Setter = textmap.default_setter, ) -> None: - """ - Injects sw tracestate from SpanContext into carrier for HTTP request + """Injects sw tracestate from SpanContext into carrier for HTTP request See also: https://www.w3.org/TR/trace-context-1/#mutating-the-tracestate-field """ @@ -83,17 +80,17 @@ def inject( # in composite with TraceContextTextMapPropagator? span = trace.get_current_span(context) span_context = span.get_span_context() - span_id = span_context.span_id + span_id = self.format_span_id(span_context.span_id) + trace_flags = self.format_trace_flags(span_context.trace_flags) trace_state = span_context.trace_state - trace_flags = span_context.trace_flags - # TODO: This isn't working # Prepare carrier with context's or new tracestate if trace_state: # Check if trace_state already contains sw KV if "sw" in trace_state.keys(): # If so, modify current span_id and trace_flags, and move to beginning of list logger.debug(f"Updating trace state with {span_id}-{trace_flags}") + # TODO: Update isn't working trace_state.update("sw", f"{span_id}-{trace_flags}") else: # If not, add sw KV to beginning of list @@ -109,7 +106,13 @@ def inject( @property def fields(self) -> typing.Set[str]: - """ - Returns a set with the fields set in `inject` - """ + """Returns a set with the fields set in `inject`""" return {self._TRACEPARENT_HEADER_NAME, self._TRACESTATE_HEADER_NAME} + + def format_span_id(self, span_id: int) -> str: + """Formats span ID as 16-byte hexadecimal string""" + return format(span_id, "016x") + + def format_trace_flags(self, trace_flags: int) -> str: + """Formats trace flags as 8-bit field""" + return format(trace_flags, "02x") From a5701a1f21791afba64d22aac1d6de9c1adcb873 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 28 Feb 2022 15:42:23 -0800 Subject: [PATCH 04/71] Add temp manual trace_state update logic --- opentelemetry_distro_solarwinds/propagator.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 1d4d7dad..56370847 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -88,10 +88,25 @@ def inject( if trace_state: # Check if trace_state already contains sw KV if "sw" in trace_state.keys(): + # If so, modify current span_id and trace_flags, and move to beginning of list logger.debug(f"Updating trace state with {span_id}-{trace_flags}") - # TODO: Update isn't working - trace_state.update("sw", f"{span_id}-{trace_flags}") + + # TODO: Python OTEL TraceState update isn't working + # trace_state.update("sw", f"{span_id}-{trace_flags}") + + ## Temp: Manual trace_state update + from collections import OrderedDict + prev_state = OrderedDict(trace_state.items()) + logger.debug(f"prev_state is {prev_state}") + prev_state["sw"] = f"{span_id}-{trace_flags}" + logger.debug(f"Updated prev_state is {prev_state}") + prev_state.move_to_end("sw", last=False) + logger.debug(f"Reordered prev_state is {prev_state}") + new_state = list(prev_state.items()) + logger.debug(f"new_state list is {new_state}") + trace_state = TraceState(new_state) + else: # If not, add sw KV to beginning of list logger.debug(f"Adding KV to trace state with {span_id}-{trace_flags}") From a6533bca1ff25f6304f84fac8e9be9134819d78f Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Tue, 1 Mar 2022 13:40:23 -0800 Subject: [PATCH 05/71] Rm SolarWindsFormat.extract duplicate code from TraceContextTextMapPropagator --- opentelemetry_distro_solarwinds/propagator.py | 39 ++----------------- 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 56370847..c3861cdc 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -28,44 +28,11 @@ def extract( context: typing.Optional[Context] = None, getter: textmap.Getter = textmap.default_getter, ) -> Context: - """Extracts sw tracestate from carrier into SpanContext""" + """Extracts sw tracestate from carrier into SpanContext + + Must be used in composite with TraceContextTextMapPropagator""" return context - # TODO: Is the below needed if this is always used - # in composite with TraceContextTextMapPropagator? - # If not, return context as currently - # TODO: If so, are basic validity checks needed? - - # # Get span_id, trace_flags from carrier's traceparent header - # traceparent_header = getter.get(carrier, self._TRACEPARENT_HEADER_NAME) - # if not traceparent_header: - # return context - # match = re.search(self._TRACEPARENT_HEADER_FORMAT_RE, traceparent_header[0]) - # if not match: - # return context - # version = match.group(1) - # trace_id = match.group(2) - # span_id = match.group(3) - # trace_flags = match.group(4) - - # # Prepare context with carrier's tracestate - # tracestate_header = getter.get(carrier, self._TRACESTATE_HEADER_NAME) - # if tracestate_header is None: - # tracestate = None - # else: - # tracestate = TraceState.from_header(tracestate_header) - - # span_context = trace.SpanContext( - # trace_id=int(trace_id, 16), - # span_id=int(span_id, 16), - # is_remote=True, - # trace_flags=trace.TraceFlags(trace_flags), - # trace_state=tracestate, - # ) - # return trace.set_span_in_context( - # trace.NonRecordingSpan(span_context), context - # ) - def inject( self, carrier: textmap.CarrierT, From 05b88873996c4a3565d569be100dcd8d53ca39e2 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 7 Mar 2022 17:03:13 -0800 Subject: [PATCH 06/71] Fix propagator use of tracestate.update --- opentelemetry_distro_solarwinds/propagator.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index c3861cdc..be63f29c 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -55,24 +55,9 @@ def inject( if trace_state: # Check if trace_state already contains sw KV if "sw" in trace_state.keys(): - # If so, modify current span_id and trace_flags, and move to beginning of list logger.debug(f"Updating trace state with {span_id}-{trace_flags}") - - # TODO: Python OTEL TraceState update isn't working - # trace_state.update("sw", f"{span_id}-{trace_flags}") - - ## Temp: Manual trace_state update - from collections import OrderedDict - prev_state = OrderedDict(trace_state.items()) - logger.debug(f"prev_state is {prev_state}") - prev_state["sw"] = f"{span_id}-{trace_flags}" - logger.debug(f"Updated prev_state is {prev_state}") - prev_state.move_to_end("sw", last=False) - logger.debug(f"Reordered prev_state is {prev_state}") - new_state = list(prev_state.items()) - logger.debug(f"new_state list is {new_state}") - trace_state = TraceState(new_state) + trace_state = trace_state.update("sw", f"{span_id}-{trace_flags}") else: # If not, add sw KV to beginning of list From a7c6244c416b155c54eb05c2d4cd46eb6a84b8b9 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 7 Mar 2022 17:54:49 -0800 Subject: [PATCH 07/71] Rename to SwSampler, WIP decision-tracestate-attributes management --- opentelemetry_distro_solarwinds/distro.py | 4 +- opentelemetry_distro_solarwinds/sampler.py | 146 ++++++++++++++++++--- 2 files changed, 133 insertions(+), 17 deletions(-) diff --git a/opentelemetry_distro_solarwinds/distro.py b/opentelemetry_distro_solarwinds/distro.py index cdfd7c3b..640484a6 100644 --- a/opentelemetry_distro_solarwinds/distro.py +++ b/opentelemetry_distro_solarwinds/distro.py @@ -6,7 +6,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry_distro_solarwinds.exporter import SolarWindsSpanExporter -from opentelemetry_distro_solarwinds.sampler import ParentBasedAoSampler +from opentelemetry_distro_solarwinds.sampler import ParentBasedSwSampler class SolarWindsDistro(BaseDistro): @@ -18,7 +18,7 @@ class SolarWindsDistro(BaseDistro): def _configure(self, **kwargs): # automatically make use of custom SolarWinds sampler trace.set_tracer_provider( - TracerProvider(sampler=ParentBasedAoSampler())) + TracerProvider(sampler=ParentBasedSwSampler())) # Automatically configure the SolarWinds Span exporter span_exporter = BatchSpanProcessor(SolarWindsSpanExporter()) trace.get_tracer_provider().add_span_processor(span_exporter) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 5f14525e..e001373d 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -4,11 +4,13 @@ """ import logging +from types import MappingProxyType from typing import Optional, Sequence from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult) -from opentelemetry.trace import SpanKind +from opentelemetry.trace import SpanKind, get_current_span +from opentelemetry.trace.span import TraceState from opentelemetry.util.types import Attributes from opentelemetry_distro_solarwinds.extension.oboe import Context @@ -16,10 +18,20 @@ logger = logging.getLogger(__name__) -class _AoSampler(Sampler): - """AppOptics Custom sampler which obeys configuration options provided by the AO Backend.""" +def oboe_to_otel_decision(do_metrics, do_sample): + """Helper to make OTel decision from oboe outputs""" + decision = Decision.DROP + if do_metrics: + # TODO: need to eck what record only actually means and how metrics work in OTel + decision = Decision.RECORD_ONLY + if do_sample: + decision = Decision.RECORD_AND_SAMPLE + return decision + +class _SwSampler(Sampler): + """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" def get_description(self) -> str: - return "AppOptics custom sampler" + return "SolarWinds custom opentelemetry sampler" def should_sample( self, @@ -31,15 +43,91 @@ def should_sample( links: Sequence["Link"] = None, trace_state: "TraceState" = None, ) -> "SamplingResult": + """ + Use liboboe trace decision for returned OTel SamplingResult + """ + + parent_span_context = get_current_span( + parent_context + ).get_span_context() + + # (0) Debugging + # import ipdb + # ipdb.set_trace() + # Unique the first time then always same + # logger.debug(f"trace_id: {trace_id}") + # logger.debug(f"trace_id as 032X: {trace_id:032X}".lower()) + # get_current_span > get_value > get_current() gets current context in execution + # logger.debug(f"parent_context: {parent_context}") + logger.debug(f"parent_span_context: {parent_span_context}") + # [] at django-a, non-empty at django-b (from custom propagator?) + # logger.debug(f"parent_span_context.trace_state: {parent_span_context.trace_state}") + # False at django-a, True at django-b + logger.debug(f"parent_span_context.is_valid: {parent_span_context.is_valid}") + # # False at django-a, True at django-b + # # At this point with super() init of ParentBased + # # we shouldn't need to check this here + # logger.debug(f"parent_span_context.is_remote: {parent_span_context.is_remote}") + + do_metrics = None + do_sample = None + trace_state = parent_span_context.trace_state + + # traceparent none (root span) or not valid + # or + # tracestate none or not parsable + # or + # tracestate does not include valid sw + if not parent_span_context.is_valid \ + or not trace_state \ + or not trace_state.get("sw", None): + + # New sampling decision + do_metrics, do_sample, \ + _, _, _, _, _, _, _, _, _ = Context.getDecisions( + f"{trace_id:032X}".lower(), + repr(trace_state) # ??? + ) + + # # Debugging + # trace_id_hex_str = f"{trace_id:032X}".lower() + # trace_state_str = repr(trace_state) + # logger.debug(f"getDecisions with {trace_id_hex_str} and {trace_state_str}: {do_sample}") - do_metrics, do_sample, _, _, _, _, _, _, _, _, _ = Context.getDecisions( - None) - decision = Decision.DROP - if do_metrics: - # TODO: need to eck what record only actually means and how metrics work in OTel - decision = Decision.RECORD_ONLY - if do_sample: - decision = Decision.RECORD_AND_SAMPLE + # New tracestate with sampling decision + trace_state = TraceState([( + "sw", + f"{parent_span_context.span_id:016X}-{do_sample:02X}".lower() + )]) + logger.debug(f"New trace_state: {trace_state}") + + # tracestate has valid sw included + else: + # Continue existing sw sampling decision + do_sample = trace_state.get("sw").split('-')[1] + if not do_sample in ["00", "01"]: + raise Exception(f"Something went wrong checking tracestate: {do_sample}") + + # Update trace_state with span_id and sw trace_flags + trace_state = trace_state.update( + "sw", + f"{parent_span_context.span_id:016X}-{do_sample}".lower() + ) + logger.debug(f"Updated trace_state: {trace_state}") + + decision = oboe_to_otel_decision(do_metrics, do_sample) + logger.debug(f"decision for otel: {decision}") + + # TODO + # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate + attributes = { + "sw.tracestate_parent_id": f"{parent_span_context.span_id:016X}".lower(), + "sw.w3c.tracestate": trace_state + } + logger.debug(f"attributes: {attributes}") + + # Return result to start_span caller + # start_span creates a new SpanContext and Span after sampling_result return SamplingResult( decision, attributes if decision != Decision.DROP else None, @@ -47,10 +135,38 @@ def should_sample( ) -class ParentBasedAoSampler(ParentBased): +class ParentBasedSwSampler(ParentBased): """ Sampler that respects its parent span's sampling decision, but otherwise - samples according to the configurations from the AO backend. + samples according to the configurations from the NH/AO backend. """ def __init__(self): - super().__init__(root=_AoSampler()) + super().__init__( + # Use liboboe if no parent span + root=_SwSampler(), + # Use liboboe if parent span is_remote + remote_parent_sampled=_SwSampler(), + remote_parent_not_sampled=_SwSampler(), + # Use OTEL defaults if parent span is_local + ) + + def should_sample( + self, + parent_context: Optional["Context"], + trace_id: int, + name: str, + kind: SpanKind = None, + attributes: Attributes = None, + links: Sequence["Link"] = None, + trace_state: "TraceState" = None + ) -> "SamplingResult": + + return super().should_sample( + parent_context, + trace_id, + name, + kind, + attributes, + links, + trace_state + ) From d91e2762ec7797f2defac4aa680763aba49d87d2 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 9 Mar 2022 09:02:57 -0800 Subject: [PATCH 08/71] WIP attributes --- opentelemetry_distro_solarwinds/propagator.py | 20 +++++++- opentelemetry_distro_solarwinds/sampler.py | 47 +++++++++---------- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index be63f29c..c86a1c80 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -21,6 +21,8 @@ class SolarWindsFormat(textmap.TextMapPropagator): + "(-.*)?[ \t]*$" ) _TRACEPARENT_HEADER_FORMAT_RE = re.compile(_TRACEPARENT_HEADER_FORMAT) + _SW_TRACESTATE_PARENT_ID_HEADER_NAME = "sw.tracestate_parent_id" + _SW_W3C_TRACESTATE_HEADER_NAME = "sw.w3c.tracestate" def extract( self, @@ -71,10 +73,26 @@ def inject( carrier, self._TRACESTATE_HEADER_NAME, trace_state.to_header() ) + # TODO Does the propagator prepare carrier with sw.tracestate_parent_id and sw.w3c.tracestate headers? + tracestate_parent_id = span_id + setter.set( + carrier, self._SW_W3C_TRACESTATE_HEADER_NAME, trace_state.to_header() + ) + setter.set( + carrier, + self._SW_TRACESTATE_PARENT_ID_HEADER_NAME, + tracestate_parent_id + ) + @property def fields(self) -> typing.Set[str]: """Returns a set with the fields set in `inject`""" - return {self._TRACEPARENT_HEADER_NAME, self._TRACESTATE_HEADER_NAME} + return { + self._TRACEPARENT_HEADER_NAME, + self._TRACESTATE_HEADER_NAME, + self._SW_TRACESTATE_PARENT_ID_HEADER_NAME, + self._SW_W3C_TRACESTATE_HEADER_NAME, + } def format_span_id(self, span_id: int) -> str: """Formats span ID as 16-byte hexadecimal string""" diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index e001373d..f2d910ae 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -54,20 +54,8 @@ def should_sample( # (0) Debugging # import ipdb # ipdb.set_trace() - # Unique the first time then always same - # logger.debug(f"trace_id: {trace_id}") - # logger.debug(f"trace_id as 032X: {trace_id:032X}".lower()) - # get_current_span > get_value > get_current() gets current context in execution - # logger.debug(f"parent_context: {parent_context}") logger.debug(f"parent_span_context: {parent_span_context}") - # [] at django-a, non-empty at django-b (from custom propagator?) - # logger.debug(f"parent_span_context.trace_state: {parent_span_context.trace_state}") - # False at django-a, True at django-b logger.debug(f"parent_span_context.is_valid: {parent_span_context.is_valid}") - # # False at django-a, True at django-b - # # At this point with super() init of ParentBased - # # we shouldn't need to check this here - # logger.debug(f"parent_span_context.is_remote: {parent_span_context.is_remote}") do_metrics = None do_sample = None @@ -89,11 +77,6 @@ def should_sample( repr(trace_state) # ??? ) - # # Debugging - # trace_id_hex_str = f"{trace_id:032X}".lower() - # trace_state_str = repr(trace_state) - # logger.debug(f"getDecisions with {trace_id_hex_str} and {trace_state_str}: {do_sample}") - # New tracestate with sampling decision trace_state = TraceState([( "sw", @@ -118,13 +101,27 @@ def should_sample( decision = oboe_to_otel_decision(do_metrics, do_sample) logger.debug(f"decision for otel: {decision}") - # TODO # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate - attributes = { - "sw.tracestate_parent_id": f"{parent_span_context.span_id:016X}".lower(), - "sw.w3c.tracestate": trace_state - } - logger.debug(f"attributes: {attributes}") + logger.debug(f"Received attributes: {attributes}") + if not attributes: + attributes = { + "sw.tracestate_parent_id": f"{parent_span_context.span_id:016X}".lower(), + "sw.w3c.tracestate": repr(trace_state) + } + logger.debug(f"New attributes: {attributes}") + else: + # Copy existing KV into new_attributes for modification + new_attributes = {} + for k,v in attributes.items(): + new_attributes[k] = v + # TODO Update sw of sw.w3c.tracestate + + # Replace + attributes = new_attributes + logger.debug(f"Updated attributes: {attributes}") + + # attributes must be immutable + attributes = MappingProxyType(attributes) # Return result to start_span caller # start_span creates a new SpanContext and Span after sampling_result @@ -144,9 +141,9 @@ def __init__(self): super().__init__( # Use liboboe if no parent span root=_SwSampler(), - # Use liboboe if parent span is_remote + # Use liboboe if parent span is_remote and sampled remote_parent_sampled=_SwSampler(), - remote_parent_not_sampled=_SwSampler(), + # Use OTEL default if parent span is_remote and NOT sampled (never sample) # Use OTEL defaults if parent span is_local ) From ae290031a92cac406bcefd5dbb561195982d6803 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 9 Mar 2022 13:53:42 -0800 Subject: [PATCH 09/71] Rm unnecessary attributes touching by Propagator --- opentelemetry_distro_solarwinds/propagator.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index c86a1c80..0bd8bf89 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -21,8 +21,6 @@ class SolarWindsFormat(textmap.TextMapPropagator): + "(-.*)?[ \t]*$" ) _TRACEPARENT_HEADER_FORMAT_RE = re.compile(_TRACEPARENT_HEADER_FORMAT) - _SW_TRACESTATE_PARENT_ID_HEADER_NAME = "sw.tracestate_parent_id" - _SW_W3C_TRACESTATE_HEADER_NAME = "sw.w3c.tracestate" def extract( self, @@ -73,25 +71,12 @@ def inject( carrier, self._TRACESTATE_HEADER_NAME, trace_state.to_header() ) - # TODO Does the propagator prepare carrier with sw.tracestate_parent_id and sw.w3c.tracestate headers? - tracestate_parent_id = span_id - setter.set( - carrier, self._SW_W3C_TRACESTATE_HEADER_NAME, trace_state.to_header() - ) - setter.set( - carrier, - self._SW_TRACESTATE_PARENT_ID_HEADER_NAME, - tracestate_parent_id - ) - @property def fields(self) -> typing.Set[str]: """Returns a set with the fields set in `inject`""" return { self._TRACEPARENT_HEADER_NAME, - self._TRACESTATE_HEADER_NAME, - self._SW_TRACESTATE_PARENT_ID_HEADER_NAME, - self._SW_W3C_TRACESTATE_HEADER_NAME, + self._TRACESTATE_HEADER_NAME } def format_span_id(self, span_id: int) -> str: From 394ae52c2d3d4952541fd560399e640185d43576 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 9 Mar 2022 15:22:54 -0800 Subject: [PATCH 10/71] SwSampler adds/updates sw.w3c.tracestate, sw.tracestate_parent_id --- opentelemetry_distro_solarwinds/sampler.py | 55 +++++++++++++++------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index f2d910ae..b0789cbc 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -30,6 +30,12 @@ def oboe_to_otel_decision(do_metrics, do_sample): class _SwSampler(Sampler): """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" + def __init__( + self, + is_root_span: bool = False + ): + self._is_root_span = is_root_span + def get_description(self) -> str: return "SolarWinds custom opentelemetry sampler" @@ -46,19 +52,14 @@ def should_sample( """ Use liboboe trace decision for returned OTel SamplingResult """ - + # TraceContextTextMapPropagator extracts trace_state etc to SpanContext parent_span_context = get_current_span( parent_context ).get_span_context() - - # (0) Debugging - # import ipdb - # ipdb.set_trace() - logger.debug(f"parent_span_context: {parent_span_context}") - logger.debug(f"parent_span_context.is_valid: {parent_span_context.is_valid}") do_metrics = None do_sample = None + span_id = parent_span_context.span_id trace_state = parent_span_context.trace_state # traceparent none (root span) or not valid @@ -71,16 +72,17 @@ def should_sample( or not trace_state.get("sw", None): # New sampling decision + # TODO: Double-check formats liboboe expects do_metrics, do_sample, \ _, _, _, _, _, _, _, _, _ = Context.getDecisions( f"{trace_id:032X}".lower(), - repr(trace_state) # ??? + f"{span_id:016X}-{parent_span_context.trace_flags:02X}".lower() ) - # New tracestate with sampling decision + # New tracestate with result of sampling decision trace_state = TraceState([( "sw", - f"{parent_span_context.span_id:016X}-{do_sample:02X}".lower() + f"{span_id:016X}-{do_sample:02X}".lower() )]) logger.debug(f"New trace_state: {trace_state}") @@ -94,7 +96,7 @@ def should_sample( # Update trace_state with span_id and sw trace_flags trace_state = trace_state.update( "sw", - f"{parent_span_context.span_id:016X}-{do_sample}".lower() + f"{span_id:016X}-{do_sample}".lower() ) logger.debug(f"Updated trace_state: {trace_state}") @@ -105,20 +107,37 @@ def should_sample( logger.debug(f"Received attributes: {attributes}") if not attributes: attributes = { - "sw.tracestate_parent_id": f"{parent_span_context.span_id:016X}".lower(), - "sw.w3c.tracestate": repr(trace_state) + "sw.tracestate_parent_id": f"{span_id:016X}".lower(), + "sw.w3c.tracestate": trace_state.to_header() } - logger.debug(f"New attributes: {attributes}") + logger.debug(f"Set new attributes: {attributes}") else: - # Copy existing KV into new_attributes for modification + # Copy existing MappingProxyType KV into new_attributes for modification + # attributes may have other vendor info etc new_attributes = {} for k,v in attributes.items(): new_attributes[k] = v - # TODO Update sw of sw.w3c.tracestate + + # Add or Update sw KV in sw.w3c.tracestate + if not new_attributes.get("sw.w3c.tracestate", None) or new_attributes.get("sw.w3c.tracestate"): + new_attributes["sw.w3c.tracestate"] = trace_state.to_header() + else: + attr_trace_state = TraceState.from_header(new_attributes["sw.w3c.tracestate"]) + attr_trace_state.update( + "sw", + f"{span_id:016X}-{do_sample}".lower() + ) + new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() + + # Only set sw.tracestate_parent_id on the entry (root) span for this service + # TODO: Or only at root span for the trace? + # Or if sw.tracestate_parent_id is not set? + # if self._is_root_span: + new_attributes["sw.tracestate_parent_id"] = f"{span_id:016X}".lower() # Replace attributes = new_attributes - logger.debug(f"Updated attributes: {attributes}") + logger.debug(f"Set updated attributes: {attributes}") # attributes must be immutable attributes = MappingProxyType(attributes) @@ -140,7 +159,7 @@ class ParentBasedSwSampler(ParentBased): def __init__(self): super().__init__( # Use liboboe if no parent span - root=_SwSampler(), + root=_SwSampler(is_root_span=True), # Use liboboe if parent span is_remote and sampled remote_parent_sampled=_SwSampler(), # Use OTEL default if parent span is_remote and NOT sampled (never sample) From f6ff7f4b413383113e0bfe2a0325514e1f7338b0 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 9 Mar 2022 15:36:16 -0800 Subject: [PATCH 11/71] SwSampler adds/updates sw.parent_span_id --- opentelemetry_distro_solarwinds/sampler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index b0789cbc..444cad47 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -3,6 +3,7 @@ The custom sampler will fetch sampling configurations for the SolarWinds backend. """ +from inspect import trace import logging from types import MappingProxyType from typing import Optional, Sequence @@ -103,10 +104,12 @@ def should_sample( decision = oboe_to_otel_decision(do_metrics, do_sample) logger.debug(f"decision for otel: {decision}") + # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate logger.debug(f"Received attributes: {attributes}") if not attributes: attributes = { + "sw.parent_span_id": f"{span_id:016X}".lower(), "sw.tracestate_parent_id": f"{span_id:016X}".lower(), "sw.w3c.tracestate": trace_state.to_header() } @@ -129,6 +132,7 @@ def should_sample( ) new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() + new_attributes["sw.parent_span_id"] = f"{span_id:016X}".lower() # Only set sw.tracestate_parent_id on the entry (root) span for this service # TODO: Or only at root span for the trace? # Or if sw.tracestate_parent_id is not set? From c380527b37ed7e0165854ceecd88e39572245850 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 9 Mar 2022 16:49:02 -0800 Subject: [PATCH 12/71] Refactor/Break up _SwSampler.should_sample into several helpers --- opentelemetry_distro_solarwinds/sampler.py | 171 +++++++++++---------- 1 file changed, 94 insertions(+), 77 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 444cad47..383e7a30 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -3,7 +3,6 @@ The custom sampler will fetch sampling configurations for the SolarWinds backend. """ -from inspect import trace import logging from types import MappingProxyType from typing import Optional, Sequence @@ -18,140 +17,158 @@ logger = logging.getLogger(__name__) - -def oboe_to_otel_decision(do_metrics, do_sample): - """Helper to make OTel decision from oboe outputs""" - decision = Decision.DROP - if do_metrics: - # TODO: need to eck what record only actually means and how metrics work in OTel - decision = Decision.RECORD_ONLY - if do_sample: - decision = Decision.RECORD_AND_SAMPLE - return decision - class _SwSampler(Sampler): """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" def __init__( self, is_root_span: bool = False ): + self._attributes = None + self._decision = None + self._do_metrics = None + self._do_sample = None self._is_root_span = is_root_span + self._parent_span_context = None + self._span_id = None + self._trace_state = None def get_description(self) -> str: return "SolarWinds custom opentelemetry sampler" - def should_sample( - self, - parent_context: Optional["Context"], - trace_id: int, - name: str, - kind: SpanKind = None, - attributes: Attributes = None, - links: Sequence["Link"] = None, - trace_state: "TraceState" = None, - ) -> "SamplingResult": - """ - Use liboboe trace decision for returned OTel SamplingResult - """ - # TraceContextTextMapPropagator extracts trace_state etc to SpanContext - parent_span_context = get_current_span( - parent_context - ).get_span_context() - - do_metrics = None - do_sample = None - span_id = parent_span_context.span_id - trace_state = parent_span_context.trace_state + def oboe_to_otel_decision(self) -> None: + """Helper to make OTel decision from oboe outputs""" + decision = Decision.DROP + if self._do_metrics: + # TODO: need to eck what record only actually means and how metrics work in OTel + decision = Decision.RECORD_ONLY + if self._do_sample: + decision = Decision.RECORD_AND_SAMPLE + return decision + + def make_trace_decision(self, trace_id: int) -> None: + """Helper to make oboe trace decision""" + # TODO: Double-check formats liboboe expects + self._do_metrics, self._do_sample, \ + _, _, _, _, _, _, _, _, _ = Context.getDecisions( + f"{trace_id:032X}".lower(), + f"{self._span_id:016X}-{self._parent_span_context.trace_flags:02X}".lower() + ) + def set_trace_state(self, trace_id: int) -> None: + """Helper to set trace_state for sampling result based on parent span. + Makes trace decision if needed + """ # traceparent none (root span) or not valid - # or - # tracestate none or not parsable - # or - # tracestate does not include valid sw - if not parent_span_context.is_valid \ - or not trace_state \ - or not trace_state.get("sw", None): + # or tracestate none or not parsable + # or tracestate does not include valid sw + if not self._parent_span_context.is_valid \ + or not self._trace_state \ + or not self._trace_state.get("sw", None): # New sampling decision - # TODO: Double-check formats liboboe expects - do_metrics, do_sample, \ - _, _, _, _, _, _, _, _, _ = Context.getDecisions( - f"{trace_id:032X}".lower(), - f"{span_id:016X}-{parent_span_context.trace_flags:02X}".lower() - ) + self.make_trace_decision(trace_id) # New tracestate with result of sampling decision - trace_state = TraceState([( + self._trace_state = TraceState([( "sw", - f"{span_id:016X}-{do_sample:02X}".lower() + f"{self._span_id:016X}-{self._do_sample:02X}".lower() )]) - logger.debug(f"New trace_state: {trace_state}") + logger.debug(f"New trace_state: {self._trace_state}") # tracestate has valid sw included else: # Continue existing sw sampling decision - do_sample = trace_state.get("sw").split('-')[1] - if not do_sample in ["00", "01"]: - raise Exception(f"Something went wrong checking tracestate: {do_sample}") + self._do_sample = self._trace_state.get("sw").split('-')[1] + if not self._do_sample in ["00", "01"]: + raise Exception(f"Something went wrong checking tracestate: {self._do_sample}") # Update trace_state with span_id and sw trace_flags - trace_state = trace_state.update( + self._trace_state = self._trace_state.update( "sw", - f"{span_id:016X}-{do_sample}".lower() + f"{self._span_id:016X}-{self._do_sample}".lower() ) - logger.debug(f"Updated trace_state: {trace_state}") - - decision = oboe_to_otel_decision(do_metrics, do_sample) - logger.debug(f"decision for otel: {decision}") + logger.debug(f"Updated trace_state: {self._trace_state}") + def set_decision(self) -> None: + """Helper to set OTel decision for sampling result""" + self._decision = self.oboe_to_otel_decision() + logger.debug(f"decision for otel: {self._decision}") + def set_attributes(self) -> None: + """Helper to set attributes based on parent span""" # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate - logger.debug(f"Received attributes: {attributes}") - if not attributes: - attributes = { - "sw.parent_span_id": f"{span_id:016X}".lower(), - "sw.tracestate_parent_id": f"{span_id:016X}".lower(), - "sw.w3c.tracestate": trace_state.to_header() + logger.debug(f"Received attributes: {self._attributes}") + if not self._attributes: + self._attributes = { + "sw.parent_span_id": f"{self._span_id:016X}".lower(), + "sw.tracestate_parent_id": f"{self._span_id:016X}".lower(), + "sw.w3c.tracestate": self._trace_state.to_header() } - logger.debug(f"Set new attributes: {attributes}") + logger.debug(f"Set new attributes: {self._attributes}") else: # Copy existing MappingProxyType KV into new_attributes for modification # attributes may have other vendor info etc new_attributes = {} - for k,v in attributes.items(): + for k,v in self._attributes.items(): new_attributes[k] = v # Add or Update sw KV in sw.w3c.tracestate if not new_attributes.get("sw.w3c.tracestate", None) or new_attributes.get("sw.w3c.tracestate"): - new_attributes["sw.w3c.tracestate"] = trace_state.to_header() + new_attributes["sw.w3c.tracestate"] = self._trace_state.to_header() else: attr_trace_state = TraceState.from_header(new_attributes["sw.w3c.tracestate"]) attr_trace_state.update( "sw", - f"{span_id:016X}-{do_sample}".lower() + f"{self._span_id:016X}-{self._do_sample}".lower() ) new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() - new_attributes["sw.parent_span_id"] = f"{span_id:016X}".lower() + new_attributes["sw.parent_span_id"] = f"{self._span_id:016X}".lower() # Only set sw.tracestate_parent_id on the entry (root) span for this service # TODO: Or only at root span for the trace? # Or if sw.tracestate_parent_id is not set? # if self._is_root_span: - new_attributes["sw.tracestate_parent_id"] = f"{span_id:016X}".lower() + new_attributes["sw.tracestate_parent_id"] = f"{self._span_id:016X}".lower() # Replace - attributes = new_attributes - logger.debug(f"Set updated attributes: {attributes}") + self._attributes = new_attributes + logger.debug(f"Set updated attributes: {self._attributes}") # attributes must be immutable - attributes = MappingProxyType(attributes) + self._attributes = MappingProxyType(self._attributes) + + def should_sample( + self, + parent_context: Optional["Context"], + trace_id: int, + name: str, + kind: SpanKind = None, + attributes: Attributes = None, + links: Sequence["Link"] = None, + trace_state: "TraceState" = None, + ) -> "SamplingResult": + """ + Use liboboe trace decision for returned OTel SamplingResult + """ + # TraceContextTextMapPropagator extracts trace_state etc to SpanContext + self._parent_span_context = get_current_span( + parent_context + ).get_span_context() + self._span_id = self._parent_span_context.span_id + self._trace_state = self._parent_span_context.trace_state + self._attributes = attributes + + # Set args for new SamplingResult + self.set_trace_state(trace_id) + self.set_decision() + self.set_attributes() # Return result to start_span caller # start_span creates a new SpanContext and Span after sampling_result return SamplingResult( - decision, - attributes if decision != Decision.DROP else None, - trace_state, + self._decision, + self._attributes if self._decision != Decision.DROP else None, + self._trace_state, ) From 5659f5b2af8cbf5295060798372e85c47aa3de7a Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 9 Mar 2022 17:12:00 -0800 Subject: [PATCH 13/71] Fix format of args sent to Context::getDecisions --- opentelemetry_distro_solarwinds/sampler.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 383e7a30..aef7252b 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -14,6 +14,7 @@ from opentelemetry.util.types import Attributes from opentelemetry_distro_solarwinds.extension.oboe import Context +from opentelemetry_distro_solarwinds.ot_ao_transformer import transform_id logger = logging.getLogger(__name__) @@ -30,6 +31,7 @@ def __init__( self._is_root_span = is_root_span self._parent_span_context = None self._span_id = None + self._traceparent = None self._trace_state = None def get_description(self) -> str: @@ -45,16 +47,19 @@ def oboe_to_otel_decision(self) -> None: decision = Decision.RECORD_AND_SAMPLE return decision - def make_trace_decision(self, trace_id: int) -> None: + def make_trace_decision(self) -> None: """Helper to make oboe trace decision""" - # TODO: Double-check formats liboboe expects + in_xtrace = self._traceparent + tracestate = f"{self._span_id:016X}-{self._parent_span_context.trace_flags:02X}".lower() + logger.debug(f"Making oboe decision with in_xtrace {in_xtrace}, tracestate {tracestate}") + self._do_metrics, self._do_sample, \ _, _, _, _, _, _, _, _, _ = Context.getDecisions( - f"{trace_id:032X}".lower(), - f"{self._span_id:016X}-{self._parent_span_context.trace_flags:02X}".lower() + in_xtrace, + tracestate ) - def set_trace_state(self, trace_id: int) -> None: + def set_trace_state(self) -> None: """Helper to set trace_state for sampling result based on parent span. Makes trace decision if needed """ @@ -66,7 +71,7 @@ def set_trace_state(self, trace_id: int) -> None: or not self._trace_state.get("sw", None): # New sampling decision - self.make_trace_decision(trace_id) + self.make_trace_decision() # New tracestate with result of sampling decision self._trace_state = TraceState([( @@ -154,12 +159,13 @@ def should_sample( self._parent_span_context = get_current_span( parent_context ).get_span_context() + self._traceparent = transform_id(self._parent_span_context) self._span_id = self._parent_span_context.span_id self._trace_state = self._parent_span_context.trace_state self._attributes = attributes # Set args for new SamplingResult - self.set_trace_state(trace_id) + self.set_trace_state() self.set_decision() self.set_attributes() From f17b2b8dd9060097b2666dd68313efeefaadc523 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 10 Mar 2022 16:02:39 -0800 Subject: [PATCH 14/71] Fix duplication of sw.parent_span_id by Sampler; stop invalid attributes creation (0000) --- opentelemetry_distro_solarwinds/sampler.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index aef7252b..0fba431d 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -101,11 +101,15 @@ def set_decision(self) -> None: def set_attributes(self) -> None: """Helper to set attributes based on parent span""" - # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate logger.debug(f"Received attributes: {self._attributes}") - if not self._attributes: + if not self._parent_span_context.is_valid or not self._trace_state: + # Trace's root span has no valid traceparent nor tracestate + # so we don't set additional attributes + logger.debug(f"No valid traceparent or no tracestate - not setting attributes") + return + # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate + elif not self._attributes: self._attributes = { - "sw.parent_span_id": f"{self._span_id:016X}".lower(), "sw.tracestate_parent_id": f"{self._span_id:016X}".lower(), "sw.w3c.tracestate": self._trace_state.to_header() } @@ -128,11 +132,7 @@ def set_attributes(self) -> None: ) new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() - new_attributes["sw.parent_span_id"] = f"{self._span_id:016X}".lower() # Only set sw.tracestate_parent_id on the entry (root) span for this service - # TODO: Or only at root span for the trace? - # Or if sw.tracestate_parent_id is not set? - # if self._is_root_span: new_attributes["sw.tracestate_parent_id"] = f"{self._span_id:016X}".lower() # Replace From 52e655a2ecbc738187a18c07b313911e532f0ea0 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 14 Mar 2022 14:00:15 -0700 Subject: [PATCH 15/71] install_requires less strict for Django ASGI support --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index fa31670a..08e5fd46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,9 +18,9 @@ classifiers = [options] python_requires = >=3.4 install_requires = - opentelemetry-api == 1.3.0 - opentelemetry-sdk == 1.3.0 - opentelemetry-instrumentation == 0.22b0 + opentelemetry-api >= 1.3.0 + opentelemetry-sdk >= 1.3.0 + opentelemetry-instrumentation >= 0.22b0 [options.entry_points] opentelemetry_distro = From 69a487a596f5d375a41a6ccab8698c3b9e3ca7e7 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 16 Mar 2022 15:50:35 -0700 Subject: [PATCH 16/71] Mv SolarWindsFormat to SolarWindsPropagator --- opentelemetry_distro_solarwinds/propagator.py | 9 +++------ setup.cfg | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 0bd8bf89..6246052b 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__file__) -class SolarWindsFormat(textmap.TextMapPropagator): +class SolarWindsPropagator(textmap.TextMapPropagator): """Extracts and injects SolarWinds tracestate header See also https://www.w3.org/TR/trace-context-1/ @@ -39,12 +39,9 @@ def inject( context: typing.Optional[Context] = None, setter: textmap.Setter = textmap.default_setter, ) -> None: - """Injects sw tracestate from SpanContext into carrier for HTTP request + """Injects sw tracestate from SpanContext into carrier for HTTP request. - See also: https://www.w3.org/TR/trace-context-1/#mutating-the-tracestate-field - """ - # TODO: Are basic validity checks necessary if this is always used - # in composite with TraceContextTextMapPropagator? + Must be used in composite with TraceContextTextMapPropagator""" span = trace.get_current_span(context) span_context = span.get_span_context() span_id = self.format_span_id(span_context.span_id) diff --git a/setup.cfg b/setup.cfg index 08e5fd46..cd46efee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,4 +26,4 @@ install_requires = opentelemetry_distro = solarwinds_distro = opentelemetry_distro_solarwinds.distro:SolarWindsDistro opentelemetry_propagator = - solarwinds = opentelemetry_distro_solarwinds.propagator:SolarWindsFormat + solarwinds = opentelemetry_distro_solarwinds.propagator:SolarWindsPropagator From cb5800e0a0a43e91578eb5ad3be8991cb0d7ba55 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 16 Mar 2022 16:44:18 -0700 Subject: [PATCH 17/71] Mv ot_ao_transformer to w3c_transformer, used by Exporter Sampler Propagator --- opentelemetry_distro_solarwinds/exporter.py | 4 +- .../ot_ao_transformer.py | 17 -------- opentelemetry_distro_solarwinds/propagator.py | 17 ++++---- opentelemetry_distro_solarwinds/sampler.py | 30 ++++++++++---- .../w3c_transformer.py | 39 +++++++++++++++++++ 5 files changed, 70 insertions(+), 37 deletions(-) delete mode 100644 opentelemetry_distro_solarwinds/ot_ao_transformer.py create mode 100644 opentelemetry_distro_solarwinds/w3c_transformer.py diff --git a/opentelemetry_distro_solarwinds/exporter.py b/opentelemetry_distro_solarwinds/exporter.py index bc6973c2..19c81022 100644 --- a/opentelemetry_distro_solarwinds/exporter.py +++ b/opentelemetry_distro_solarwinds/exporter.py @@ -11,7 +11,7 @@ from opentelemetry_distro_solarwinds.extension.oboe import (Context, Metadata, Reporter) -from opentelemetry_distro_solarwinds.ot_ao_transformer import transform_id +from opentelemetry_distro_solarwinds.w3c_transformer import traceparent_from_context logger = logging.getLogger(__file__) @@ -122,4 +122,4 @@ def _initialize_solarwinds_reporter(self): @staticmethod def _build_metadata(span_context): - return Metadata.fromString(transform_id(span_context)) + return Metadata.fromString(traceparent_from_context(span_context)) diff --git a/opentelemetry_distro_solarwinds/ot_ao_transformer.py b/opentelemetry_distro_solarwinds/ot_ao_transformer.py deleted file mode 100644 index 21f01cdf..00000000 --- a/opentelemetry_distro_solarwinds/ot_ao_transformer.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Provides functionality to transform OpenTelemetry Data to SolarWinds AppOptics data. -""" - -import logging -import os - -logger = logging.getLogger(__file__) - - -def transform_id(span_context): - """Generates a liboboe W3C compatible trace_context from provided OTel span context.""" - xtr = "00-{0:032X}-{1:016X}-{2:02X}".format(span_context.trace_id, - span_context.span_id, - span_context.trace_flags) - logger.debug("Generated trace_context %s from span context %s", xtr, - span_context) - return xtr diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 6246052b..50f0b091 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -7,6 +7,11 @@ from opentelemetry.propagators import textmap from opentelemetry.trace.span import TraceState +from opentelemetry_distro_solarwinds.w3c_transformer import ( + span_id_from_int, + trace_flags_from_int +) + logger = logging.getLogger(__file__) class SolarWindsPropagator(textmap.TextMapPropagator): @@ -44,8 +49,8 @@ def inject( Must be used in composite with TraceContextTextMapPropagator""" span = trace.get_current_span(context) span_context = span.get_span_context() - span_id = self.format_span_id(span_context.span_id) - trace_flags = self.format_trace_flags(span_context.trace_flags) + span_id = span_id_from_int(span_context.span_id) + trace_flags = trace_flags_from_int(span_context.trace_flags) trace_state = span_context.trace_state # Prepare carrier with context's or new tracestate @@ -75,11 +80,3 @@ def fields(self) -> typing.Set[str]: self._TRACEPARENT_HEADER_NAME, self._TRACESTATE_HEADER_NAME } - - def format_span_id(self, span_id: int) -> str: - """Formats span ID as 16-byte hexadecimal string""" - return format(span_id, "016x") - - def format_trace_flags(self, trace_flags: int) -> str: - """Formats trace flags as 8-bit field""" - return format(trace_flags, "02x") diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 0fba431d..dcb1a5e9 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -14,7 +14,12 @@ from opentelemetry.util.types import Attributes from opentelemetry_distro_solarwinds.extension.oboe import Context -from opentelemetry_distro_solarwinds.ot_ao_transformer import transform_id +from opentelemetry_distro_solarwinds.w3c_transformer import ( + span_id_from_int, + traceparent_from_context, + sw_from_context, + sw_from_span_and_decision +) logger = logging.getLogger(__name__) @@ -50,7 +55,7 @@ def oboe_to_otel_decision(self) -> None: def make_trace_decision(self) -> None: """Helper to make oboe trace decision""" in_xtrace = self._traceparent - tracestate = f"{self._span_id:016X}-{self._parent_span_context.trace_flags:02X}".lower() + tracestate = sw_from_context(self._parent_span_context) logger.debug(f"Making oboe decision with in_xtrace {in_xtrace}, tracestate {tracestate}") self._do_metrics, self._do_sample, \ @@ -76,7 +81,10 @@ def set_trace_state(self) -> None: # New tracestate with result of sampling decision self._trace_state = TraceState([( "sw", - f"{self._span_id:016X}-{self._do_sample:02X}".lower() + sw_from_span_and_decision( + self._span_id, + self._do_sample + ) )]) logger.debug(f"New trace_state: {self._trace_state}") @@ -90,7 +98,10 @@ def set_trace_state(self) -> None: # Update trace_state with span_id and sw trace_flags self._trace_state = self._trace_state.update( "sw", - f"{self._span_id:016X}-{self._do_sample}".lower() + sw_from_span_and_decision( + self._span_id, + self._do_sample + ) ) logger.debug(f"Updated trace_state: {self._trace_state}") @@ -110,7 +121,7 @@ def set_attributes(self) -> None: # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate elif not self._attributes: self._attributes = { - "sw.tracestate_parent_id": f"{self._span_id:016X}".lower(), + "sw.tracestate_parent_id": span_id_from_int(self._span_id), "sw.w3c.tracestate": self._trace_state.to_header() } logger.debug(f"Set new attributes: {self._attributes}") @@ -128,12 +139,15 @@ def set_attributes(self) -> None: attr_trace_state = TraceState.from_header(new_attributes["sw.w3c.tracestate"]) attr_trace_state.update( "sw", - f"{self._span_id:016X}-{self._do_sample}".lower() + sw_from_span_and_decision( + self._span_id, + self._do_sample + ) ) new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() # Only set sw.tracestate_parent_id on the entry (root) span for this service - new_attributes["sw.tracestate_parent_id"] = f"{self._span_id:016X}".lower() + new_attributes["sw.tracestate_parent_id"] = span_id_from_int(self._span_id) # Replace self._attributes = new_attributes @@ -159,7 +173,7 @@ def should_sample( self._parent_span_context = get_current_span( parent_context ).get_span_context() - self._traceparent = transform_id(self._parent_span_context) + self._traceparent = traceparent_from_context(self._parent_span_context) self._span_id = self._parent_span_context.span_id self._trace_state = self._parent_span_context.trace_state self._attributes = attributes diff --git a/opentelemetry_distro_solarwinds/w3c_transformer.py b/opentelemetry_distro_solarwinds/w3c_transformer.py new file mode 100644 index 00000000..23dc76b4 --- /dev/null +++ b/opentelemetry_distro_solarwinds/w3c_transformer.py @@ -0,0 +1,39 @@ +"""Provides functionality to transform OpenTelemetry Data to SolarWinds AppOptics data. +""" + +import logging +from opentelemetry.context.context import Context + +logger = logging.getLogger(__file__) + +def span_id_from_int(span_id: int) -> str: + """Formats span ID as 16-byte hexadecimal string""" + return "{:016x}".format(span_id) + +def trace_flags_from_int(trace_flags: int) -> str: + """Formats trace flags as 8-bit field""" + return "{:02x}".format(trace_flags) + +def traceparent_from_context(span_context: Context) -> str: + """Generates a liboboe W3C compatible trace_context from provided OTel span context.""" + xtr = "00-{0:032X}-{1:016X}-{2:02X}".format(span_context.trace_id, + span_context.span_id, + span_context.trace_flags) + logger.debug("Generated traceparent %s from SpanContext %s", xtr, + span_context) + return xtr + +def sw_from_context(span_context: Context) -> str: + """Formats tracestate sw value from SpanContext + as 16-byte span_id with 8-bit trace_flags. + + Example: 1122334455667788-01""" + return "{0:016x}-{1:02x}".format(span_context.span_id, + span_context.trace_flags) + +def sw_from_span_and_decision(span_id: int, decision: str) -> str: + """Formats tracestate sw value from span_id and liboboe decision + as 16-byte span_id with 8-bit trace_flags. + + Example: 1122334455667788-01""" + return "{0:016x}-{1}".format(span_id, decision) From 537cf0a1c9fe6e515054eead3f8adec18a9c26d6 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 16 Mar 2022 17:04:25 -0700 Subject: [PATCH 18/71] Simplify variables used in Propagator.inject --- opentelemetry_distro_solarwinds/propagator.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 50f0b091..09ddc5ce 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -7,10 +7,7 @@ from opentelemetry.propagators import textmap from opentelemetry.trace.span import TraceState -from opentelemetry_distro_solarwinds.w3c_transformer import ( - span_id_from_int, - trace_flags_from_int -) +from opentelemetry_distro_solarwinds.w3c_transformer import sw_from_context logger = logging.getLogger(__file__) @@ -49,25 +46,24 @@ def inject( Must be used in composite with TraceContextTextMapPropagator""" span = trace.get_current_span(context) span_context = span.get_span_context() - span_id = span_id_from_int(span_context.span_id) - trace_flags = trace_flags_from_int(span_context.trace_flags) trace_state = span_context.trace_state + sw_value = sw_from_context(span_context) # Prepare carrier with context's or new tracestate if trace_state: # Check if trace_state already contains sw KV if "sw" in trace_state.keys(): # If so, modify current span_id and trace_flags, and move to beginning of list - logger.debug(f"Updating trace state with {span_id}-{trace_flags}") - trace_state = trace_state.update("sw", f"{span_id}-{trace_flags}") + logger.debug(f"Updating trace state with {sw_value}") + trace_state = trace_state.update("sw", sw_value) else: # If not, add sw KV to beginning of list - logger.debug(f"Adding KV to trace state with {span_id}-{trace_flags}") - trace_state.add("sw", f"{span_id}-{trace_flags}") + logger.debug(f"Adding KV to trace state with {sw_value}") + trace_state.add("sw", sw_value) else: - logger.debug(f"Creating new trace state with {span_id}-{trace_flags}") - trace_state = TraceState([("sw", f"{span_id}-{trace_flags}")]) + logger.debug(f"Creating new trace state with {sw_value}") + trace_state = TraceState([("sw", sw_value)]) setter.set( carrier, self._TRACESTATE_HEADER_NAME, trace_state.to_header() From 88529a1c34c923fa10300bdce89d257f25b779e2 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 17 Mar 2022 15:26:29 -0700 Subject: [PATCH 19/71] Refactor of _SwSampler --- opentelemetry_distro_solarwinds/sampler.py | 321 +++++++++++------- .../w3c_transformer.py | 4 - 2 files changed, 200 insertions(+), 125 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index dcb1a5e9..d47ecc43 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -4,13 +4,14 @@ """ import logging +import re from types import MappingProxyType from typing import Optional, Sequence from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult) -from opentelemetry.trace import SpanKind, get_current_span -from opentelemetry.trace.span import TraceState +from opentelemetry.trace import Link, SpanKind, get_current_span +from opentelemetry.trace.span import SpanContext, TraceState from opentelemetry.util.types import Attributes from opentelemetry_distro_solarwinds.extension.oboe import Context @@ -21,140 +22,212 @@ sw_from_span_and_decision ) +SW_FORMAT = "^\w{16}-0[0|1]$" logger = logging.getLogger(__name__) -class _SwSampler(Sampler): - """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" + +class _LiboboeDecision(): + """Represents a liboboe decision""" def __init__( self, - is_root_span: bool = False + do_metrics: str, + do_sample: str ): - self._attributes = None - self._decision = None - self._do_metrics = None - self._do_sample = None + self.do_metrics = do_metrics + self.do_sample = do_sample + + +class _SwSampler(Sampler): + """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" + def __init__(self, is_root_span: bool = False): + # TODO use or remove self._is_root_span = is_root_span - self._parent_span_context = None - self._span_id = None - self._traceparent = None - self._trace_state = None def get_description(self) -> str: return "SolarWinds custom opentelemetry sampler" - def oboe_to_otel_decision(self) -> None: - """Helper to make OTel decision from oboe outputs""" + def create_new_liboboe_decision( + self, + parent_span_context: SpanContext + ) -> _LiboboeDecision: + """Creates new liboboe decision using parent span context.""" + in_xtrace = traceparent_from_context(parent_span_context) + tracestate = sw_from_context(parent_span_context) + logger.debug(f"Making oboe decision with in_xtrace {in_xtrace}, tracestate {tracestate}") + do_metrics, do_sample, \ + _, _, _, _, _, _, _, _, _ = Context.getDecisions( + in_xtrace, + tracestate + ) + return _LiboboeDecision(do_metrics, do_sample) + + def continue_liboboe_decision( + self, + parent_span_context: SpanContext + ) -> _LiboboeDecision: + """Creates liboboe decision to continue parent trace state decision""" + trace_state = parent_span_context.trace_state + try: + sw_value = trace_state.get("sw") + except AttributeError: + logger.warning("Cannot continue decision if sw not in \ + tracestate ({0}). Making new decision.".format(trace_state)) + return self.create_new_liboboe_decision(parent_span_context) + if not re.match(SW_FORMAT, sw_value): + logger.warning("Cannot continue decision if tracestate sw not \ + in format <16_byte_span_id>-<8_bit_trace_flags>, nor if \ + trace_flags is anything but 01 or 00 ({0}). Making \ + new decision.".format(trace_state)) + return self.create_new_liboboe_decision(parent_span_context) + + do_sample = sw_value.split("-")[1] + # TODO how do metrics work in OTel + do_metrics = None + logger.debug("Continuing decision as do_metrics: {0}, do_sample: \ + {1}".format(do_metrics, do_sample)) + return _LiboboeDecision(do_metrics, do_sample) + + def calculate_liboboe_decision( + self, + parent_span_context: SpanContext + ) -> _LiboboeDecision: + """Calculates oboe trace decision based on parent span context.""" + # No (valid) parent i.e. root span + if not parent_span_context or not parent_span_context.is_valid: + decision = self.create_new_liboboe_decision(parent_span_context) + else: + # tracestate nonexistent/non-parsable, or no sw KV + trace_state = parent_span_context.trace_state + if not trace_state or not trace_state.get("sw", None): + decision = self.create_new_liboboe_decision(parent_span_context) + # tracestate has sw KV + else: + decision = self.continue_liboboe_decision(parent_span_context) + return decision + + def otel_decision_from_liboboe( + self, + liboboe_decision: _LiboboeDecision + ) -> None: + """Formats OTel decision from liboboe decision""" decision = Decision.DROP - if self._do_metrics: + if liboboe_decision.do_metrics: # TODO: need to eck what record only actually means and how metrics work in OTel decision = Decision.RECORD_ONLY - if self._do_sample: + if liboboe_decision.do_sample: decision = Decision.RECORD_AND_SAMPLE + logger.debug(f"OTel decision created: {decision}") return decision - def make_trace_decision(self) -> None: - """Helper to make oboe trace decision""" - in_xtrace = self._traceparent - tracestate = sw_from_context(self._parent_span_context) - logger.debug(f"Making oboe decision with in_xtrace {in_xtrace}, tracestate {tracestate}") - - self._do_metrics, self._do_sample, \ - _, _, _, _, _, _, _, _, _ = Context.getDecisions( - in_xtrace, - tracestate - ) - - def set_trace_state(self) -> None: - """Helper to set trace_state for sampling result based on parent span. - Makes trace decision if needed - """ - # traceparent none (root span) or not valid - # or tracestate none or not parsable - # or tracestate does not include valid sw - if not self._parent_span_context.is_valid \ - or not self._trace_state \ - or not self._trace_state.get("sw", None): - - # New sampling decision - self.make_trace_decision() - - # New tracestate with result of sampling decision - self._trace_state = TraceState([( - "sw", - sw_from_span_and_decision( - self._span_id, - self._do_sample - ) - )]) - logger.debug(f"New trace_state: {self._trace_state}") + def create_new_trace_state( + self, + decision: _LiboboeDecision, + parent_span_context: SpanContext + ) -> TraceState: + """Creates new TraceState based on trace decision and parent span id.""" + trace_state = TraceState([( + "sw", + sw_from_span_and_decision( + parent_span_context.span_id, + decision.do_sample + ) + )]) + logger.debug(f"Created new trace_state: {trace_state}") + return trace_state - # tracestate has valid sw included + def calculate_trace_state( + self, + decision: _LiboboeDecision, + parent_span_context: SpanContext + ) -> TraceState: + """Calculates trace_state based on parent span context and trace decision""" + # No valid parent i.e. root span + if not parent_span_context.is_valid: + trace_state = self.create_new_trace_state(decision, parent_span_context) else: - # Continue existing sw sampling decision - self._do_sample = self._trace_state.get("sw").split('-')[1] - if not self._do_sample in ["00", "01"]: - raise Exception(f"Something went wrong checking tracestate: {self._do_sample}") - - # Update trace_state with span_id and sw trace_flags - self._trace_state = self._trace_state.update( - "sw", - sw_from_span_and_decision( - self._span_id, - self._do_sample + # tracestate nonexistent/non-parsable, or no sw KV + trace_state = parent_span_context.trace_state + if not trace_state or not trace_state.get("sw", None): + trace_state = self.create_new_trace_state(decision, parent_span_context) + # tracestate has sw KV + else: + # Update trace_state with span_id and sw trace_flags + trace_state = trace_state.update( + "sw", + sw_from_span_and_decision( + parent_span_context.span_id, + decision.do_sample + ) ) - ) - logger.debug(f"Updated trace_state: {self._trace_state}") - - def set_decision(self) -> None: - """Helper to set OTel decision for sampling result""" - self._decision = self.oboe_to_otel_decision() - logger.debug(f"decision for otel: {self._decision}") - - def set_attributes(self) -> None: - """Helper to set attributes based on parent span""" - logger.debug(f"Received attributes: {self._attributes}") - if not self._parent_span_context.is_valid or not self._trace_state: - # Trace's root span has no valid traceparent nor tracestate - # so we don't set additional attributes - logger.debug(f"No valid traceparent or no tracestate - not setting attributes") - return + logger.debug(f"Updated trace_state: {trace_state}") + return trace_state + + def calculate_attributes( + self, + attributes: Attributes, + decision: _LiboboeDecision, + trace_state: TraceState, + parent_span_context: SpanContext + ) -> Attributes or None: + """Calculates Attributes or None based on trace decision, trace state, + parent span context, and existing attributes.""" + logger.debug(f"Received attributes: {attributes}") + + # Don't set attributes if not tracing + if self.otel_decision_from_liboboe(decision) == Decision.DROP: + logger.debug(f"Trace decision is to drop - not setting \ + attributes") + return None + # Trace's root span has no valid traceparent nor tracestate + # so we don't set additional attributes + if not parent_span_context.is_valid or not trace_state: + logger.debug(f"No valid traceparent or no tracestate - not \ + setting attributes") + return None + # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate - elif not self._attributes: - self._attributes = { - "sw.tracestate_parent_id": span_id_from_int(self._span_id), - "sw.w3c.tracestate": self._trace_state.to_header() + if not attributes: + attributes = { + "sw.tracestate_parent_id": span_id_from_int( + parent_span_context.span_id + ), + "sw.w3c.tracestate": trace_state.to_header() } - logger.debug(f"Set new attributes: {self._attributes}") + logger.debug(f"Created new attributes: {attributes}") else: # Copy existing MappingProxyType KV into new_attributes for modification # attributes may have other vendor info etc new_attributes = {} - for k,v in self._attributes.items(): + for k,v in attributes.items(): new_attributes[k] = v - # Add or Update sw KV in sw.w3c.tracestate - if not new_attributes.get("sw.w3c.tracestate", None) or new_attributes.get("sw.w3c.tracestate"): - new_attributes["sw.w3c.tracestate"] = self._trace_state.to_header() + if not new_attributes.get("sw.w3c.tracestate", None): + # Add new sw.w3c.tracestate KV + new_attributes["sw.w3c.tracestate"] = trace_state.to_header() else: - attr_trace_state = TraceState.from_header(new_attributes["sw.w3c.tracestate"]) + # Update existing sw.w3c.tracestate KV + attr_trace_state = TraceState.from_header( + new_attributes["sw.w3c.tracestate"] + ) attr_trace_state.update( "sw", sw_from_span_and_decision( - self._span_id, - self._do_sample + parent_span_context.span_id, + decision.do_sample ) ) new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() # Only set sw.tracestate_parent_id on the entry (root) span for this service - new_attributes["sw.tracestate_parent_id"] = span_id_from_int(self._span_id) + new_attributes["sw.tracestate_parent_id"] = span_id_from_int( + parent_span_context.span_id + ) - # Replace - self._attributes = new_attributes - logger.debug(f"Set updated attributes: {self._attributes}") + attributes = new_attributes + logger.debug(f"Set updated attributes: {attributes}") - # attributes must be immutable - self._attributes = MappingProxyType(self._attributes) + # attributes must be immutable for SamplingResult + return MappingProxyType(attributes) def should_sample( self, @@ -167,28 +240,32 @@ def should_sample( trace_state: "TraceState" = None, ) -> "SamplingResult": """ - Use liboboe trace decision for returned OTel SamplingResult + Calculates SamplingResult based on calculation of new/continued trace + decision, new/updated trace state, and new/updated attributes. """ - # TraceContextTextMapPropagator extracts trace_state etc to SpanContext - self._parent_span_context = get_current_span( + parent_span_context = get_current_span( parent_context ).get_span_context() - self._traceparent = traceparent_from_context(self._parent_span_context) - self._span_id = self._parent_span_context.span_id - self._trace_state = self._parent_span_context.trace_state - self._attributes = attributes - - # Set args for new SamplingResult - self.set_trace_state() - self.set_decision() - self.set_attributes() - - # Return result to start_span caller - # start_span creates a new SpanContext and Span after sampling_result + + decision = self.calculate_liboboe_decision(parent_span_context) + + # TODO Set differently if not decision.RECORD_AND_SAMPLE + trace_state = self.calculate_trace_state( + decision, + parent_span_context + ) + attributes = self.calculate_attributes( + attributes, + decision, + trace_state, + parent_span_context + ) + decision = self.otel_decision_from_liboboe(decision) + return SamplingResult( - self._decision, - self._attributes if self._decision != Decision.DROP else None, - self._trace_state, + decision, + attributes if decision != Decision.DROP else None, + trace_state, ) @@ -198,13 +275,15 @@ class ParentBasedSwSampler(ParentBased): samples according to the configurations from the NH/AO backend. """ def __init__(self): + """ + Uses _SwSampler/liboboe if no parent span. + Uses _SwSampler/liboboe if parent span is_remote and sampled. + Uses OTEL default if parent span is_remote and NOT sampled (never sample). + Uses OTEL defaults if parent span is_local. + """ super().__init__( - # Use liboboe if no parent span root=_SwSampler(is_root_span=True), - # Use liboboe if parent span is_remote and sampled - remote_parent_sampled=_SwSampler(), - # Use OTEL default if parent span is_remote and NOT sampled (never sample) - # Use OTEL defaults if parent span is_local + remote_parent_sampled=_SwSampler() ) def should_sample( diff --git a/opentelemetry_distro_solarwinds/w3c_transformer.py b/opentelemetry_distro_solarwinds/w3c_transformer.py index 23dc76b4..a8401e2b 100644 --- a/opentelemetry_distro_solarwinds/w3c_transformer.py +++ b/opentelemetry_distro_solarwinds/w3c_transformer.py @@ -10,10 +10,6 @@ def span_id_from_int(span_id: int) -> str: """Formats span ID as 16-byte hexadecimal string""" return "{:016x}".format(span_id) -def trace_flags_from_int(trace_flags: int) -> str: - """Formats trace flags as 8-bit field""" - return "{:02x}".format(trace_flags) - def traceparent_from_context(span_context: Context) -> str: """Generates a liboboe W3C compatible trace_context from provided OTel span context.""" xtr = "00-{0:032X}-{1:016X}-{2:02X}".format(span_context.trace_id, From 761e34f63b4d81ccfea5ebf4901b6eb564791330 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 17 Mar 2022 15:50:27 -0700 Subject: [PATCH 20/71] Rm ununsed variables --- opentelemetry_distro_solarwinds/propagator.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 09ddc5ce..8f0feff7 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -1,5 +1,4 @@ import logging -import re import typing from opentelemetry import trace @@ -12,17 +11,11 @@ logger = logging.getLogger(__file__) class SolarWindsPropagator(textmap.TextMapPropagator): - """Extracts and injects SolarWinds tracestate header - - See also https://www.w3.org/TR/trace-context-1/ + """Extracts and injects SolarWinds tracestate header. + Must be used in composite with TraceContextTextMapPropagator. """ _TRACEPARENT_HEADER_NAME = "traceparent" _TRACESTATE_HEADER_NAME = "tracestate" - _TRACEPARENT_HEADER_FORMAT = ( - "^[ \t]*([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})" - + "(-.*)?[ \t]*$" - ) - _TRACEPARENT_HEADER_FORMAT_RE = re.compile(_TRACEPARENT_HEADER_FORMAT) def extract( self, @@ -30,9 +23,7 @@ def extract( context: typing.Optional[Context] = None, getter: textmap.Getter = textmap.default_getter, ) -> Context: - """Extracts sw tracestate from carrier into SpanContext - - Must be used in composite with TraceContextTextMapPropagator""" + """Extracts sw tracestate from carrier into SpanContext""" return context def inject( @@ -41,9 +32,7 @@ def inject( context: typing.Optional[Context] = None, setter: textmap.Setter = textmap.default_setter, ) -> None: - """Injects sw tracestate from SpanContext into carrier for HTTP request. - - Must be used in composite with TraceContextTextMapPropagator""" + """Injects sw tracestate from SpanContext into carrier for HTTP request""" span = trace.get_current_span(context) span_context = span.get_span_context() trace_state = span_context.trace_state From 07ec508b3596553ca218011537ccc5391d621afd Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 17 Mar 2022 15:50:47 -0700 Subject: [PATCH 21/71] Fix trace_flags and some log formatting --- opentelemetry_distro_solarwinds/sampler.py | 15 ++++++++------- .../w3c_transformer.py | 6 +++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index d47ecc43..ae5aa2d4 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -17,6 +17,7 @@ from opentelemetry_distro_solarwinds.extension.oboe import Context from opentelemetry_distro_solarwinds.w3c_transformer import ( span_id_from_int, + trace_flags_from_int, traceparent_from_context, sw_from_context, sw_from_span_and_decision @@ -83,8 +84,10 @@ def continue_liboboe_decision( do_sample = sw_value.split("-")[1] # TODO how do metrics work in OTel do_metrics = None - logger.debug("Continuing decision as do_metrics: {0}, do_sample: \ - {1}".format(do_metrics, do_sample)) + logger.debug("Continuing decision as do_metrics: {0}, do_sample: {1}".format( + do_metrics, + do_sample + )) return _LiboboeDecision(do_metrics, do_sample) def calculate_liboboe_decision( @@ -129,7 +132,7 @@ def create_new_trace_state( "sw", sw_from_span_and_decision( parent_span_context.span_id, - decision.do_sample + trace_flags_from_int(decision.do_sample) ) )]) logger.debug(f"Created new trace_state: {trace_state}") @@ -175,14 +178,12 @@ def calculate_attributes( # Don't set attributes if not tracing if self.otel_decision_from_liboboe(decision) == Decision.DROP: - logger.debug(f"Trace decision is to drop - not setting \ - attributes") + logger.debug(f"Trace decision is to drop - not setting attributes") return None # Trace's root span has no valid traceparent nor tracestate # so we don't set additional attributes if not parent_span_context.is_valid or not trace_state: - logger.debug(f"No valid traceparent or no tracestate - not \ - setting attributes") + logger.debug(f"No valid traceparent or no tracestate - not setting attributes") return None # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate diff --git a/opentelemetry_distro_solarwinds/w3c_transformer.py b/opentelemetry_distro_solarwinds/w3c_transformer.py index a8401e2b..5dc03cb2 100644 --- a/opentelemetry_distro_solarwinds/w3c_transformer.py +++ b/opentelemetry_distro_solarwinds/w3c_transformer.py @@ -10,12 +10,16 @@ def span_id_from_int(span_id: int) -> str: """Formats span ID as 16-byte hexadecimal string""" return "{:016x}".format(span_id) +def trace_flags_from_int(trace_flags: int) -> str: + """Formats trace flags as 8-bit field""" + return "{:02x}".format(trace_flags) + def traceparent_from_context(span_context: Context) -> str: """Generates a liboboe W3C compatible trace_context from provided OTel span context.""" xtr = "00-{0:032X}-{1:016X}-{2:02X}".format(span_context.trace_id, span_context.span_id, span_context.trace_flags) - logger.debug("Generated traceparent %s from SpanContext %s", xtr, + logger.debug("Generated traceparent %s from %s", xtr, span_context) return xtr From 77dbfb6c0895727a21f3ad8a80a7e846e3d9f39b Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 17 Mar 2022 16:19:34 -0700 Subject: [PATCH 22/71] Rm f-strings for older Python versions --- opentelemetry_distro_solarwinds/propagator.py | 6 +++--- opentelemetry_distro_solarwinds/sampler.py | 21 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 8f0feff7..eabb98b1 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -43,15 +43,15 @@ def inject( # Check if trace_state already contains sw KV if "sw" in trace_state.keys(): # If so, modify current span_id and trace_flags, and move to beginning of list - logger.debug(f"Updating trace state with {sw_value}") + logger.debug("Updating trace state with {0}".format(sw_value)) trace_state = trace_state.update("sw", sw_value) else: # If not, add sw KV to beginning of list - logger.debug(f"Adding KV to trace state with {sw_value}") + logger.debug("Adding KV to trace state with {0}".format(sw_value)) trace_state.add("sw", sw_value) else: - logger.debug(f"Creating new trace state with {sw_value}") + logger.debug("Creating new trace state with {0}".format(sw_value)) trace_state = TraceState([("sw", sw_value)]) setter.set( diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index ae5aa2d4..0c2ef33e 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -54,7 +54,10 @@ def create_new_liboboe_decision( """Creates new liboboe decision using parent span context.""" in_xtrace = traceparent_from_context(parent_span_context) tracestate = sw_from_context(parent_span_context) - logger.debug(f"Making oboe decision with in_xtrace {in_xtrace}, tracestate {tracestate}") + logger.debug("Making oboe decision with in_xtrace {0}, tracestate {1}".format( + in_xtrace, + tracestate + )) do_metrics, do_sample, \ _, _, _, _, _, _, _, _, _ = Context.getDecisions( in_xtrace, @@ -119,7 +122,7 @@ def otel_decision_from_liboboe( decision = Decision.RECORD_ONLY if liboboe_decision.do_sample: decision = Decision.RECORD_AND_SAMPLE - logger.debug(f"OTel decision created: {decision}") + logger.debug("OTel decision created: {0}".format(decision)) return decision def create_new_trace_state( @@ -135,7 +138,7 @@ def create_new_trace_state( trace_flags_from_int(decision.do_sample) ) )]) - logger.debug(f"Created new trace_state: {trace_state}") + logger.debug("Created new trace_state: {0}".format(trace_state)) return trace_state def calculate_trace_state( @@ -162,7 +165,7 @@ def calculate_trace_state( decision.do_sample ) ) - logger.debug(f"Updated trace_state: {trace_state}") + logger.debug("Updated trace_state: {0}".format(trace_state)) return trace_state def calculate_attributes( @@ -174,16 +177,16 @@ def calculate_attributes( ) -> Attributes or None: """Calculates Attributes or None based on trace decision, trace state, parent span context, and existing attributes.""" - logger.debug(f"Received attributes: {attributes}") + logger.debug("Received attributes: {0}".format(attributes)) # Don't set attributes if not tracing if self.otel_decision_from_liboboe(decision) == Decision.DROP: - logger.debug(f"Trace decision is to drop - not setting attributes") + logger.debug("Trace decision is to drop - not setting attributes") return None # Trace's root span has no valid traceparent nor tracestate # so we don't set additional attributes if not parent_span_context.is_valid or not trace_state: - logger.debug(f"No valid traceparent or no tracestate - not setting attributes") + logger.debug("No valid traceparent or no tracestate - not setting attributes") return None # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate @@ -194,7 +197,7 @@ def calculate_attributes( ), "sw.w3c.tracestate": trace_state.to_header() } - logger.debug(f"Created new attributes: {attributes}") + logger.debug("Created new attributes: {0}".format(attributes)) else: # Copy existing MappingProxyType KV into new_attributes for modification # attributes may have other vendor info etc @@ -225,7 +228,7 @@ def calculate_attributes( ) attributes = new_attributes - logger.debug(f"Set updated attributes: {attributes}") + logger.debug("Set updated attributes: {0}".format(attributes)) # attributes must be immutable for SamplingResult return MappingProxyType(attributes) From c4ca0556097b0aad09181606395f61bdd60888c4 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 17 Mar 2022 16:35:06 -0700 Subject: [PATCH 23/71] Mv tracestate sw value format checks to _SwSampler.calculate_liboboe_decision --- opentelemetry_distro_solarwinds/sampler.py | 60 +++++++++------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 0c2ef33e..2ba87d52 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -65,34 +65,6 @@ def create_new_liboboe_decision( ) return _LiboboeDecision(do_metrics, do_sample) - def continue_liboboe_decision( - self, - parent_span_context: SpanContext - ) -> _LiboboeDecision: - """Creates liboboe decision to continue parent trace state decision""" - trace_state = parent_span_context.trace_state - try: - sw_value = trace_state.get("sw") - except AttributeError: - logger.warning("Cannot continue decision if sw not in \ - tracestate ({0}). Making new decision.".format(trace_state)) - return self.create_new_liboboe_decision(parent_span_context) - if not re.match(SW_FORMAT, sw_value): - logger.warning("Cannot continue decision if tracestate sw not \ - in format <16_byte_span_id>-<8_bit_trace_flags>, nor if \ - trace_flags is anything but 01 or 00 ({0}). Making \ - new decision.".format(trace_state)) - return self.create_new_liboboe_decision(parent_span_context) - - do_sample = sw_value.split("-")[1] - # TODO how do metrics work in OTel - do_metrics = None - logger.debug("Continuing decision as do_metrics: {0}, do_sample: {1}".format( - do_metrics, - do_sample - )) - return _LiboboeDecision(do_metrics, do_sample) - def calculate_liboboe_decision( self, parent_span_context: SpanContext @@ -100,16 +72,34 @@ def calculate_liboboe_decision( """Calculates oboe trace decision based on parent span context.""" # No (valid) parent i.e. root span if not parent_span_context or not parent_span_context.is_valid: - decision = self.create_new_liboboe_decision(parent_span_context) + return self.create_new_liboboe_decision(parent_span_context) else: - # tracestate nonexistent/non-parsable, or no sw KV trace_state = parent_span_context.trace_state - if not trace_state or not trace_state.get("sw", None): - decision = self.create_new_liboboe_decision(parent_span_context) - # tracestate has sw KV + # tracestate nonexistent + if not trace_state: + return self.create_new_liboboe_decision(parent_span_context) + # tracestate does not have sw KV + elif not trace_state.get("sw", None): + return self.create_new_liboboe_decision else: - decision = self.continue_liboboe_decision(parent_span_context) - return decision + sw_value = trace_state.get("sw") + # tracestate has invalid sw value + if not re.match(SW_FORMAT, sw_value): + logger.warning("Cannot continue decision if tracestate sw not \ + in format <16_byte_span_id>-<8_bit_trace_flags>, nor if \ + trace_flags is anything but 01 or 00 ({0}). Making \ + new decision.".format(trace_state)) + return self.create_new_liboboe_decision(parent_span_context) + # tracestate has valid sw value + else: + do_sample = sw_value.split("-")[1] + # TODO how do metrics work in OTel + do_metrics = None + logger.debug("Continuing decision as do_metrics: {0}, do_sample: {1}".format( + do_metrics, + do_sample + )) + return _LiboboeDecision(do_metrics, do_sample) def otel_decision_from_liboboe( self, From 30899bfd36df04e8ae55c6b3f8365d841a93b8ac Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 17 Mar 2022 16:47:01 -0700 Subject: [PATCH 24/71] Misc fixups for style and comments --- opentelemetry_distro_solarwinds/exporter.py | 4 ++-- opentelemetry_distro_solarwinds/propagator.py | 2 +- opentelemetry_distro_solarwinds/sampler.py | 4 ++-- opentelemetry_distro_solarwinds/w3c_transformer.py | 11 +++++------ setup.py | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/opentelemetry_distro_solarwinds/exporter.py b/opentelemetry_distro_solarwinds/exporter.py index 19c81022..d17944d7 100644 --- a/opentelemetry_distro_solarwinds/exporter.py +++ b/opentelemetry_distro_solarwinds/exporter.py @@ -36,7 +36,7 @@ def export(self, spans): md = self._build_metadata(span.get_span_context()) if span.parent and span.parent.is_valid: # If there is a parent, we need to add an edge to this parent to this entry event - logger.debug("Continue trace from %s", md.toString()) + logger.debug("Continue trace from {0}".format(md.toString())) parent_md = self._build_metadata(span.parent) evt = Context.startTrace(md, int(span.start_time / 1000), parent_md) @@ -44,7 +44,7 @@ def export(self, spans): # In OpenTelemrtry, there are no events with individual IDs, but only a span ID # and trace ID. Thus, the entry event needs to be generated such that it has the # same op ID as the span ID of the OTel span. - logger.debug("Start a new trace %s", md.toString()) + logger.debug("Start a new trace {0}".format(md.toString())) evt = Context.startTrace(md, int(span.start_time / 1000)) evt.addInfo('Layer', span.name) evt.addInfo('Language', 'Python') diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index eabb98b1..5c0629fa 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -23,7 +23,7 @@ def extract( context: typing.Optional[Context] = None, getter: textmap.Getter = textmap.default_getter, ) -> Context: - """Extracts sw tracestate from carrier into SpanContext""" + """Extracts sw tracestate from carrier into OTel Context""" return context def inject( diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 2ba87d52..a842dc1f 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -54,7 +54,7 @@ def create_new_liboboe_decision( """Creates new liboboe decision using parent span context.""" in_xtrace = traceparent_from_context(parent_span_context) tracestate = sw_from_context(parent_span_context) - logger.debug("Making oboe decision with in_xtrace {0}, tracestate {1}".format( + logger.debug("Creating new oboe decision with in_xtrace {0}, tracestate {1}".format( in_xtrace, tracestate )) @@ -104,7 +104,7 @@ def calculate_liboboe_decision( def otel_decision_from_liboboe( self, liboboe_decision: _LiboboeDecision - ) -> None: + ) -> Decision: """Formats OTel decision from liboboe decision""" decision = Decision.DROP if liboboe_decision.do_metrics: diff --git a/opentelemetry_distro_solarwinds/w3c_transformer.py b/opentelemetry_distro_solarwinds/w3c_transformer.py index 5dc03cb2..bdc3601f 100644 --- a/opentelemetry_distro_solarwinds/w3c_transformer.py +++ b/opentelemetry_distro_solarwinds/w3c_transformer.py @@ -19,15 +19,14 @@ def traceparent_from_context(span_context: Context) -> str: xtr = "00-{0:032X}-{1:016X}-{2:02X}".format(span_context.trace_id, span_context.span_id, span_context.trace_flags) - logger.debug("Generated traceparent %s from %s", xtr, - span_context) + logger.debug("Generated traceparent {0} from {1}".format(xtr, span_context)) return xtr def sw_from_context(span_context: Context) -> str: - """Formats tracestate sw value from SpanContext - as 16-byte span_id with 8-bit trace_flags. + """Formats tracestate sw value from SpanContext as 16-byte span_id + with 8-bit trace_flags. - Example: 1122334455667788-01""" + Example: 1a2b3c4d5e6f7g8h-01""" return "{0:016x}-{1:02x}".format(span_context.span_id, span_context.trace_flags) @@ -35,5 +34,5 @@ def sw_from_span_and_decision(span_id: int, decision: str) -> str: """Formats tracestate sw value from span_id and liboboe decision as 16-byte span_id with 8-bit trace_flags. - Example: 1122334455667788-01""" + Example: 1a2b3c4d5e6f7g8h-01""" return "{0:016x}-{1}".format(span_id, decision) diff --git a/setup.py b/setup.py index b8b3ceb5..884945e9 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def link_oboe_lib(src_lib): if os.path.exists(dst): # if the destination library files exist already, they need to be deleted, otherwise linking will fail os.remove(dst) - log.info("Removed %s" % dst) + log.info("Removed {0}".format(dst)) os.symlink(src_lib, dst) log.info("Created new link at {} to {}".format(dst, src_lib)) except Exception as ecp: From 7e4e8c91739a7902b6a0cf109e02c92359fa9899 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 18 Mar 2022 14:57:08 -0700 Subject: [PATCH 25/71] Sampler lets liboboe do all the decision-making --- opentelemetry_distro_solarwinds/sampler.py | 67 +++++----------------- 1 file changed, 15 insertions(+), 52 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index a842dc1f..20b497f2 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -4,7 +4,6 @@ """ import logging -import re from types import MappingProxyType from typing import Optional, Sequence @@ -23,12 +22,11 @@ sw_from_span_and_decision ) -SW_FORMAT = "^\w{16}-0[0|1]$" logger = logging.getLogger(__name__) class _LiboboeDecision(): - """Represents a liboboe decision""" + """Convenience representation of a liboboe decision""" def __init__( self, do_metrics: str, @@ -40,18 +38,16 @@ def __init__( class _SwSampler(Sampler): """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" - def __init__(self, is_root_span: bool = False): - # TODO use or remove - self._is_root_span = is_root_span def get_description(self) -> str: return "SolarWinds custom opentelemetry sampler" - def create_new_liboboe_decision( + def calculate_liboboe_decision( self, parent_span_context: SpanContext ) -> _LiboboeDecision: - """Creates new liboboe decision using parent span context.""" + """Calculates oboe trace decision based on parent span context, for + non-existent or remote parent spans only.""" in_xtrace = traceparent_from_context(parent_span_context) tracestate = sw_from_context(parent_span_context) logger.debug("Creating new oboe decision with in_xtrace {0}, tracestate {1}".format( @@ -65,42 +61,6 @@ def create_new_liboboe_decision( ) return _LiboboeDecision(do_metrics, do_sample) - def calculate_liboboe_decision( - self, - parent_span_context: SpanContext - ) -> _LiboboeDecision: - """Calculates oboe trace decision based on parent span context.""" - # No (valid) parent i.e. root span - if not parent_span_context or not parent_span_context.is_valid: - return self.create_new_liboboe_decision(parent_span_context) - else: - trace_state = parent_span_context.trace_state - # tracestate nonexistent - if not trace_state: - return self.create_new_liboboe_decision(parent_span_context) - # tracestate does not have sw KV - elif not trace_state.get("sw", None): - return self.create_new_liboboe_decision - else: - sw_value = trace_state.get("sw") - # tracestate has invalid sw value - if not re.match(SW_FORMAT, sw_value): - logger.warning("Cannot continue decision if tracestate sw not \ - in format <16_byte_span_id>-<8_bit_trace_flags>, nor if \ - trace_flags is anything but 01 or 00 ({0}). Making \ - new decision.".format(trace_state)) - return self.create_new_liboboe_decision(parent_span_context) - # tracestate has valid sw value - else: - do_sample = sw_value.split("-")[1] - # TODO how do metrics work in OTel - do_metrics = None - logger.debug("Continuing decision as do_metrics: {0}, do_sample: {1}".format( - do_metrics, - do_sample - )) - return _LiboboeDecision(do_metrics, do_sample) - def otel_decision_from_liboboe( self, liboboe_decision: _LiboboeDecision @@ -136,8 +96,9 @@ def calculate_trace_state( decision: _LiboboeDecision, parent_span_context: SpanContext ) -> TraceState: - """Calculates trace_state based on parent span context and trace decision""" - # No valid parent i.e. root span + """Calculates trace_state based on parent span context and trace decision, + for non-existent or remote parent spans only.""" + # No valid parent i.e. root span, or parent is remote if not parent_span_context.is_valid: trace_state = self.create_new_trace_state(decision, parent_span_context) else: @@ -166,7 +127,9 @@ def calculate_attributes( parent_span_context: SpanContext ) -> Attributes or None: """Calculates Attributes or None based on trace decision, trace state, - parent span context, and existing attributes.""" + parent span context, and existing attributes. + + For non-existent or remote parent spans only.""" logger.debug("Received attributes: {0}".format(attributes)) # Don't set attributes if not tracing @@ -243,7 +206,7 @@ def should_sample( decision = self.calculate_liboboe_decision(parent_span_context) - # TODO Set differently if not decision.RECORD_AND_SAMPLE + # Always calculate trace_state for propagation trace_state = self.calculate_trace_state( decision, parent_span_context @@ -271,13 +234,13 @@ class ParentBasedSwSampler(ParentBased): def __init__(self): """ Uses _SwSampler/liboboe if no parent span. - Uses _SwSampler/liboboe if parent span is_remote and sampled. - Uses OTEL default if parent span is_remote and NOT sampled (never sample). + Uses _SwSampler/liboboe if parent span is_remote. Uses OTEL defaults if parent span is_local. """ super().__init__( - root=_SwSampler(is_root_span=True), - remote_parent_sampled=_SwSampler() + root=_SwSampler(), + remote_parent_sampled=_SwSampler(), + remote_parent_not_sampled=_SwSampler() ) def should_sample( From d9c0e6345c13a800a2351856f182e6f1f3636dec Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 21 Mar 2022 15:59:18 -0700 Subject: [PATCH 26/71] Fix flag format at trace_state and attribute update --- opentelemetry_distro_solarwinds/sampler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 20b497f2..069964e5 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -113,7 +113,7 @@ def calculate_trace_state( "sw", sw_from_span_and_decision( parent_span_context.span_id, - decision.do_sample + trace_flags_from_int(decision.do_sample) ) ) logger.debug("Updated trace_state: {0}".format(trace_state)) @@ -170,7 +170,7 @@ def calculate_attributes( "sw", sw_from_span_and_decision( parent_span_context.span_id, - decision.do_sample + trace_flags_from_int(decision.do_sample) ) ) new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() From 520453660970a918fab3374e54cbf1c2e098a5e2 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 21 Mar 2022 20:05:38 -0700 Subject: [PATCH 27/71] Update Propagator debug logs for clarity --- opentelemetry_distro_solarwinds/propagator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 5c0629fa..bb5367ef 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -43,15 +43,15 @@ def inject( # Check if trace_state already contains sw KV if "sw" in trace_state.keys(): # If so, modify current span_id and trace_flags, and move to beginning of list - logger.debug("Updating trace state with {0}".format(sw_value)) + logger.debug("Updating trace state for injection with {0}".format(sw_value)) trace_state = trace_state.update("sw", sw_value) else: # If not, add sw KV to beginning of list - logger.debug("Adding KV to trace state with {0}".format(sw_value)) + logger.debug("Adding KV to trace state for injection with {0}".format(sw_value)) trace_state.add("sw", sw_value) else: - logger.debug("Creating new trace state with {0}".format(sw_value)) + logger.debug("Creating new trace state for injection with {0}".format(sw_value)) trace_state = TraceState([("sw", sw_value)]) setter.set( From 12d4cbd0a6535c9a796e7d3d3c92fd09cfb08e2c Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 21 Mar 2022 20:50:39 -0700 Subject: [PATCH 28/71] Fix sw.tracestate_parent_id assignment --- opentelemetry_distro_solarwinds/sampler.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 069964e5..32841fa6 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -145,11 +145,13 @@ def calculate_attributes( # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate if not attributes: attributes = { - "sw.tracestate_parent_id": span_id_from_int( - parent_span_context.span_id - ), "sw.w3c.tracestate": trace_state.to_header() } + # Only set sw.tracestate_parent_id on the entry (root) span for this service + sw_value = parent_span_context.trace_state.get("sw", None) + if sw_value: + attributes["sw.tracestate_parent_id"] = sw_value + logger.debug("Created new attributes: {0}".format(attributes)) else: # Copy existing MappingProxyType KV into new_attributes for modification @@ -175,11 +177,6 @@ def calculate_attributes( ) new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() - # Only set sw.tracestate_parent_id on the entry (root) span for this service - new_attributes["sw.tracestate_parent_id"] = span_id_from_int( - parent_span_context.span_id - ) - attributes = new_attributes logger.debug("Set updated attributes: {0}".format(attributes)) From bd315a60124659d6c29b322f70c8b2f75a1565eb Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 21 Mar 2022 21:20:47 -0700 Subject: [PATCH 29/71] Preserve non-vendor values of injected tracestate header --- opentelemetry_distro_solarwinds/sampler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 32841fa6..7f3cc5bc 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -102,11 +102,10 @@ def calculate_trace_state( if not parent_span_context.is_valid: trace_state = self.create_new_trace_state(decision, parent_span_context) else: - # tracestate nonexistent/non-parsable, or no sw KV trace_state = parent_span_context.trace_state - if not trace_state or not trace_state.get("sw", None): + if not trace_state: + # tracestate nonexistent/non-parsable trace_state = self.create_new_trace_state(decision, parent_span_context) - # tracestate has sw KV else: # Update trace_state with span_id and sw trace_flags trace_state = trace_state.update( From 8f5a21571aff4c9eaf87f59847b43932b14d0f0c Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Tue, 22 Mar 2022 17:22:54 -0700 Subject: [PATCH 30/71] Add TraceOptions class and an init with headers from propagator.extract --- opentelemetry_distro_solarwinds/propagator.py | 31 ++++++- opentelemetry_distro_solarwinds/sampler.py | 4 +- .../traceoptions.py | 91 +++++++++++++++++++ 3 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 opentelemetry_distro_solarwinds/traceoptions.py diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index bb5367ef..96eb34f5 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -6,6 +6,7 @@ from opentelemetry.propagators import textmap from opentelemetry.trace.span import TraceState +from opentelemetry_distro_solarwinds.traceoptions import TraceOptions from opentelemetry_distro_solarwinds.w3c_transformer import sw_from_context logger = logging.getLogger(__file__) @@ -14,8 +15,8 @@ class SolarWindsPropagator(textmap.TextMapPropagator): """Extracts and injects SolarWinds tracestate header. Must be used in composite with TraceContextTextMapPropagator. """ - _TRACEPARENT_HEADER_NAME = "traceparent" _TRACESTATE_HEADER_NAME = "tracestate" + _TRACEOPTIONS_HEADER_NAME = "x-trace-options" def extract( self, @@ -23,7 +24,25 @@ def extract( context: typing.Optional[Context] = None, getter: textmap.Getter = textmap.default_getter, ) -> Context: - """Extracts sw tracestate from carrier into OTel Context""" + """Extracts sw trace options from carrier into OTel Context""" + if context is None: + context = Context() + + traceoptions_header = getter.get( + carrier, + self._TRACEOPTIONS_HEADER_NAME + ) + if not traceoptions_header: + return context + logger.debug("Extracted traceoptions_header: {0}".format(traceoptions_header[0])) + + traceoptions = TraceOptions(traceoptions_header[0], context) + logger.debug("Created TraceOptions: {0}".format(traceoptions.__dict__)) + + context.update({ + # TODO assign individual KVs for each traceoptions + self._TRACEOPTIONS_HEADER_NAME: traceoptions_header[0] + }) return context def inject( @@ -32,7 +51,7 @@ def inject( context: typing.Optional[Context] = None, setter: textmap.Setter = textmap.default_setter, ) -> None: - """Injects sw tracestate from SpanContext into carrier for HTTP request""" + """Injects sw tracestate and trace options from SpanContext into carrier for HTTP request""" span = trace.get_current_span(context) span_context = span.get_span_context() trace_state = span_context.trace_state @@ -58,10 +77,12 @@ def inject( carrier, self._TRACESTATE_HEADER_NAME, trace_state.to_header() ) + # TODO: Prepare carrier with x-trace-options + @property def fields(self) -> typing.Set[str]: """Returns a set with the fields set in `inject`""" return { - self._TRACEPARENT_HEADER_NAME, - self._TRACESTATE_HEADER_NAME + self._TRACESTATE_HEADER_NAME, + self._TRACEOPTIONS_HEADER_NAME } diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 7f3cc5bc..741113b9 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -15,7 +15,6 @@ from opentelemetry_distro_solarwinds.extension.oboe import Context from opentelemetry_distro_solarwinds.w3c_transformer import ( - span_id_from_int, trace_flags_from_int, traceparent_from_context, sw_from_context, @@ -200,6 +199,9 @@ def should_sample( parent_context ).get_span_context() + # TODO: get individual x-trace-options from parent_context + logger.debug("parent_context is {0}".format(parent_context)) + decision = self.calculate_liboboe_decision(parent_span_context) # Always calculate trace_state for propagation diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py new file mode 100644 index 00000000..c76196b1 --- /dev/null +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -0,0 +1,91 @@ +import logging +import re +import typing + +from opentelemetry.context.context import Context + +logger = logging.getLogger(__file__) + +class TraceOptions(): + """Formats X-Trace-Options for trigger tracing""" + + _TRACEOPTIONS_CUSTOM = ("^custom-[^\s]*$") + _TRACEOPTIONS_CUSTOM_RE = re.compile(_TRACEOPTIONS_CUSTOM) + + def __init__(self, + options: str, + context: typing.Optional[Context] = None + ): + """ + Args: + options: A string of x-trace-options + context: OTel context that may contain x-trace-options + + Examples of options: + "trigger-trace" + "trigger-trace;sw-keys=check-id:check-1013,website-id:booking-demo" + "trigger-trace;custom-key1=value1" + """ + self.custom_kvs = {} + self.sw_keys = {} + self.trigger_trace = False + self.ts = 0 + self.ignored = [] + + # TODO: What if OTel context already has traceoptions + + # each of options delimited by semicolon + traceoptions = re.split(r";+", options) + for option in traceoptions: + # KVs (e.g. sw-keys or custom-key1) are assigned by equals + option_kv = option.split("=", 2) + if not option_kv[0]: + continue + + option_key = option_kv[0].strip() + if option_key == "trigger-trace": + if len(option_kv) > 1: + logger.warning("trigger-trace must be standalone flag. Ignoring.") + self.ignored.append("trigger-trace") + else: + self.trigger_trace = True + + elif option_key == "sw-keys": + # each of sw-keys KVs delimited by comma + sw_kvs = re.split(r",+", option_kv[1]) + for assignment in sw_kvs: + # each of sw-keys values assigned by colon + sw_kv = assignment.split(":", 2) + if not sw_kv[0]: + logger.warning( + "Could not parse sw-key assignment {0}. Ignoring.".format( + assignment + )) + self.ignore.append(assignment) + continue + self.sw_keys.update({ + sw_kv[0]: sw_kv[1] + }) + + elif re.match(self._TRACEOPTIONS_CUSTOM_RE, option_key): + self.custom_kvs[option_key] = option_kv[1].strip() + + elif option_key == "ts": + try: + self.ts = int(option_kv[1]) + except ValueError as e: + logger.warning("ts must be base 10 int. Ignoring.") + self.ignore.append("ts") + + else: + logger.warning( + "{0} is not a recognized trace option. Ignoring".format( + option_key + )) + self.ignored.append(option_key) + + if self.ignored: + logger.warning( + "Some x-trace-options were ignored: {0}".format( + ", ".join(self.ignored) + )) From 60522b7153f21c81345c7b9129416e48478a599d Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 23 Mar 2022 15:56:59 -0700 Subject: [PATCH 31/71] Mv TraceOptions to XTraceOptions; add from_context and iter,str overloads --- opentelemetry_distro_solarwinds/propagator.py | 24 +++---- .../traceoptions.py | 67 +++++++++++++++++-- 2 files changed, 70 insertions(+), 21 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 96eb34f5..e1606453 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -6,7 +6,7 @@ from opentelemetry.propagators import textmap from opentelemetry.trace.span import TraceState -from opentelemetry_distro_solarwinds.traceoptions import TraceOptions +from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions from opentelemetry_distro_solarwinds.w3c_transformer import sw_from_context logger = logging.getLogger(__file__) @@ -16,7 +16,7 @@ class SolarWindsPropagator(textmap.TextMapPropagator): Must be used in composite with TraceContextTextMapPropagator. """ _TRACESTATE_HEADER_NAME = "tracestate" - _TRACEOPTIONS_HEADER_NAME = "x-trace-options" + _XTRACEOPTIONS_HEADER_NAME = "x-trace-options" def extract( self, @@ -28,21 +28,15 @@ def extract( if context is None: context = Context() - traceoptions_header = getter.get( + xtraceoptions_header = getter.get( carrier, - self._TRACEOPTIONS_HEADER_NAME + self._XTRACEOPTIONS_HEADER_NAME ) - if not traceoptions_header: + if not xtraceoptions_header: return context - logger.debug("Extracted traceoptions_header: {0}".format(traceoptions_header[0])) - - traceoptions = TraceOptions(traceoptions_header[0], context) - logger.debug("Created TraceOptions: {0}".format(traceoptions.__dict__)) - - context.update({ - # TODO assign individual KVs for each traceoptions - self._TRACEOPTIONS_HEADER_NAME: traceoptions_header[0] - }) + logger.debug("Extracted xtraceoptions_header: {0}".format(xtraceoptions_header[0])) + xtraceoptions = XTraceOptions(xtraceoptions_header[0], context) + context.update(dict(xtraceoptions)) return context def inject( @@ -84,5 +78,5 @@ def fields(self) -> typing.Set[str]: """Returns a set with the fields set in `inject`""" return { self._TRACESTATE_HEADER_NAME, - self._TRACEOPTIONS_HEADER_NAME + self._XTRACEOPTIONS_HEADER_NAME } diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index c76196b1..1bc30540 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__file__) -class TraceOptions(): +class XTraceOptions(): """Formats X-Trace-Options for trigger tracing""" _TRACEOPTIONS_CUSTOM = ("^custom-[^\s]*$") @@ -32,7 +32,9 @@ def __init__(self, self.ts = 0 self.ignored = [] - # TODO: What if OTel context already has traceoptions + if context: + self.from_context(context) + return # each of options delimited by semicolon traceoptions = re.split(r";+", options) @@ -62,10 +64,8 @@ def __init__(self, assignment )) self.ignore.append(assignment) - continue - self.sw_keys.update({ - sw_kv[0]: sw_kv[1] - }) + else: + self.sw_keys.update({sw_kv[0]: sw_kv[1]}) elif re.match(self._TRACEOPTIONS_CUSTOM_RE, option_key): self.custom_kvs[option_key] = option_kv[1].strip() @@ -89,3 +89,58 @@ def __init__(self, "Some x-trace-options were ignored: {0}".format( ", ".join(self.ignored) )) + + def __iter__(self) -> typing.Iterator: + """Iterable representation of XTraceOptions""" + yield from self.__dict__.items() + + def __str__(self) -> str: + """String representation of XTraceOptions""" + options_str = "" + + if self.trigger_trace: + options_str += "trigger-trace" + + if len(self.sw_keys) > 0: + if len(options_str) > 0: + options_str += ";" + options_str += "sw-keys=" + for i, (k, v) in enumerate(self.sw_keys.items()): + options_str += "{0}:{1}".format(k, v) + if i < len(self.sw_keys) - 1: + options_str += "," + + if len(self.custom_kvs) > 0: + if len(options_str) > 0: + options_str += ";" + for i, (k, v) in enumerate(self.custom_kvs.items()): + options_str += "{0}={1}".format(k, v) + if i < len(self.custom_kvs) - 1: + options_str += ";" + + if self.ts > 0: + if len(options_str) > 0: + options_str += ";" + options_str += "ts={0}".format(self.ts) + + return options_str + + def from_context( + self, + context: typing.Optional[Context] + ) -> None: + """ + Args: + context: OTel context that may contain x-trace-options + """ + if "trigger_trace" in context and context["trigger_trace"]: + self.trigger_trace = True + + if "sw_keys" in context and context["sw_keys"]: + self.sw_keys = context["sw_keys"] + + if "custom_kvs" in context and context["custom_kvs"]: + self.custom_kvs = context["custom_kvs"] + + if "ts" in context and context["ts"] > 0: + self.ts = context["ts"] From 3e175ba525ed1934a81696b9fdc39233c568a8a6 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 23 Mar 2022 17:14:48 -0700 Subject: [PATCH 32/71] Sampler passes xtraceoptions from otel context to liboboe --- opentelemetry_distro_solarwinds/sampler.py | 101 ++++++++++++++++++--- 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 741113b9..5e00aa99 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import Optional, Sequence +from opentelemetry.context.context import Context as OtelContext from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult) from opentelemetry.trace import Link, SpanKind, get_current_span @@ -14,6 +15,7 @@ from opentelemetry.util.types import Attributes from opentelemetry_distro_solarwinds.extension.oboe import Context +from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions from opentelemetry_distro_solarwinds.w3c_transformer import ( trace_flags_from_int, traceparent_from_context, @@ -31,6 +33,7 @@ def __init__( do_metrics: str, do_sample: str ): + # TODO all liboboe outputs self.do_metrics = do_metrics self.do_sample = do_sample @@ -43,21 +46,91 @@ def get_description(self) -> str: def calculate_liboboe_decision( self, - parent_span_context: SpanContext + parent_span_context: SpanContext, + parent_context: Optional[OtelContext] = None, ) -> _LiboboeDecision: """Calculates oboe trace decision based on parent span context, for non-existent or remote parent spans only.""" - in_xtrace = traceparent_from_context(parent_span_context) - tracestate = sw_from_context(parent_span_context) - logger.debug("Creating new oboe decision with in_xtrace {0}, tracestate {1}".format( - in_xtrace, - tracestate + tracestring = traceparent_from_context(parent_span_context) + sw_member_value = sw_from_context(parent_span_context) + + # TODO: config --> enable/disable tracing, sample_rate, tt mode + tracing_mode = 1 + sample_rate = 1000000 + trigger_tracing_mode_disabled = 0 + + xtraceoptions = XTraceOptions("", parent_context) + logger.debug("parent_context is {0}".format(parent_context)) + logger.debug("xtraceoptions is {0}".format(xtraceoptions)) + + if xtraceoptions.trigger_trace: + trigger_trace = 1 + else: + trigger_trace = 0 + timestamp = xtraceoptions.ts + signature = None + options = str(xtraceoptions) + + logger.debug( + "Creating new oboe decision with " + "tracestring: {0}, " + "sw_member_value: {1}, " + "tracing_mode: {2}, " + "sample_rate: {3}, " + "trigger_trace: {4}, " + "trigger_tracing_mode_disabled: {5}, " + "options: {6}, " + "signature: {7}, " + "timestamp: {8}".format( + tracestring, + sw_member_value, + tracing_mode, + sample_rate, + trigger_trace, + trigger_tracing_mode_disabled, + options, + signature, + timestamp )) do_metrics, do_sample, \ - _, _, _, _, _, _, _, _, _ = Context.getDecisions( - in_xtrace, - tracestate + rate, source, bucket_rate, bucket_cap, \ + type, auth, status_msg, auth_msg, status = Context.getDecisions( + tracestring, + sw_member_value, + tracing_mode, + sample_rate, + trigger_trace, + trigger_tracing_mode_disabled, + options, + signature, + timestamp ) + logger.debug( + "Got liboboe decision outputs " + "do_metrics: {0}, " + "do_sample: {1}, " + "rate: {2}, " + "source: {3}, " + "bucket_rate: {4}, " + "bucket_cap: {5}, " + "type: {6}, " + "auth: {7}, " + "status_msg: {8}, " + "auth_msg: {9}, " + "status: {10}".format( + do_metrics, + do_sample, + rate, + source, + bucket_rate, + bucket_cap, + type, + auth, + status_msg, + auth_msg, + status + ) + ) return _LiboboeDecision(do_metrics, do_sample) def otel_decision_from_liboboe( @@ -183,7 +256,7 @@ def calculate_attributes( def should_sample( self, - parent_context: Optional["Context"], + parent_context: Optional[OtelContext], trace_id: int, name: str, kind: SpanKind = None, @@ -199,10 +272,10 @@ def should_sample( parent_context ).get_span_context() - # TODO: get individual x-trace-options from parent_context - logger.debug("parent_context is {0}".format(parent_context)) - - decision = self.calculate_liboboe_decision(parent_span_context) + decision = self.calculate_liboboe_decision( + parent_span_context, + parent_context + ) # Always calculate trace_state for propagation trace_state = self.calculate_trace_state( From 38639c9d915edae8146bba55b799f17b08da4d9c Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 24 Mar 2022 13:46:06 -0700 Subject: [PATCH 33/71] Sampler passes liboboe tracestate from header tracestate, not traceparent --- opentelemetry_distro_solarwinds/sampler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 5e00aa99..80cacb13 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -19,7 +19,6 @@ from opentelemetry_distro_solarwinds.w3c_transformer import ( trace_flags_from_int, traceparent_from_context, - sw_from_context, sw_from_span_and_decision ) @@ -52,7 +51,7 @@ def calculate_liboboe_decision( """Calculates oboe trace decision based on parent span context, for non-existent or remote parent spans only.""" tracestring = traceparent_from_context(parent_span_context) - sw_member_value = sw_from_context(parent_span_context) + sw_member_value = parent_span_context.trace_state.get("sw") # TODO: config --> enable/disable tracing, sample_rate, tt mode tracing_mode = 1 From f95bf46d1d390b5aa4c7ed1435ed42341d0275b5 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 24 Mar 2022 14:53:38 -0700 Subject: [PATCH 34/71] Fix XTraceOptions init when other headers provided --- opentelemetry_distro_solarwinds/traceoptions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index 1bc30540..87fc4aa9 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -13,7 +13,7 @@ class XTraceOptions(): _TRACEOPTIONS_CUSTOM_RE = re.compile(_TRACEOPTIONS_CUSTOM) def __init__(self, - options: str, + options_header: str, context: typing.Optional[Context] = None ): """ @@ -32,12 +32,12 @@ def __init__(self, self.ts = 0 self.ignored = [] - if context: + if not options_header: self.from_context(context) return # each of options delimited by semicolon - traceoptions = re.split(r";+", options) + traceoptions = re.split(r";+", options_header) for option in traceoptions: # KVs (e.g. sw-keys or custom-key1) are assigned by equals option_kv = option.split("=", 2) @@ -133,6 +133,10 @@ def from_context( Args: context: OTel context that may contain x-trace-options """ + logger.debug("Setting XTraceOptions from_context with {0}".format(context)) + if not context: + return + if "trigger_trace" in context and context["trigger_trace"]: self.trigger_trace = True From af11112b2e42db7a9235b75658ed16c970cf77e6 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 28 Mar 2022 11:58:15 -0700 Subject: [PATCH 35/71] Add TT signature handling; Sampler gives liboboe None if parent not valid/remote; update LiboboeDecision attrs --- opentelemetry_distro_solarwinds/propagator.py | 23 ++++-- opentelemetry_distro_solarwinds/sampler.py | 72 ++++++++++++++----- 2 files changed, 71 insertions(+), 24 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index e1606453..22a40cc4 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -17,6 +17,7 @@ class SolarWindsPropagator(textmap.TextMapPropagator): """ _TRACESTATE_HEADER_NAME = "tracestate" _XTRACEOPTIONS_HEADER_NAME = "x-trace-options" + _XTRACEOPTIONS_SIGNATURE_HEADER_NAME = "x-trace-options-signature" def extract( self, @@ -24,7 +25,7 @@ def extract( context: typing.Optional[Context] = None, getter: textmap.Getter = textmap.default_getter, ) -> Context: - """Extracts sw trace options from carrier into OTel Context""" + """Extracts sw trace options and signature from carrier into OTel Context""" if context is None: context = Context() @@ -32,11 +33,21 @@ def extract( carrier, self._XTRACEOPTIONS_HEADER_NAME ) - if not xtraceoptions_header: - return context - logger.debug("Extracted xtraceoptions_header: {0}".format(xtraceoptions_header[0])) - xtraceoptions = XTraceOptions(xtraceoptions_header[0], context) - context.update(dict(xtraceoptions)) + if xtraceoptions_header: + logger.debug("Extracted xtraceoptions_header: {0}".format(xtraceoptions_header[0])) + xtraceoptions = XTraceOptions(xtraceoptions_header[0], context) + context.update(dict(xtraceoptions)) + + signature_header = getter.get( + carrier, + self._XTRACEOPTIONS_SIGNATURE_HEADER_NAME + ) + if signature_header: + logger.debug("Extracted signature_header: {0}".format(signature_header[0])) + context.update({ + self._XTRACEOPTIONS_SIGNATURE_HEADER_NAME: signature_header[0] + }) + return context def inject( diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 80cacb13..0e7c69f6 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -29,15 +29,35 @@ class _LiboboeDecision(): """Convenience representation of a liboboe decision""" def __init__( self, - do_metrics: str, - do_sample: str + do_metrics: int, + do_sample: int, + sample_rate: int, + sample_source: int, + bucket_rate: float, + bucket_cap: float, + decision_type: int, + auth: int, + status_msg: str, + auth_msg: str, + status: int, ): - # TODO all liboboe outputs self.do_metrics = do_metrics self.do_sample = do_sample + self.sample_rate = sample_rate + self.sample_source = sample_source + self.bucket_rate = bucket_rate + self.bucket_cap = bucket_cap + self.decision_type = decision_type + self.auth = auth + self.status_msg = status_msg + self.auth_msg = auth_msg + self.status = status class _SwSampler(Sampler): + + _XTRACEOPTIONS_SIGNATURE_HEADER_NAME = "x-trace-options-signature" + """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" def get_description(self) -> str: @@ -48,27 +68,31 @@ def calculate_liboboe_decision( parent_span_context: SpanContext, parent_context: Optional[OtelContext] = None, ) -> _LiboboeDecision: - """Calculates oboe trace decision based on parent span context, for - non-existent or remote parent spans only.""" - tracestring = traceparent_from_context(parent_span_context) + """Calculates oboe trace decision based on parent span context.""" + tracestring = None + if parent_span_context.is_valid and parent_span_context.is_remote: + tracestring = traceparent_from_context(parent_span_context) sw_member_value = parent_span_context.trace_state.get("sw") # TODO: config --> enable/disable tracing, sample_rate, tt mode tracing_mode = 1 - sample_rate = 1000000 + sample_rate = 1 trigger_tracing_mode_disabled = 0 - xtraceoptions = XTraceOptions("", parent_context) logger.debug("parent_context is {0}".format(parent_context)) + xtraceoptions = XTraceOptions("", parent_context) logger.debug("xtraceoptions is {0}".format(xtraceoptions)) - if xtraceoptions.trigger_trace: - trigger_trace = 1 - else: - trigger_trace = 0 - timestamp = xtraceoptions.ts + options = None + trigger_trace = 0 signature = None - options = str(xtraceoptions) + timestamp = None + if xtraceoptions: + options = str(xtraceoptions) + trigger_trace = xtraceoptions.trigger_trace + if parent_context: + signature = parent_context.get(self._XTRACEOPTIONS_SIGNATURE_HEADER_NAME, None) + timestamp = xtraceoptions.ts logger.debug( "Creating new oboe decision with " @@ -92,8 +116,8 @@ def calculate_liboboe_decision( timestamp )) do_metrics, do_sample, \ - rate, source, bucket_rate, bucket_cap, \ - type, auth, status_msg, auth_msg, status = Context.getDecisions( + rate, source, bucket_rate, bucket_cap, decision_type, \ + auth, status_msg, auth_msg, status = Context.getDecisions( tracestring, sw_member_value, tracing_mode, @@ -123,14 +147,26 @@ def calculate_liboboe_decision( source, bucket_rate, bucket_cap, - type, + decision_type, auth, status_msg, auth_msg, status ) ) - return _LiboboeDecision(do_metrics, do_sample) + return _LiboboeDecision( + do_metrics, + do_sample, + rate, + source, + bucket_rate, + bucket_cap, + decision_type, + auth, + status_msg, + auth_msg, + status + ) def otel_decision_from_liboboe( self, From 322949bcf9f2b8b5f4b75509e849a8e7e0e77de1 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 30 Mar 2022 14:07:23 -0700 Subject: [PATCH 36/71] Add SolarWindsResponsePropagator (wip); up otel-instrumentation version --- opentelemetry_distro_solarwinds/distro.py | 6 +- .../response_propagator.py | 59 +++++++++++++++++++ setup.cfg | 2 +- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 opentelemetry_distro_solarwinds/response_propagator.py diff --git a/opentelemetry_distro_solarwinds/distro.py b/opentelemetry_distro_solarwinds/distro.py index 640484a6..3e28bc7b 100644 --- a/opentelemetry_distro_solarwinds/distro.py +++ b/opentelemetry_distro_solarwinds/distro.py @@ -2,10 +2,12 @@ from opentelemetry import trace from opentelemetry.instrumentation.distro import BaseDistro +from opentelemetry.instrumentation.propagators import set_global_response_propagator from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry_distro_solarwinds.exporter import SolarWindsSpanExporter +from opentelemetry_distro_solarwinds.response_propagator import SolarWindsTraceResponsePropagator from opentelemetry_distro_solarwinds.sampler import ParentBasedSwSampler @@ -16,9 +18,11 @@ class SolarWindsDistro(BaseDistro): - no functionality added at this time """ def _configure(self, **kwargs): - # automatically make use of custom SolarWinds sampler + # Automatically make use of custom SolarWinds sampler trace.set_tracer_provider( TracerProvider(sampler=ParentBasedSwSampler())) # Automatically configure the SolarWinds Span exporter span_exporter = BatchSpanProcessor(SolarWindsSpanExporter()) trace.get_tracer_provider().add_span_processor(span_exporter) + # Set global HTTP response propagator + set_global_response_propagator(SolarWindsTraceResponsePropagator()) \ No newline at end of file diff --git a/opentelemetry_distro_solarwinds/response_propagator.py b/opentelemetry_distro_solarwinds/response_propagator.py new file mode 100644 index 00000000..557a2436 --- /dev/null +++ b/opentelemetry_distro_solarwinds/response_propagator.py @@ -0,0 +1,59 @@ +import logging +import typing + +from opentelemetry import trace +from opentelemetry.context.context import Context +from opentelemetry.instrumentation.propagators import ResponsePropagator +from opentelemetry.propagators import textmap + +from opentelemetry_distro_solarwinds.w3c_transformer import traceparent_from_context + +logger = logging.getLogger(__file__) + +class SolarWindsTraceResponsePropagator(ResponsePropagator): + """ + """ + _HTTP_HEADER_ACCESS_CONTROL_EXPOSE_HEADERS = "Access-Control-Expose-Headers" + _XTRACE_HEADER_NAME = "x-trace" + _XTRACEOPTIONS_RESPONSE_HEADER_NAME = "x-trace-options-response" + + def inject( + self, + carrier: textmap.CarrierT, + context: typing.Optional[Context] = None, + setter: textmap.Setter = textmap.default_setter, + ) -> None: + """Injects x-trace and options-response into the HTTP response carrier.""" + span = trace.get_current_span(context) + span_context = span.get_span_context() + if span_context == trace.INVALID_SPAN_CONTEXT: + return + + x_trace = traceparent_from_context(span_context) + + # TODO: Should be based on x-trace-options tt + sampled = 'trigger-trace=' + if span_context.trace_flags: + sampled += 'ok' + else: + sampled += 'rate-exceeded' + + exposed_headers = "{0},{1}".format( + self._XTRACE_HEADER_NAME, + self._XTRACEOPTIONS_RESPONSE_HEADER_NAME + ) + setter.set( + carrier, + self._XTRACE_HEADER_NAME, + x_trace, + ) + setter.set( + carrier, + self._XTRACEOPTIONS_RESPONSE_HEADER_NAME, + sampled, + ) + setter.set( + carrier, + self._HTTP_HEADER_ACCESS_CONTROL_EXPOSE_HEADERS, + exposed_headers, + ) \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index cd46efee..53e7625a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ python_requires = >=3.4 install_requires = opentelemetry-api >= 1.3.0 opentelemetry-sdk >= 1.3.0 - opentelemetry-instrumentation >= 0.22b0 + opentelemetry-instrumentation >= 0.29b0 [options.entry_points] opentelemetry_distro = From ff201c70c52c660489605c268ab67340b9f30d00 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 31 Mar 2022 17:11:59 -0700 Subject: [PATCH 37/71] Sampler creates xtraceoptions_response for tracestate piggyback; ResponsePropagator recovers response value from tracestate; XTraceOptions handles signature --- opentelemetry_distro_solarwinds/propagator.py | 37 +++++---- .../response_propagator.py | 50 +++++++---- opentelemetry_distro_solarwinds/sampler.py | 82 ++++++++++++++++--- .../traceoptions.py | 52 +++++++----- 4 files changed, 158 insertions(+), 63 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 22a40cc4..b30a6a35 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__file__) class SolarWindsPropagator(textmap.TextMapPropagator): - """Extracts and injects SolarWinds tracestate header. + """Extracts and injects SolarWinds headers for trace propagation. Must be used in composite with TraceContextTextMapPropagator. """ _TRACESTATE_HEADER_NAME = "tracestate" @@ -25,7 +25,9 @@ def extract( context: typing.Optional[Context] = None, getter: textmap.Getter = textmap.default_getter, ) -> Context: - """Extracts sw trace options and signature from carrier into OTel Context""" + """Extracts sw trace options and signature from carrier into OTel + Context. Note: tracestate is extracted by TraceContextTextMapPropagator + """ if context is None: context = Context() @@ -33,21 +35,29 @@ def extract( carrier, self._XTRACEOPTIONS_HEADER_NAME ) - if xtraceoptions_header: - logger.debug("Extracted xtraceoptions_header: {0}".format(xtraceoptions_header[0])) - xtraceoptions = XTraceOptions(xtraceoptions_header[0], context) - context.update(dict(xtraceoptions)) + if not xtraceoptions_header: + logger.debug("No xtraceoptions to extract; ignoring signature") + return context + logger.debug("Extracted xtraceoptions_header: {0}".format( + xtraceoptions_header[0] + )) signature_header = getter.get( carrier, self._XTRACEOPTIONS_SIGNATURE_HEADER_NAME ) if signature_header: - logger.debug("Extracted signature_header: {0}".format(signature_header[0])) - context.update({ - self._XTRACEOPTIONS_SIGNATURE_HEADER_NAME: signature_header[0] - }) - + xtraceoptions = XTraceOptions( + context, + xtraceoptions_header[0], + signature_header[0], + ) + else: + xtraceoptions = XTraceOptions( + context, + xtraceoptions_header[0], + ) + context.update(dict(xtraceoptions)) return context def inject( @@ -82,12 +92,9 @@ def inject( carrier, self._TRACESTATE_HEADER_NAME, trace_state.to_header() ) - # TODO: Prepare carrier with x-trace-options - @property def fields(self) -> typing.Set[str]: """Returns a set with the fields set in `inject`""" return { - self._TRACESTATE_HEADER_NAME, - self._XTRACEOPTIONS_HEADER_NAME + self._TRACESTATE_HEADER_NAME } diff --git a/opentelemetry_distro_solarwinds/response_propagator.py b/opentelemetry_distro_solarwinds/response_propagator.py index 557a2436..a4243c50 100644 --- a/opentelemetry_distro_solarwinds/response_propagator.py +++ b/opentelemetry_distro_solarwinds/response_propagator.py @@ -5,7 +5,9 @@ from opentelemetry.context.context import Context from opentelemetry.instrumentation.propagators import ResponsePropagator from opentelemetry.propagators import textmap +from opentelemetry.trace.span import TraceState +from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions from opentelemetry_distro_solarwinds.w3c_transformer import traceparent_from_context logger = logging.getLogger(__file__) @@ -30,30 +32,44 @@ def inject( return x_trace = traceparent_from_context(span_context) - - # TODO: Should be based on x-trace-options tt - sampled = 'trigger-trace=' - if span_context.trace_flags: - sampled += 'ok' - else: - sampled += 'rate-exceeded' - - exposed_headers = "{0},{1}".format( - self._XTRACE_HEADER_NAME, - self._XTRACEOPTIONS_RESPONSE_HEADER_NAME - ) setter.set( carrier, self._XTRACE_HEADER_NAME, x_trace, ) - setter.set( - carrier, - self._XTRACEOPTIONS_RESPONSE_HEADER_NAME, - sampled, + exposed_headers = self._XTRACE_HEADER_NAME + + xtraceoptions_response = self.recover_response_from_tracestate( + span_context.trace_state ) + if xtraceoptions_response: + exposed_headers += ",{0}".format( + self._XTRACEOPTIONS_RESPONSE_HEADER_NAME + ) + setter.set( + carrier, + self._XTRACEOPTIONS_RESPONSE_HEADER_NAME, + xtraceoptions_response, + ) setter.set( carrier, self._HTTP_HEADER_ACCESS_CONTROL_EXPOSE_HEADERS, exposed_headers, - ) \ No newline at end of file + ) + + def recover_response_from_tracestate( + self, + tracestate: TraceState + ) -> str: + """Use tracestate to recover xtraceoptions response by + converting delimiters: + `####` becomes `=` + `....` becomes `,` + """ + sanitized = tracestate.get( + XTraceOptions.get_sw_xtraceoptions_response_key(), + None + ) + if not sanitized: + return + return sanitized.replace("####", "=").replace("....", ",") diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 0e7c69f6..99d0240f 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -55,9 +55,6 @@ def __init__( class _SwSampler(Sampler): - - _XTRACEOPTIONS_SIGNATURE_HEADER_NAME = "x-trace-options-signature" - """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" def get_description(self) -> str: @@ -67,6 +64,7 @@ def calculate_liboboe_decision( self, parent_span_context: SpanContext, parent_context: Optional[OtelContext] = None, + xtraceoptions: Optional[XTraceOptions] = None, ) -> _LiboboeDecision: """Calculates oboe trace decision based on parent span context.""" tracestring = None @@ -80,7 +78,6 @@ def calculate_liboboe_decision( trigger_tracing_mode_disabled = 0 logger.debug("parent_context is {0}".format(parent_context)) - xtraceoptions = XTraceOptions("", parent_context) logger.debug("xtraceoptions is {0}".format(xtraceoptions)) options = None @@ -90,8 +87,7 @@ def calculate_liboboe_decision( if xtraceoptions: options = str(xtraceoptions) trigger_trace = xtraceoptions.trigger_trace - if parent_context: - signature = parent_context.get(self._XTRACEOPTIONS_SIGNATURE_HEADER_NAME, None) + signature = xtraceoptions.signature timestamp = xtraceoptions.ts logger.debug( @@ -182,10 +178,50 @@ def otel_decision_from_liboboe( logger.debug("OTel decision created: {0}".format(decision)) return decision + def create_xtraceoptions_response_value( + self, + decision: _LiboboeDecision, + parent_span_context: SpanContext, + xtraceoptions: XTraceOptions + ) -> str: + """Use decision and xtraceoptions to create sanitized xtraceoptions + response kv with W3C tracestate-allowed delimiters: + `####` instead of `=` + `....` instead of `,` + """ + response = [] + + if xtraceoptions.signature: + response.append("auth####{0}".format(decision.auth_msg)) + + if not decision.auth or decision.auth < 1: + trigger_msg = "" + if xtraceoptions.trigger_trace: + # If a traceparent header was provided then oboe does not generate the message + tracestring = None + if parent_span_context.is_valid and parent_span_context.is_remote: + tracestring = traceparent_from_context(parent_span_context) + + if tracestring and decision.type == 0: + trigger_msg = "ignored" + else: + trigger_msg = decision.status_msg + else: + trigger_msg = "not-requested" + response.append("trigger-trace####{0}".format(trigger_msg)) + + if xtraceoptions.ignored: + response.append( + "ignored####{0}".format("....".join(decision.ignored)) + ) + + return ";".join(response) + def create_new_trace_state( self, decision: _LiboboeDecision, - parent_span_context: SpanContext + parent_span_context: SpanContext, + xtraceoptions: Optional[XTraceOptions] = None, ) -> TraceState: """Creates new TraceState based on trace decision and parent span id.""" trace_state = TraceState([( @@ -195,25 +231,44 @@ def create_new_trace_state( trace_flags_from_int(decision.do_sample) ) )]) + if xtraceoptions and xtraceoptions.trigger_trace: + trace_state.add( + XTraceOptions.get_sw_xtraceoptions_response_key(), + self.create_xtraceoptions_response_value( + decision, + parent_span_context, + xtraceoptions + ) + ) logger.debug("Created new trace_state: {0}".format(trace_state)) return trace_state def calculate_trace_state( self, decision: _LiboboeDecision, - parent_span_context: SpanContext + parent_span_context: SpanContext, + xtraceoptions: Optional[XTraceOptions] = None, ) -> TraceState: """Calculates trace_state based on parent span context and trace decision, for non-existent or remote parent spans only.""" # No valid parent i.e. root span, or parent is remote if not parent_span_context.is_valid: - trace_state = self.create_new_trace_state(decision, parent_span_context) + trace_state = self.create_new_trace_state( + decision, + parent_span_context, + xtraceoptions + ) else: trace_state = parent_span_context.trace_state if not trace_state: # tracestate nonexistent/non-parsable - trace_state = self.create_new_trace_state(decision, parent_span_context) + trace_state = self.create_new_trace_state( + decision, + parent_span_context, + xtraceoptions + ) else: + # TODO: Should traceoptions piggyback ever be updated? # Update trace_state with span_id and sw trace_flags trace_state = trace_state.update( "sw", @@ -306,16 +361,19 @@ def should_sample( parent_span_context = get_current_span( parent_context ).get_span_context() + xtraceoptions = XTraceOptions(parent_context) decision = self.calculate_liboboe_decision( parent_span_context, - parent_context + parent_context, + xtraceoptions ) # Always calculate trace_state for propagation trace_state = self.calculate_trace_state( decision, - parent_span_context + parent_span_context, + xtraceoptions ) attributes = self.calculate_attributes( attributes, diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index 87fc4aa9..f68285a2 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -9,24 +9,37 @@ class XTraceOptions(): """Formats X-Trace-Options for trigger tracing""" - _TRACEOPTIONS_CUSTOM = ("^custom-[^\s]*$") - _TRACEOPTIONS_CUSTOM_RE = re.compile(_TRACEOPTIONS_CUSTOM) + _SW_XTRACEOPTIONS_RESPONSE_KEY = "xtrace_options_response" + _XTRACEOPTIONS_CUSTOM = ("^custom-[^\s]*$") + _XTRACEOPTIONS_CUSTOM_RE = re.compile(_XTRACEOPTIONS_CUSTOM) + + _OPTION_KEYS = [ + "custom_kvs", + "signature", + "sw_keys", + "trigger_trace", + "ts", + "ignored" + ] def __init__(self, - options_header: str, - context: typing.Optional[Context] = None + context: typing.Optional[Context] = None, + options_header: str = None, + signature_header: str = None ): """ Args: - options: A string of x-trace-options context: OTel context that may contain x-trace-options + options_header: A string of x-trace-options + signature_header: A string required for signed trigger trace - Examples of options: + Examples of options_header: "trigger-trace" "trigger-trace;sw-keys=check-id:check-1013,website-id:booking-demo" "trigger-trace;custom-key1=value1" """ self.custom_kvs = {} + self.signature = None self.sw_keys = {} self.trigger_trace = False self.ts = 0 @@ -67,7 +80,7 @@ def __init__(self, else: self.sw_keys.update({sw_kv[0]: sw_kv[1]}) - elif re.match(self._TRACEOPTIONS_CUSTOM_RE, option_key): + elif re.match(self._XTRACEOPTIONS_CUSTOM_RE, option_key): self.custom_kvs[option_key] = option_kv[1].strip() elif option_key == "ts": @@ -89,13 +102,19 @@ def __init__(self, "Some x-trace-options were ignored: {0}".format( ", ".join(self.ignored) )) + + if signature_header: + self.signature = signature_header def __iter__(self) -> typing.Iterator: """Iterable representation of XTraceOptions""" yield from self.__dict__.items() + # TODO this should be named something else if excludes signature def __str__(self) -> str: - """String representation of XTraceOptions""" + """String representation of XTraceOptions + + Note: Does not include signature.""" options_str = "" if self.trigger_trace: @@ -136,15 +155,10 @@ def from_context( logger.debug("Setting XTraceOptions from_context with {0}".format(context)) if not context: return + for option_key in self._OPTION_KEYS: + if context.get(option_key, None): + setattr(self, option_key, context[option_key]) - if "trigger_trace" in context and context["trigger_trace"]: - self.trigger_trace = True - - if "sw_keys" in context and context["sw_keys"]: - self.sw_keys = context["sw_keys"] - - if "custom_kvs" in context and context["custom_kvs"]: - self.custom_kvs = context["custom_kvs"] - - if "ts" in context and context["ts"] > 0: - self.ts = context["ts"] + @classmethod + def get_sw_xtraceoptions_response_key(cls) -> str: + return cls._SW_XTRACEOPTIONS_RESPONSE_KEY From 05f03c7b37edb716c3cdf9502a52f8c9c5d0add5 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 4 Apr 2022 15:22:07 -0700 Subject: [PATCH 38/71] Fix sampler tracestate creation; mv XTraceOptions str overload to to_options_header and tidy logic --- opentelemetry_distro_solarwinds/sampler.py | 6 +-- .../traceoptions.py | 38 +++++++------------ 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 99d0240f..f01fd7d6 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -78,14 +78,14 @@ def calculate_liboboe_decision( trigger_tracing_mode_disabled = 0 logger.debug("parent_context is {0}".format(parent_context)) - logger.debug("xtraceoptions is {0}".format(xtraceoptions)) + logger.debug("xtraceoptions is {0}".format(dict(xtraceoptions))) options = None trigger_trace = 0 signature = None timestamp = None if xtraceoptions: - options = str(xtraceoptions) + options = xtraceoptions.to_options_header() trigger_trace = xtraceoptions.trigger_trace signature = xtraceoptions.signature timestamp = xtraceoptions.ts @@ -232,7 +232,7 @@ def create_new_trace_state( ) )]) if xtraceoptions and xtraceoptions.trigger_trace: - trace_state.add( + trace_state = trace_state.add( XTraceOptions.get_sw_xtraceoptions_response_key(), self.create_xtraceoptions_response_value( decision, diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index f68285a2..b9e3030a 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -110,39 +110,27 @@ def __iter__(self) -> typing.Iterator: """Iterable representation of XTraceOptions""" yield from self.__dict__.items() - # TODO this should be named something else if excludes signature - def __str__(self) -> str: - """String representation of XTraceOptions - - Note: Does not include signature.""" - options_str = "" + def to_options_header(self) -> str: + """String representation of XTraceOptions, without signature.""" + options = [] if self.trigger_trace: - options_str += "trigger-trace" + options.append("trigger-trace") if len(self.sw_keys) > 0: - if len(options_str) > 0: - options_str += ";" - options_str += "sw-keys=" - for i, (k, v) in enumerate(self.sw_keys.items()): - options_str += "{0}:{1}".format(k, v) - if i < len(self.sw_keys) - 1: - options_str += "," + sw_keys = [] + for _, (k, v) in enumerate(self.sw_keys.items()): + sw_keys.append(":".join([k, v])) + options.append("=".join(["sw-keys", ",".join(sw_keys)])) if len(self.custom_kvs) > 0: - if len(options_str) > 0: - options_str += ";" - for i, (k, v) in enumerate(self.custom_kvs.items()): - options_str += "{0}={1}".format(k, v) - if i < len(self.custom_kvs) - 1: - options_str += ";" - + for _, (k, v) in enumerate(self.custom_kvs.items()): + options.append("=".join([k, v])) + if self.ts > 0: - if len(options_str) > 0: - options_str += ";" - options_str += "ts={0}".format(self.ts) + options.append("=".join(["ts", self.ts])) - return options_str + return ";".join(options) def from_context( self, From 3b62bc5e120edf75e20c7cdb931128e36ae0c3fc Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 4 Apr 2022 16:10:49 -0700 Subject: [PATCH 39/71] Sampler create x-trace-options-response if parent tracestate exists --- opentelemetry_distro_solarwinds/sampler.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index f01fd7d6..b7d645b6 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -202,7 +202,7 @@ def create_xtraceoptions_response_value( if parent_span_context.is_valid and parent_span_context.is_remote: tracestring = traceparent_from_context(parent_span_context) - if tracestring and decision.type == 0: + if tracestring and decision.decision_type == 0: trigger_msg = "ignored" else: trigger_msg = decision.status_msg @@ -223,7 +223,8 @@ def create_new_trace_state( parent_span_context: SpanContext, xtraceoptions: Optional[XTraceOptions] = None, ) -> TraceState: - """Creates new TraceState based on trace decision and parent span id.""" + """Creates new TraceState based on trace decision, parent span id, + and x-trace-options if provided""" trace_state = TraceState([( "sw", sw_from_span_and_decision( @@ -249,8 +250,9 @@ def calculate_trace_state( parent_span_context: SpanContext, xtraceoptions: Optional[XTraceOptions] = None, ) -> TraceState: - """Calculates trace_state based on parent span context and trace decision, - for non-existent or remote parent spans only.""" + """Calculates trace_state based on parent span context, trace decision, + and x-trace-options if provided -- for non-existent or remote parent + spans only.""" # No valid parent i.e. root span, or parent is remote if not parent_span_context.is_valid: trace_state = self.create_new_trace_state( @@ -268,7 +270,6 @@ def calculate_trace_state( xtraceoptions ) else: - # TODO: Should traceoptions piggyback ever be updated? # Update trace_state with span_id and sw trace_flags trace_state = trace_state.update( "sw", @@ -277,6 +278,17 @@ def calculate_trace_state( trace_flags_from_int(decision.do_sample) ) ) + # Update trace_state with x-trace-options-response + # Not a propagated header, so always an add + if xtraceoptions and xtraceoptions.trigger_trace: + trace_state = trace_state.add( + XTraceOptions.get_sw_xtraceoptions_response_key(), + self.create_xtraceoptions_response_value( + decision, + parent_span_context, + xtraceoptions + ) + ) logger.debug("Updated trace_state: {0}".format(trace_state)) return trace_state From 1dfd489183fc8506077b1aee3403c49fd251a09f Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 4 Apr 2022 16:24:20 -0700 Subject: [PATCH 40/71] Fix bug in to_options_header --- opentelemetry_distro_solarwinds/traceoptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index b9e3030a..1810dc49 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -128,7 +128,7 @@ def to_options_header(self) -> str: options.append("=".join([k, v])) if self.ts > 0: - options.append("=".join(["ts", self.ts])) + options.append("=".join(["ts", str(self.ts)])) return ";".join(options) From 99d2fc449399ec9f35f4cad0678bbac1b6ed81b3 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 4 Apr 2022 17:30:51 -0700 Subject: [PATCH 41/71] Add W3CTransformer classmethods --- opentelemetry_distro_solarwinds/exporter.py | 6 +- opentelemetry_distro_solarwinds/propagator.py | 4 +- .../response_propagator.py | 4 +- opentelemetry_distro_solarwinds/sampler.py | 26 +++--- .../w3c_transformer.py | 84 ++++++++++++------- 5 files changed, 75 insertions(+), 49 deletions(-) diff --git a/opentelemetry_distro_solarwinds/exporter.py b/opentelemetry_distro_solarwinds/exporter.py index d17944d7..6b0cf1b4 100644 --- a/opentelemetry_distro_solarwinds/exporter.py +++ b/opentelemetry_distro_solarwinds/exporter.py @@ -11,7 +11,7 @@ from opentelemetry_distro_solarwinds.extension.oboe import (Context, Metadata, Reporter) -from opentelemetry_distro_solarwinds.w3c_transformer import traceparent_from_context +from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer logger = logging.getLogger(__file__) @@ -122,4 +122,6 @@ def _initialize_solarwinds_reporter(self): @staticmethod def _build_metadata(span_context): - return Metadata.fromString(traceparent_from_context(span_context)) + return Metadata.fromString( + W3CTransformer.traceparent_from_context(span_context) + ) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index b30a6a35..8723e371 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -7,7 +7,7 @@ from opentelemetry.trace.span import TraceState from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions -from opentelemetry_distro_solarwinds.w3c_transformer import sw_from_context +from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer logger = logging.getLogger(__file__) @@ -70,7 +70,7 @@ def inject( span = trace.get_current_span(context) span_context = span.get_span_context() trace_state = span_context.trace_state - sw_value = sw_from_context(span_context) + sw_value = W3CTransformer.sw_from_context(span_context) # Prepare carrier with context's or new tracestate if trace_state: diff --git a/opentelemetry_distro_solarwinds/response_propagator.py b/opentelemetry_distro_solarwinds/response_propagator.py index a4243c50..43d10515 100644 --- a/opentelemetry_distro_solarwinds/response_propagator.py +++ b/opentelemetry_distro_solarwinds/response_propagator.py @@ -8,7 +8,7 @@ from opentelemetry.trace.span import TraceState from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions -from opentelemetry_distro_solarwinds.w3c_transformer import traceparent_from_context +from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer logger = logging.getLogger(__file__) @@ -31,7 +31,7 @@ def inject( if span_context == trace.INVALID_SPAN_CONTEXT: return - x_trace = traceparent_from_context(span_context) + x_trace = W3CTransformer.traceparent_from_context(span_context) setter.set( carrier, self._XTRACE_HEADER_NAME, diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index b7d645b6..83ba492c 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -16,11 +16,7 @@ from opentelemetry_distro_solarwinds.extension.oboe import Context from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions -from opentelemetry_distro_solarwinds.w3c_transformer import ( - trace_flags_from_int, - traceparent_from_context, - sw_from_span_and_decision -) +from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer logger = logging.getLogger(__name__) @@ -69,7 +65,9 @@ def calculate_liboboe_decision( """Calculates oboe trace decision based on parent span context.""" tracestring = None if parent_span_context.is_valid and parent_span_context.is_remote: - tracestring = traceparent_from_context(parent_span_context) + tracestring = W3CTransformer.traceparent_from_context( + parent_span_context + ) sw_member_value = parent_span_context.trace_state.get("sw") # TODO: config --> enable/disable tracing, sample_rate, tt mode @@ -200,7 +198,9 @@ def create_xtraceoptions_response_value( # If a traceparent header was provided then oboe does not generate the message tracestring = None if parent_span_context.is_valid and parent_span_context.is_remote: - tracestring = traceparent_from_context(parent_span_context) + tracestring = W3CTransformer.traceparent_from_context( + parent_span_context + ) if tracestring and decision.decision_type == 0: trigger_msg = "ignored" @@ -227,9 +227,9 @@ def create_new_trace_state( and x-trace-options if provided""" trace_state = TraceState([( "sw", - sw_from_span_and_decision( + W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, - trace_flags_from_int(decision.do_sample) + W3CTransformer.trace_flags_from_int(decision.do_sample) ) )]) if xtraceoptions and xtraceoptions.trigger_trace: @@ -273,9 +273,9 @@ def calculate_trace_state( # Update trace_state with span_id and sw trace_flags trace_state = trace_state.update( "sw", - sw_from_span_and_decision( + W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, - trace_flags_from_int(decision.do_sample) + W3CTransformer.trace_flags_from_int(decision.do_sample) ) ) # Update trace_state with x-trace-options-response @@ -343,9 +343,9 @@ def calculate_attributes( ) attr_trace_state.update( "sw", - sw_from_span_and_decision( + W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, - trace_flags_from_int(decision.do_sample) + W3CTransformer.trace_flags_from_int(decision.do_sample) ) ) new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() diff --git a/opentelemetry_distro_solarwinds/w3c_transformer.py b/opentelemetry_distro_solarwinds/w3c_transformer.py index bdc3601f..22229f7c 100644 --- a/opentelemetry_distro_solarwinds/w3c_transformer.py +++ b/opentelemetry_distro_solarwinds/w3c_transformer.py @@ -6,33 +6,57 @@ logger = logging.getLogger(__file__) -def span_id_from_int(span_id: int) -> str: - """Formats span ID as 16-byte hexadecimal string""" - return "{:016x}".format(span_id) - -def trace_flags_from_int(trace_flags: int) -> str: - """Formats trace flags as 8-bit field""" - return "{:02x}".format(trace_flags) - -def traceparent_from_context(span_context: Context) -> str: - """Generates a liboboe W3C compatible trace_context from provided OTel span context.""" - xtr = "00-{0:032X}-{1:016X}-{2:02X}".format(span_context.trace_id, - span_context.span_id, - span_context.trace_flags) - logger.debug("Generated traceparent {0} from {1}".format(xtr, span_context)) - return xtr - -def sw_from_context(span_context: Context) -> str: - """Formats tracestate sw value from SpanContext as 16-byte span_id - with 8-bit trace_flags. - - Example: 1a2b3c4d5e6f7g8h-01""" - return "{0:016x}-{1:02x}".format(span_context.span_id, - span_context.trace_flags) - -def sw_from_span_and_decision(span_id: int, decision: str) -> str: - """Formats tracestate sw value from span_id and liboboe decision - as 16-byte span_id with 8-bit trace_flags. - - Example: 1a2b3c4d5e6f7g8h-01""" - return "{0:016x}-{1}".format(span_id, decision) +class W3CTransformer(): + """Transforms inputs to W3C-compliant data for SolarWinds context propagation""" + + _DECISION = "{}" + _SPAN_ID_HEX = "{:016x}" + _TRACE_FLAGS_HEX = "{:02x}" + _TRACE_ID_HEX = "{:032x}" + _VERSION = "00" + + @classmethod + def span_id_from_int(cls, span_id: int) -> str: + """Formats span ID as 16-byte hexadecimal string""" + return cls._SPAN_ID_HEX.format(span_id) + + @classmethod + def trace_flags_from_int(cls, trace_flags: int) -> str: + """Formats trace flags as 8-bit field""" + return cls._TRACE_FLAGS_HEX.format(trace_flags) + + @classmethod + def traceparent_from_context(cls, span_context: Context) -> str: + """Generates an uppercase liboboe W3C compatible trace_context from + provided OTel span context.""" + template = "-".join([ + cls._VERSION, + cls._TRACE_ID_HEX, + cls._SPAN_ID_HEX, + cls._TRACE_FLAGS_HEX + ]) + xtr = template.format( + span_context.trace_id, + span_context.span_id, + span_context.trace_flags + ).upper() + logger.debug("Generated traceparent {} from {}".format(xtr, span_context)) + return xtr + + @classmethod + def sw_from_context(cls, span_context: Context) -> str: + """Formats tracestate sw value from SpanContext as 16-byte span_id + with 8-bit trace_flags. + + Example: 1a2b3c4d5e6f7g8h-01""" + sw = "-".join([cls._SPAN_ID_HEX, cls._TRACE_FLAGS_HEX]) + return sw.format(span_context.span_id, span_context.trace_flags) + + @classmethod + def sw_from_span_and_decision(cls, span_id: int, decision: str) -> str: + """Formats tracestate sw value from span_id and liboboe decision + as 16-byte span_id with 8-bit trace_flags. + + Example: 1a2b3c4d5e6f7g8h-01""" + sw = "-".join([cls._SPAN_ID_HEX, cls._DECISION]) + return sw.format(span_id, decision) From 32df3d08cc30bd3551580f8bd38a09d837008f3c Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 6 Apr 2022 14:17:34 -0700 Subject: [PATCH 42/71] Fix Sampler OBOE_SETTINGS_UNSET of tracing_mode, sample_rate, trigger_tracing_mode_disabled --- opentelemetry_distro_solarwinds/sampler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 83ba492c..d704c2d3 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -70,10 +70,10 @@ def calculate_liboboe_decision( ) sw_member_value = parent_span_context.trace_state.get("sw") - # TODO: config --> enable/disable tracing, sample_rate, tt mode - tracing_mode = 1 - sample_rate = 1 - trigger_tracing_mode_disabled = 0 + # TODO: local config --> enable/disable tracing, sample_rate, tt mode + tracing_mode = -1 + sample_rate = -1 + trigger_tracing_mode_disabled = -1 logger.debug("parent_context is {0}".format(parent_context)) logger.debug("xtraceoptions is {0}".format(dict(xtraceoptions))) From 7444d4336917365662882db242d445b831d3532b Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 6 Apr 2022 16:11:54 -0700 Subject: [PATCH 43/71] String formatting a bit less brittle --- opentelemetry_distro_solarwinds/exporter.py | 4 +- opentelemetry_distro_solarwinds/propagator.py | 8 +-- .../response_propagator.py | 2 +- opentelemetry_distro_solarwinds/sampler.py | 62 +++++++++---------- .../traceoptions.py | 8 +-- setup.py | 2 +- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/opentelemetry_distro_solarwinds/exporter.py b/opentelemetry_distro_solarwinds/exporter.py index 6b0cf1b4..8c1ed70d 100644 --- a/opentelemetry_distro_solarwinds/exporter.py +++ b/opentelemetry_distro_solarwinds/exporter.py @@ -36,7 +36,7 @@ def export(self, spans): md = self._build_metadata(span.get_span_context()) if span.parent and span.parent.is_valid: # If there is a parent, we need to add an edge to this parent to this entry event - logger.debug("Continue trace from {0}".format(md.toString())) + logger.debug("Continue trace from {}".format(md.toString())) parent_md = self._build_metadata(span.parent) evt = Context.startTrace(md, int(span.start_time / 1000), parent_md) @@ -44,7 +44,7 @@ def export(self, spans): # In OpenTelemrtry, there are no events with individual IDs, but only a span ID # and trace ID. Thus, the entry event needs to be generated such that it has the # same op ID as the span ID of the OTel span. - logger.debug("Start a new trace {0}".format(md.toString())) + logger.debug("Start a new trace {}".format(md.toString())) evt = Context.startTrace(md, int(span.start_time / 1000)) evt.addInfo('Layer', span.name) evt.addInfo('Language', 'Python') diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 8723e371..ea7846e4 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -38,7 +38,7 @@ def extract( if not xtraceoptions_header: logger.debug("No xtraceoptions to extract; ignoring signature") return context - logger.debug("Extracted xtraceoptions_header: {0}".format( + logger.debug("Extracted xtraceoptions_header: {}".format( xtraceoptions_header[0] )) @@ -77,15 +77,15 @@ def inject( # Check if trace_state already contains sw KV if "sw" in trace_state.keys(): # If so, modify current span_id and trace_flags, and move to beginning of list - logger.debug("Updating trace state for injection with {0}".format(sw_value)) + logger.debug("Updating trace state for injection with {}".format(sw_value)) trace_state = trace_state.update("sw", sw_value) else: # If not, add sw KV to beginning of list - logger.debug("Adding KV to trace state for injection with {0}".format(sw_value)) + logger.debug("Adding KV to trace state for injection with {}".format(sw_value)) trace_state.add("sw", sw_value) else: - logger.debug("Creating new trace state for injection with {0}".format(sw_value)) + logger.debug("Creating new trace state for injection with {}".format(sw_value)) trace_state = TraceState([("sw", sw_value)]) setter.set( diff --git a/opentelemetry_distro_solarwinds/response_propagator.py b/opentelemetry_distro_solarwinds/response_propagator.py index 43d10515..7e4c0616 100644 --- a/opentelemetry_distro_solarwinds/response_propagator.py +++ b/opentelemetry_distro_solarwinds/response_propagator.py @@ -43,7 +43,7 @@ def inject( span_context.trace_state ) if xtraceoptions_response: - exposed_headers += ",{0}".format( + exposed_headers += ",{}".format( self._XTRACEOPTIONS_RESPONSE_HEADER_NAME ) setter.set( diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index d704c2d3..553dfc8f 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -75,8 +75,8 @@ def calculate_liboboe_decision( sample_rate = -1 trigger_tracing_mode_disabled = -1 - logger.debug("parent_context is {0}".format(parent_context)) - logger.debug("xtraceoptions is {0}".format(dict(xtraceoptions))) + logger.debug("parent_context is {}".format(parent_context)) + logger.debug("xtraceoptions is {}".format(dict(xtraceoptions))) options = None trigger_trace = 0 @@ -90,15 +90,15 @@ def calculate_liboboe_decision( logger.debug( "Creating new oboe decision with " - "tracestring: {0}, " - "sw_member_value: {1}, " - "tracing_mode: {2}, " - "sample_rate: {3}, " - "trigger_trace: {4}, " - "trigger_tracing_mode_disabled: {5}, " - "options: {6}, " - "signature: {7}, " - "timestamp: {8}".format( + "tracestring: {}, " + "sw_member_value: {}, " + "tracing_mode: {}, " + "sample_rate: {}, " + "trigger_trace: {}, " + "trigger_tracing_mode_disabled: {}, " + "options: {}, " + "signature: {}, " + "timestamp: {}".format( tracestring, sw_member_value, tracing_mode, @@ -124,17 +124,17 @@ def calculate_liboboe_decision( ) logger.debug( "Got liboboe decision outputs " - "do_metrics: {0}, " - "do_sample: {1}, " - "rate: {2}, " - "source: {3}, " - "bucket_rate: {4}, " - "bucket_cap: {5}, " - "type: {6}, " - "auth: {7}, " - "status_msg: {8}, " - "auth_msg: {9}, " - "status: {10}".format( + "do_metrics: {}, " + "do_sample: {}, " + "rate: {}, " + "source: {}, " + "bucket_rate: {}, " + "bucket_cap: {}, " + "type: {}, " + "auth: {}, " + "status_msg: {}, " + "auth_msg: {}, " + "status: {}".format( do_metrics, do_sample, rate, @@ -173,7 +173,7 @@ def otel_decision_from_liboboe( decision = Decision.RECORD_ONLY if liboboe_decision.do_sample: decision = Decision.RECORD_AND_SAMPLE - logger.debug("OTel decision created: {0}".format(decision)) + logger.debug("OTel decision created: {}".format(decision)) return decision def create_xtraceoptions_response_value( @@ -190,7 +190,7 @@ def create_xtraceoptions_response_value( response = [] if xtraceoptions.signature: - response.append("auth####{0}".format(decision.auth_msg)) + response.append("auth####{}".format(decision.auth_msg)) if not decision.auth or decision.auth < 1: trigger_msg = "" @@ -208,11 +208,11 @@ def create_xtraceoptions_response_value( trigger_msg = decision.status_msg else: trigger_msg = "not-requested" - response.append("trigger-trace####{0}".format(trigger_msg)) + response.append("trigger-trace####{}".format(trigger_msg)) if xtraceoptions.ignored: response.append( - "ignored####{0}".format("....".join(decision.ignored)) + "ignored####{}".format("....".join(decision.ignored)) ) return ";".join(response) @@ -241,7 +241,7 @@ def create_new_trace_state( xtraceoptions ) ) - logger.debug("Created new trace_state: {0}".format(trace_state)) + logger.debug("Created new trace_state: {}".format(trace_state)) return trace_state def calculate_trace_state( @@ -289,7 +289,7 @@ def calculate_trace_state( xtraceoptions ) ) - logger.debug("Updated trace_state: {0}".format(trace_state)) + logger.debug("Updated trace_state: {}".format(trace_state)) return trace_state def calculate_attributes( @@ -303,7 +303,7 @@ def calculate_attributes( parent span context, and existing attributes. For non-existent or remote parent spans only.""" - logger.debug("Received attributes: {0}".format(attributes)) + logger.debug("Received attributes: {}".format(attributes)) # Don't set attributes if not tracing if self.otel_decision_from_liboboe(decision) == Decision.DROP: @@ -325,7 +325,7 @@ def calculate_attributes( if sw_value: attributes["sw.tracestate_parent_id"] = sw_value - logger.debug("Created new attributes: {0}".format(attributes)) + logger.debug("Created new attributes: {}".format(attributes)) else: # Copy existing MappingProxyType KV into new_attributes for modification # attributes may have other vendor info etc @@ -351,7 +351,7 @@ def calculate_attributes( new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() attributes = new_attributes - logger.debug("Set updated attributes: {0}".format(attributes)) + logger.debug("Set updated attributes: {}".format(attributes)) # attributes must be immutable for SamplingResult return MappingProxyType(attributes) diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index 1810dc49..186e4b9b 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -73,7 +73,7 @@ def __init__(self, sw_kv = assignment.split(":", 2) if not sw_kv[0]: logger.warning( - "Could not parse sw-key assignment {0}. Ignoring.".format( + "Could not parse sw-key assignment {}. Ignoring.".format( assignment )) self.ignore.append(assignment) @@ -92,14 +92,14 @@ def __init__(self, else: logger.warning( - "{0} is not a recognized trace option. Ignoring".format( + "{} is not a recognized trace option. Ignoring".format( option_key )) self.ignored.append(option_key) if self.ignored: logger.warning( - "Some x-trace-options were ignored: {0}".format( + "Some x-trace-options were ignored: {}".format( ", ".join(self.ignored) )) @@ -140,7 +140,7 @@ def from_context( Args: context: OTel context that may contain x-trace-options """ - logger.debug("Setting XTraceOptions from_context with {0}".format(context)) + logger.debug("Setting XTraceOptions from_context with {}".format(context)) if not context: return for option_key in self._OPTION_KEYS: diff --git a/setup.py b/setup.py index 884945e9..7ef9e778 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def link_oboe_lib(src_lib): if os.path.exists(dst): # if the destination library files exist already, they need to be deleted, otherwise linking will fail os.remove(dst) - log.info("Removed {0}".format(dst)) + log.info("Removed {}".format(dst)) os.symlink(src_lib, dst) log.info("Created new link at {} to {}".format(dst, src_lib)) except Exception as ecp: From 84a863e0743cce5f5d2ce0bee91c5d7d873872ec Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 6 Apr 2022 16:19:59 -0700 Subject: [PATCH 44/71] Fix trace_state KV add at inject --- opentelemetry_distro_solarwinds/propagator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index ea7846e4..427cca7a 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -83,7 +83,7 @@ def inject( else: # If not, add sw KV to beginning of list logger.debug("Adding KV to trace state for injection with {}".format(sw_value)) - trace_state.add("sw", sw_value) + trace_state = trace_state.add("sw", sw_value) else: logger.debug("Creating new trace state for injection with {}".format(sw_value)) trace_state = TraceState([("sw", sw_value)]) From de885658b6f0ab0d95b06c4ddbb8278f22b923dd Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 6 Apr 2022 16:28:12 -0700 Subject: [PATCH 45/71] Style and comment for ResponsePropagator --- .../response_propagator.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/opentelemetry_distro_solarwinds/response_propagator.py b/opentelemetry_distro_solarwinds/response_propagator.py index 7e4c0616..b21bc49f 100644 --- a/opentelemetry_distro_solarwinds/response_propagator.py +++ b/opentelemetry_distro_solarwinds/response_propagator.py @@ -13,8 +13,7 @@ logger = logging.getLogger(__file__) class SolarWindsTraceResponsePropagator(ResponsePropagator): - """ - """ + """Propagator that injects SW values into HTTP responses""" _HTTP_HEADER_ACCESS_CONTROL_EXPOSE_HEADERS = "Access-Control-Expose-Headers" _XTRACE_HEADER_NAME = "x-trace" _XTRACEOPTIONS_RESPONSE_HEADER_NAME = "x-trace-options-response" @@ -37,15 +36,15 @@ def inject( self._XTRACE_HEADER_NAME, x_trace, ) - exposed_headers = self._XTRACE_HEADER_NAME + exposed_headers = [self._XTRACE_HEADER_NAME] xtraceoptions_response = self.recover_response_from_tracestate( span_context.trace_state ) if xtraceoptions_response: - exposed_headers += ",{}".format( + exposed_headers.append("{}".format( self._XTRACEOPTIONS_RESPONSE_HEADER_NAME - ) + )) setter.set( carrier, self._XTRACEOPTIONS_RESPONSE_HEADER_NAME, @@ -54,7 +53,7 @@ def inject( setter.set( carrier, self._HTTP_HEADER_ACCESS_CONTROL_EXPOSE_HEADERS, - exposed_headers, + ",".join(exposed_headers), ) def recover_response_from_tracestate( From ec2c317bdf6796aa19a8f0584615107b8b052d57 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 8 Apr 2022 14:20:39 -0700 Subject: [PATCH 46/71] Python 3.6+, otel SDK 1.10.0, instrumentation 0.29.b0 --- setup.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 53e7625a..7d13bf18 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,11 +16,11 @@ classifiers = Programming Language :: Python :: 3.8 [options] -python_requires = >=3.4 +python_requires = >=3.6 install_requires = - opentelemetry-api >= 1.3.0 - opentelemetry-sdk >= 1.3.0 - opentelemetry-instrumentation >= 0.29b0 + opentelemetry-api == 1.10.0 + opentelemetry-sdk == 1.10.0 + opentelemetry-instrumentation == 0.29b0 [options.entry_points] opentelemetry_distro = From 59b1bac8828e3b7610a732937ebb3cfda3373911 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 8 Apr 2022 14:47:11 -0700 Subject: [PATCH 47/71] xtraceoptions log lines to debug level --- opentelemetry_distro_solarwinds/traceoptions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index 186e4b9b..3295cdf6 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -60,7 +60,7 @@ def __init__(self, option_key = option_kv[0].strip() if option_key == "trigger-trace": if len(option_kv) > 1: - logger.warning("trigger-trace must be standalone flag. Ignoring.") + logger.debug("trigger-trace must be standalone flag. Ignoring.") self.ignored.append("trigger-trace") else: self.trigger_trace = True @@ -72,7 +72,7 @@ def __init__(self, # each of sw-keys values assigned by colon sw_kv = assignment.split(":", 2) if not sw_kv[0]: - logger.warning( + logger.debug( "Could not parse sw-key assignment {}. Ignoring.".format( assignment )) @@ -87,18 +87,18 @@ def __init__(self, try: self.ts = int(option_kv[1]) except ValueError as e: - logger.warning("ts must be base 10 int. Ignoring.") + logger.debug("ts must be base 10 int. Ignoring.") self.ignore.append("ts") else: - logger.warning( + logger.debug( "{} is not a recognized trace option. Ignoring".format( option_key )) self.ignored.append(option_key) if self.ignored: - logger.warning( + logger.debug( "Some x-trace-options were ignored: {}".format( ", ".join(self.ignored) )) From 45f191065b8d26b58de4473f29dacf8b06db4316 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 8 Apr 2022 17:21:34 -0700 Subject: [PATCH 48/71] Assign several keys/attrs as constants --- opentelemetry_distro_solarwinds/__init__.py | 6 ++ opentelemetry_distro_solarwinds/propagator.py | 9 +-- .../response_propagator.py | 15 ++++- opentelemetry_distro_solarwinds/sampler.py | 66 ++++++++++++------- .../traceoptions.py | 31 ++++++--- 5 files changed, 90 insertions(+), 37 deletions(-) diff --git a/opentelemetry_distro_solarwinds/__init__.py b/opentelemetry_distro_solarwinds/__init__.py index f102a9ca..d805f60a 100644 --- a/opentelemetry_distro_solarwinds/__init__.py +++ b/opentelemetry_distro_solarwinds/__init__.py @@ -1 +1,7 @@ __version__ = "0.0.1" + +COMMA = "," +COMMA_W3C_SANITIZED = "...." +EQUALS = "=" +EQUALS_W3C_SANITIZED = "####" +SW_TRACESTATE_KEY = "sw" \ No newline at end of file diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 427cca7a..4baf0676 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -6,6 +6,7 @@ from opentelemetry.propagators import textmap from opentelemetry.trace.span import TraceState +from opentelemetry_distro_solarwinds import SW_TRACESTATE_KEY from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer @@ -75,18 +76,18 @@ def inject( # Prepare carrier with context's or new tracestate if trace_state: # Check if trace_state already contains sw KV - if "sw" in trace_state.keys(): + if SW_TRACESTATE_KEY in trace_state.keys(): # If so, modify current span_id and trace_flags, and move to beginning of list logger.debug("Updating trace state for injection with {}".format(sw_value)) - trace_state = trace_state.update("sw", sw_value) + trace_state = trace_state.update(SW_TRACESTATE_KEY, sw_value) else: # If not, add sw KV to beginning of list logger.debug("Adding KV to trace state for injection with {}".format(sw_value)) - trace_state = trace_state.add("sw", sw_value) + trace_state = trace_state.add(SW_TRACESTATE_KEY, sw_value) else: logger.debug("Creating new trace state for injection with {}".format(sw_value)) - trace_state = TraceState([("sw", sw_value)]) + trace_state = TraceState([(SW_TRACESTATE_KEY, sw_value)]) setter.set( carrier, self._TRACESTATE_HEADER_NAME, trace_state.to_header() diff --git a/opentelemetry_distro_solarwinds/response_propagator.py b/opentelemetry_distro_solarwinds/response_propagator.py index b21bc49f..39d5a149 100644 --- a/opentelemetry_distro_solarwinds/response_propagator.py +++ b/opentelemetry_distro_solarwinds/response_propagator.py @@ -7,6 +7,12 @@ from opentelemetry.propagators import textmap from opentelemetry.trace.span import TraceState +from opentelemetry_distro_solarwinds import ( + COMMA, + COMMA_W3C_SANITIZED, + EQUALS, + EQUALS_W3C_SANITIZED +) from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer @@ -62,8 +68,8 @@ def recover_response_from_tracestate( ) -> str: """Use tracestate to recover xtraceoptions response by converting delimiters: - `####` becomes `=` - `....` becomes `,` + EQUALS_W3C_SANITIZED becomes EQUALS + COMMA_W3C_SANITIZED becomes COMMA """ sanitized = tracestate.get( XTraceOptions.get_sw_xtraceoptions_response_key(), @@ -71,4 +77,7 @@ def recover_response_from_tracestate( ) if not sanitized: return - return sanitized.replace("####", "=").replace("....", ",") + return sanitized.replace( + EQUALS_W3C_SANITIZED, + EQUALS + ).replace(COMMA_W3C_SANITIZED, COMMA) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 553dfc8f..54358a76 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -14,6 +14,11 @@ from opentelemetry.trace.span import SpanContext, TraceState from opentelemetry.util.types import Attributes +from opentelemetry_distro_solarwinds import ( + COMMA_W3C_SANITIZED, + EQUALS_W3C_SANITIZED, + SW_TRACESTATE_KEY +) from opentelemetry_distro_solarwinds.extension.oboe import Context from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer @@ -53,6 +58,14 @@ def __init__( class _SwSampler(Sampler): """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" + _SW_TRACESTATE_CAPTURE_KEY = "sw.w3c.tracestate" + _SW_TRACESTATE_ROOT_KEY = "sw.tracestate_parent_id" + _XTRACEOPTIONS_RESP_AUTH = "auth" + _XTRACEOPTIONS_RESP_IGNORED = "ignored" + _XTRACEOPTIONS_RESP_TRIGGER_IGNORED = "ignored" + _XTRACEOPTIONS_RESP_TRIGGER_NOT_REQUESTED = "not-requested" + _XTRACEOPTIONS_RESP_TRIGGER_TRACE = "trigger-trace" + def get_description(self) -> str: return "SolarWinds custom opentelemetry sampler" @@ -68,7 +81,7 @@ def calculate_liboboe_decision( tracestring = W3CTransformer.traceparent_from_context( parent_span_context ) - sw_member_value = parent_span_context.trace_state.get("sw") + sw_member_value = parent_span_context.trace_state.get(SW_TRACESTATE_KEY) # TODO: local config --> enable/disable tracing, sample_rate, tt mode tracing_mode = -1 @@ -184,13 +197,16 @@ def create_xtraceoptions_response_value( ) -> str: """Use decision and xtraceoptions to create sanitized xtraceoptions response kv with W3C tracestate-allowed delimiters: - `####` instead of `=` - `....` instead of `,` + EQUALS_W3C_SANITIZED instead of EQUALS + COMMA_W3C_SANITIZED instead of COMMA """ response = [] if xtraceoptions.signature: - response.append("auth####{}".format(decision.auth_msg)) + response.append(EQUALS_W3C_SANITIZED.join([ + self._XTRACEOPTIONS_RESP_AUTH, + decision.auth_msg + ])) if not decision.auth or decision.auth < 1: trigger_msg = "" @@ -203,16 +219,22 @@ def create_xtraceoptions_response_value( ) if tracestring and decision.decision_type == 0: - trigger_msg = "ignored" + trigger_msg = self._XTRACEOPTIONS_RESP_TRIGGER_IGNORED else: trigger_msg = decision.status_msg else: - trigger_msg = "not-requested" - response.append("trigger-trace####{}".format(trigger_msg)) + trigger_msg = self.XTRACEOPTIONS_TRIGGER_NOT_REQUESTED + response.append(EQUALS_W3C_SANITIZED.join([ + self._XTRACEOPTIONS_RESP_TRIGGER_TRACE, + trigger_msg + ])) if xtraceoptions.ignored: response.append( - "ignored####{}".format("....".join(decision.ignored)) + EQUALS_W3C_SANITIZED.join([ + self._XTRACEOPTIONS_RESP_IGNORED, + (COMMA_W3C_SANITIZED.join(decision.ignored)) + ]) ) return ";".join(response) @@ -226,7 +248,7 @@ def create_new_trace_state( """Creates new TraceState based on trace decision, parent span id, and x-trace-options if provided""" trace_state = TraceState([( - "sw", + SW_TRACESTATE_KEY, W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, W3CTransformer.trace_flags_from_int(decision.do_sample) @@ -272,7 +294,7 @@ def calculate_trace_state( else: # Update trace_state with span_id and sw trace_flags trace_state = trace_state.update( - "sw", + SW_TRACESTATE_KEY, W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, W3CTransformer.trace_flags_from_int(decision.do_sample) @@ -315,15 +337,15 @@ def calculate_attributes( logger.debug("No valid traceparent or no tracestate - not setting attributes") return None - # Set attributes with sw.tracestate_parent_id and sw.w3c.tracestate + # Set attributes with self._SW_TRACESTATE_ROOT_KEY and self._SW_TRACESTATE_CAPTURE_KEY if not attributes: attributes = { - "sw.w3c.tracestate": trace_state.to_header() + self._SW_TRACESTATE_CAPTURE_KEY: trace_state.to_header() } - # Only set sw.tracestate_parent_id on the entry (root) span for this service - sw_value = parent_span_context.trace_state.get("sw", None) + # Only set self._SW_TRACESTATE_ROOT_KEY on the entry (root) span for this service + sw_value = parent_span_context.trace_state.get(SW_TRACESTATE_KEY, None) if sw_value: - attributes["sw.tracestate_parent_id"] = sw_value + attributes[self._SW_TRACESTATE_ROOT_KEY] = sw_value logger.debug("Created new attributes: {}".format(attributes)) else: @@ -333,22 +355,22 @@ def calculate_attributes( for k,v in attributes.items(): new_attributes[k] = v - if not new_attributes.get("sw.w3c.tracestate", None): - # Add new sw.w3c.tracestate KV - new_attributes["sw.w3c.tracestate"] = trace_state.to_header() + if not new_attributes.get(self._SW_TRACESTATE_CAPTURE_KEY, None): + # Add new self._SW_TRACESTATE_CAPTURE_KEY KV + new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = trace_state.to_header() else: - # Update existing sw.w3c.tracestate KV + # Update existing self._SW_TRACESTATE_CAPTURE_KEY KV attr_trace_state = TraceState.from_header( - new_attributes["sw.w3c.tracestate"] + new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] ) attr_trace_state.update( - "sw", + SW_TRACESTATE_KEY, W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, W3CTransformer.trace_flags_from_int(decision.do_sample) ) ) - new_attributes["sw.w3c.tracestate"] = attr_trace_state.to_header() + new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = attr_trace_state.to_header() attributes = new_attributes logger.debug("Set updated attributes: {}".format(attributes)) diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index 3295cdf6..9f044c52 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -13,6 +13,10 @@ class XTraceOptions(): _XTRACEOPTIONS_CUSTOM = ("^custom-[^\s]*$") _XTRACEOPTIONS_CUSTOM_RE = re.compile(_XTRACEOPTIONS_CUSTOM) + _XTRACEOPTIONS_HEADER_KEY_SW_KEYS = "sw-keys" + _XTRACEOPTIONS_HEADER_KEY_TRIGGER_TRACE = "trigger-trace" + _XTRACEOPTIONS_HEADER_KEY_TS = "ts" + _OPTION_KEYS = [ "custom_kvs", "signature", @@ -58,14 +62,16 @@ def __init__(self, continue option_key = option_kv[0].strip() - if option_key == "trigger-trace": + if option_key == self._XTRACEOPTIONS_HEADER_KEY_TRIGGER_TRACE: if len(option_kv) > 1: logger.debug("trigger-trace must be standalone flag. Ignoring.") - self.ignored.append("trigger-trace") + self.ignored.append( + self._XTRACEOPTIONS_HEADER_KEY_TRIGGER_TRACE + ) else: self.trigger_trace = True - elif option_key == "sw-keys": + elif option_key == self._XTRACEOPTIONS_HEADER_KEY_SW_KEYS: # each of sw-keys KVs delimited by comma sw_kvs = re.split(r",+", option_kv[1]) for assignment in sw_kvs: @@ -83,12 +89,12 @@ def __init__(self, elif re.match(self._XTRACEOPTIONS_CUSTOM_RE, option_key): self.custom_kvs[option_key] = option_kv[1].strip() - elif option_key == "ts": + elif option_key == self._XTRACEOPTIONS_HEADER_KEY_TS: try: self.ts = int(option_kv[1]) except ValueError as e: logger.debug("ts must be base 10 int. Ignoring.") - self.ignore.append("ts") + self.ignore.append(self._XTRACEOPTIONS_HEADER_KEY_TS) else: logger.debug( @@ -115,20 +121,29 @@ def to_options_header(self) -> str: options = [] if self.trigger_trace: - options.append("trigger-trace") + options.append(self._XTRACEOPTIONS_HEADER_KEY_TRIGGER_TRACE) if len(self.sw_keys) > 0: sw_keys = [] for _, (k, v) in enumerate(self.sw_keys.items()): sw_keys.append(":".join([k, v])) - options.append("=".join(["sw-keys", ",".join(sw_keys)])) + options.append( + "=".join([ + self._XTRACEOPTIONS_HEADER_KEY_SW_KEYS, + ",".join(sw_keys) + ]) + ) if len(self.custom_kvs) > 0: for _, (k, v) in enumerate(self.custom_kvs.items()): options.append("=".join([k, v])) if self.ts > 0: - options.append("=".join(["ts", str(self.ts)])) + options.append( + "=".join([ + self._XTRACEOPTIONS_HEADER_KEY_TS, str(self.ts) + ]) + ) return ";".join(options) From 552398ce9ba34aeffb4b39990dc08fac28e1c2c2 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 11 Apr 2022 11:58:12 -0700 Subject: [PATCH 49/71] Fix sw.tracestate_parent_id, sw.w3c.tracestate KV/attribute creation --- opentelemetry_distro_solarwinds/sampler.py | 19 +++++++++++++++---- .../w3c_transformer.py | 5 +++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 54358a76..35cdd763 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -314,6 +314,13 @@ def calculate_trace_state( logger.debug("Updated trace_state: {}".format(trace_state)) return trace_state + def remove_response_from_sw( + self, + trace_state: TraceState + ) -> TraceState: + """Remove xtraceoptions response from tracestate""" + return trace_state.delete(XTraceOptions.get_sw_xtraceoptions_response_key()) + def calculate_attributes( self, attributes: Attributes, @@ -339,13 +346,15 @@ def calculate_attributes( # Set attributes with self._SW_TRACESTATE_ROOT_KEY and self._SW_TRACESTATE_CAPTURE_KEY if not attributes: + trace_state_no_response = self.remove_response_from_sw(trace_state) attributes = { - self._SW_TRACESTATE_CAPTURE_KEY: trace_state.to_header() + self._SW_TRACESTATE_CAPTURE_KEY: trace_state_no_response.to_header() } # Only set self._SW_TRACESTATE_ROOT_KEY on the entry (root) span for this service sw_value = parent_span_context.trace_state.get(SW_TRACESTATE_KEY, None) if sw_value: - attributes[self._SW_TRACESTATE_ROOT_KEY] = sw_value + attributes[self._SW_TRACESTATE_ROOT_KEY] \ + = W3CTransformer.span_id_from_sw(sw_value) logger.debug("Created new attributes: {}".format(attributes)) else: @@ -357,7 +366,8 @@ def calculate_attributes( if not new_attributes.get(self._SW_TRACESTATE_CAPTURE_KEY, None): # Add new self._SW_TRACESTATE_CAPTURE_KEY KV - new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = trace_state.to_header() + trace_state_no_response = self.remove_response_from_sw(trace_state) + new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = trace_state_no_response.to_header() else: # Update existing self._SW_TRACESTATE_CAPTURE_KEY KV attr_trace_state = TraceState.from_header( @@ -370,7 +380,8 @@ def calculate_attributes( W3CTransformer.trace_flags_from_int(decision.do_sample) ) ) - new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = attr_trace_state.to_header() + trace_state_no_response = self.remove_response_from_sw(attr_trace_state) + new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = trace_state_no_response.to_header() attributes = new_attributes logger.debug("Set updated attributes: {}".format(attributes)) diff --git a/opentelemetry_distro_solarwinds/w3c_transformer.py b/opentelemetry_distro_solarwinds/w3c_transformer.py index 22229f7c..822065ce 100644 --- a/opentelemetry_distro_solarwinds/w3c_transformer.py +++ b/opentelemetry_distro_solarwinds/w3c_transformer.py @@ -20,6 +20,11 @@ def span_id_from_int(cls, span_id: int) -> str: """Formats span ID as 16-byte hexadecimal string""" return cls._SPAN_ID_HEX.format(span_id) + @classmethod + def span_id_from_sw(cls, sw: str) -> str: + """Formats span ID from sw tracestate value""" + return sw.split("-")[0] + @classmethod def trace_flags_from_int(cls, trace_flags: int) -> str: """Formats trace flags as 8-bit field""" From 58acd70dbde8406c3fce3035d74281733323e8fa Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 11 Apr 2022 14:20:19 -0700 Subject: [PATCH 50/71] Rm _LiboboeDecision class to use dict instead --- opentelemetry_distro_solarwinds/sampler.py | 117 ++++++--------------- 1 file changed, 32 insertions(+), 85 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 35cdd763..21419832 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -5,7 +5,7 @@ import logging from types import MappingProxyType -from typing import Optional, Sequence +from typing import Optional, Sequence, Dict from opentelemetry.context.context import Context as OtelContext from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, @@ -26,35 +26,6 @@ logger = logging.getLogger(__name__) -class _LiboboeDecision(): - """Convenience representation of a liboboe decision""" - def __init__( - self, - do_metrics: int, - do_sample: int, - sample_rate: int, - sample_source: int, - bucket_rate: float, - bucket_cap: float, - decision_type: int, - auth: int, - status_msg: str, - auth_msg: str, - status: int, - ): - self.do_metrics = do_metrics - self.do_sample = do_sample - self.sample_rate = sample_rate - self.sample_source = sample_source - self.bucket_rate = bucket_rate - self.bucket_cap = bucket_cap - self.decision_type = decision_type - self.auth = auth - self.status_msg = status_msg - self.auth_msg = auth_msg - self.status = status - - class _SwSampler(Sampler): """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" @@ -74,7 +45,7 @@ def calculate_liboboe_decision( parent_span_context: SpanContext, parent_context: Optional[OtelContext] = None, xtraceoptions: Optional[XTraceOptions] = None, - ) -> _LiboboeDecision: + ) -> Dict: """Calculates oboe trace decision based on parent span context.""" tracestring = None if parent_span_context.is_valid and parent_span_context.is_remote: @@ -135,63 +106,39 @@ def calculate_liboboe_decision( signature, timestamp ) - logger.debug( - "Got liboboe decision outputs " - "do_metrics: {}, " - "do_sample: {}, " - "rate: {}, " - "source: {}, " - "bucket_rate: {}, " - "bucket_cap: {}, " - "type: {}, " - "auth: {}, " - "status_msg: {}, " - "auth_msg: {}, " - "status: {}".format( - do_metrics, - do_sample, - rate, - source, - bucket_rate, - bucket_cap, - decision_type, - auth, - status_msg, - auth_msg, - status - ) - ) - return _LiboboeDecision( - do_metrics, - do_sample, - rate, - source, - bucket_rate, - bucket_cap, - decision_type, - auth, - status_msg, - auth_msg, - status - ) + decision = { + "do_metrics": do_metrics, + "do_sample": do_sample, + "rate": rate, + "source": source, + "bucket_rate": bucket_rate, + "bucket_cap": bucket_cap, + "decision_type": decision_type, + "auth": auth, + "status_msg": status_msg, + "auth_msg": auth_msg, + "status": status, + } + logger.debug("Got liboboe decision outputs: {}".format(decision)) + return decision def otel_decision_from_liboboe( self, - liboboe_decision: _LiboboeDecision + liboboe_decision: Dict ) -> Decision: """Formats OTel decision from liboboe decision""" decision = Decision.DROP - if liboboe_decision.do_metrics: + if liboboe_decision["do_metrics"]: # TODO: need to eck what record only actually means and how metrics work in OTel decision = Decision.RECORD_ONLY - if liboboe_decision.do_sample: + if liboboe_decision["do_sample"]: decision = Decision.RECORD_AND_SAMPLE logger.debug("OTel decision created: {}".format(decision)) return decision def create_xtraceoptions_response_value( self, - decision: _LiboboeDecision, + decision: Dict, parent_span_context: SpanContext, xtraceoptions: XTraceOptions ) -> str: @@ -205,10 +152,10 @@ def create_xtraceoptions_response_value( if xtraceoptions.signature: response.append(EQUALS_W3C_SANITIZED.join([ self._XTRACEOPTIONS_RESP_AUTH, - decision.auth_msg + decision["auth_msg"] ])) - if not decision.auth or decision.auth < 1: + if not decision["auth"] or decision["auth"] < 1: trigger_msg = "" if xtraceoptions.trigger_trace: # If a traceparent header was provided then oboe does not generate the message @@ -218,10 +165,10 @@ def create_xtraceoptions_response_value( parent_span_context ) - if tracestring and decision.decision_type == 0: + if tracestring and decision["decision_type"] == 0: trigger_msg = self._XTRACEOPTIONS_RESP_TRIGGER_IGNORED else: - trigger_msg = decision.status_msg + trigger_msg = decision["status_msg"] else: trigger_msg = self.XTRACEOPTIONS_TRIGGER_NOT_REQUESTED response.append(EQUALS_W3C_SANITIZED.join([ @@ -233,7 +180,7 @@ def create_xtraceoptions_response_value( response.append( EQUALS_W3C_SANITIZED.join([ self._XTRACEOPTIONS_RESP_IGNORED, - (COMMA_W3C_SANITIZED.join(decision.ignored)) + (COMMA_W3C_SANITIZED.join(decision["ignored"])) ]) ) @@ -241,7 +188,7 @@ def create_xtraceoptions_response_value( def create_new_trace_state( self, - decision: _LiboboeDecision, + decision: Dict, parent_span_context: SpanContext, xtraceoptions: Optional[XTraceOptions] = None, ) -> TraceState: @@ -251,7 +198,7 @@ def create_new_trace_state( SW_TRACESTATE_KEY, W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, - W3CTransformer.trace_flags_from_int(decision.do_sample) + W3CTransformer.trace_flags_from_int(decision["do_sample"]) ) )]) if xtraceoptions and xtraceoptions.trigger_trace: @@ -268,7 +215,7 @@ def create_new_trace_state( def calculate_trace_state( self, - decision: _LiboboeDecision, + decision: Dict, parent_span_context: SpanContext, xtraceoptions: Optional[XTraceOptions] = None, ) -> TraceState: @@ -297,7 +244,7 @@ def calculate_trace_state( SW_TRACESTATE_KEY, W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, - W3CTransformer.trace_flags_from_int(decision.do_sample) + W3CTransformer.trace_flags_from_int(decision["do_sample"]) ) ) # Update trace_state with x-trace-options-response @@ -324,7 +271,7 @@ def remove_response_from_sw( def calculate_attributes( self, attributes: Attributes, - decision: _LiboboeDecision, + decision: Dict, trace_state: TraceState, parent_span_context: SpanContext ) -> Attributes or None: @@ -377,7 +324,7 @@ def calculate_attributes( SW_TRACESTATE_KEY, W3CTransformer.sw_from_span_and_decision( parent_span_context.span_id, - W3CTransformer.trace_flags_from_int(decision.do_sample) + W3CTransformer.trace_flags_from_int(decision["do_sample"]) ) ) trace_state_no_response = self.remove_response_from_sw(attr_trace_state) From 783cac0f4c807e6529eb67d0fdc93bc31746afa2 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 11 Apr 2022 15:32:32 -0700 Subject: [PATCH 51/71] distro set_global_textmap hardcode instead of env var for OTEL_PROPAGATORS --- opentelemetry_distro_solarwinds/distro.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/opentelemetry_distro_solarwinds/distro.py b/opentelemetry_distro_solarwinds/distro.py index 3e28bc7b..628e9a42 100644 --- a/opentelemetry_distro_solarwinds/distro.py +++ b/opentelemetry_distro_solarwinds/distro.py @@ -1,12 +1,19 @@ """Module to configure OpenTelemetry agent to work with SolarWinds backend""" from opentelemetry import trace +from opentelemetry.baggage.propagation import W3CBaggagePropagator from opentelemetry.instrumentation.distro import BaseDistro from opentelemetry.instrumentation.propagators import set_global_response_propagator +from opentelemetry.propagate import set_global_textmap +from opentelemetry.propagators.composite import CompositePropagator from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, +) from opentelemetry_distro_solarwinds.exporter import SolarWindsSpanExporter +from opentelemetry_distro_solarwinds.propagator import SolarWindsPropagator from opentelemetry_distro_solarwinds.response_propagator import SolarWindsTraceResponsePropagator from opentelemetry_distro_solarwinds.sampler import ParentBasedSwSampler @@ -24,5 +31,13 @@ def _configure(self, **kwargs): # Automatically configure the SolarWinds Span exporter span_exporter = BatchSpanProcessor(SolarWindsSpanExporter()) trace.get_tracer_provider().add_span_processor(span_exporter) + # Configure a CompositePropagator including SolarWinds + set_global_textmap( + CompositePropagator([ + TraceContextTextMapPropagator(), + W3CBaggagePropagator(), + SolarWindsPropagator() + ]) + ) # Set global HTTP response propagator set_global_response_propagator(SolarWindsTraceResponsePropagator()) \ No newline at end of file From b58e29be87f42f8ce699da7c5a7a7c05645a88aa Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 11 Apr 2022 16:06:31 -0700 Subject: [PATCH 52/71] traceparent_from_context returns lowercase --- opentelemetry_distro_solarwinds/w3c_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry_distro_solarwinds/w3c_transformer.py b/opentelemetry_distro_solarwinds/w3c_transformer.py index 822065ce..4907f97c 100644 --- a/opentelemetry_distro_solarwinds/w3c_transformer.py +++ b/opentelemetry_distro_solarwinds/w3c_transformer.py @@ -44,7 +44,7 @@ def traceparent_from_context(cls, span_context: Context) -> str: span_context.trace_id, span_context.span_id, span_context.trace_flags - ).upper() + ) logger.debug("Generated traceparent {} from {}".format(xtr, span_context)) return xtr From b08517ec834387e1d6d4bbfd41f71c658e70e331 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 11 Apr 2022 16:54:18 -0700 Subject: [PATCH 53/71] Fixups: variable assignments, rm ununsed param --- opentelemetry_distro_solarwinds/sampler.py | 23 ++++++++++------------ 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 21419832..dd8e89b3 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -43,7 +43,6 @@ def get_description(self) -> str: def calculate_liboboe_decision( self, parent_span_context: SpanContext, - parent_context: Optional[OtelContext] = None, xtraceoptions: Optional[XTraceOptions] = None, ) -> Dict: """Calculates oboe trace decision based on parent span context.""" @@ -59,7 +58,6 @@ def calculate_liboboe_decision( sample_rate = -1 trigger_tracing_mode_disabled = -1 - logger.debug("parent_context is {}".format(parent_context)) logger.debug("xtraceoptions is {}".format(dict(xtraceoptions))) options = None @@ -355,30 +353,29 @@ def should_sample( ).get_span_context() xtraceoptions = XTraceOptions(parent_context) - decision = self.calculate_liboboe_decision( + liboboe_decision = self.calculate_liboboe_decision( parent_span_context, - parent_context, xtraceoptions ) # Always calculate trace_state for propagation - trace_state = self.calculate_trace_state( - decision, + new_trace_state = self.calculate_trace_state( + liboboe_decision, parent_span_context, xtraceoptions ) - attributes = self.calculate_attributes( + new_attributes = self.calculate_attributes( attributes, - decision, - trace_state, + liboboe_decision, + new_trace_state, parent_span_context ) - decision = self.otel_decision_from_liboboe(decision) + otel_decision = self.otel_decision_from_liboboe(liboboe_decision) return SamplingResult( - decision, - attributes if decision != Decision.DROP else None, - trace_state, + otel_decision, + new_attributes if otel_decision != Decision.DROP else None, + new_trace_state, ) From f7f487a635fa737583e74a90b9e45aa9d4f209ef Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Tue, 12 Apr 2022 15:09:25 -0700 Subject: [PATCH 54/71] Fix comment --- opentelemetry_distro_solarwinds/w3c_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry_distro_solarwinds/w3c_transformer.py b/opentelemetry_distro_solarwinds/w3c_transformer.py index 4907f97c..9859ac17 100644 --- a/opentelemetry_distro_solarwinds/w3c_transformer.py +++ b/opentelemetry_distro_solarwinds/w3c_transformer.py @@ -32,7 +32,7 @@ def trace_flags_from_int(cls, trace_flags: int) -> str: @classmethod def traceparent_from_context(cls, span_context: Context) -> str: - """Generates an uppercase liboboe W3C compatible trace_context from + """Generates a liboboe W3C compatible trace_context from provided OTel span context.""" template = "-".join([ cls._VERSION, From 3aad13d351a61e2c425081394516af53a47ffcfe Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 13 Apr 2022 17:15:52 -0700 Subject: [PATCH 55/71] NH-11236 sampler creates Service Entry Internal KVs --- opentelemetry_distro_solarwinds/sampler.py | 35 +++++++++++++++------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index dd8e89b3..6e32cc47 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -29,6 +29,10 @@ class _SwSampler(Sampler): """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" + _INTERNAL_BUCKET_CAPACITY = "BucketCapacity" + _INTERNAL_BUCKET_RATE = "BucketRate" + _INTERNAL_SAMPLE_RATE = "SampleRate" + _INTERNAL_SAMPLE_SOURCE = "SampleSource" _SW_TRACESTATE_CAPTURE_KEY = "sw.w3c.tracestate" _SW_TRACESTATE_ROOT_KEY = "sw.tracestate_parent_id" _XTRACEOPTIONS_RESP_AUTH = "auth" @@ -283,25 +287,38 @@ def calculate_attributes( if self.otel_decision_from_liboboe(decision) == Decision.DROP: logger.debug("Trace decision is to drop - not setting attributes") return None + + new_attributes = {} + # Trace's root span has no valid traceparent nor tracestate - # so we don't set additional attributes + # so set service entry internal KVs here and only here if not parent_span_context.is_valid or not trace_state: - logger.debug("No valid traceparent or no tracestate - not setting attributes") - return None + new_attributes = { + self._INTERNAL_BUCKET_CAPACITY: "{}".format(decision["bucket_cap"]), + self._INTERNAL_BUCKET_RATE: "{}".format(decision["bucket_rate"]), + self._INTERNAL_SAMPLE_RATE: decision["rate"], + self._INTERNAL_SAMPLE_SOURCE: decision["source"] + } + logger.debug( + "No valid traceparent or no tracestate - only setting " + "service entry internal KVs attributes: {}".format(new_attributes) + ) + # attributes must be immutable for SamplingResult + return MappingProxyType(new_attributes) # Set attributes with self._SW_TRACESTATE_ROOT_KEY and self._SW_TRACESTATE_CAPTURE_KEY if not attributes: trace_state_no_response = self.remove_response_from_sw(trace_state) - attributes = { + new_attributes = { self._SW_TRACESTATE_CAPTURE_KEY: trace_state_no_response.to_header() } # Only set self._SW_TRACESTATE_ROOT_KEY on the entry (root) span for this service sw_value = parent_span_context.trace_state.get(SW_TRACESTATE_KEY, None) if sw_value: - attributes[self._SW_TRACESTATE_ROOT_KEY] \ + new_attributes[self._SW_TRACESTATE_ROOT_KEY] \ = W3CTransformer.span_id_from_sw(sw_value) - logger.debug("Created new attributes: {}".format(attributes)) + logger.debug("Created new attributes: {}".format(new_attributes)) else: # Copy existing MappingProxyType KV into new_attributes for modification # attributes may have other vendor info etc @@ -327,12 +344,10 @@ def calculate_attributes( ) trace_state_no_response = self.remove_response_from_sw(attr_trace_state) new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = trace_state_no_response.to_header() + logger.debug("Set updated attributes: {}".format(new_attributes)) - attributes = new_attributes - logger.debug("Set updated attributes: {}".format(attributes)) - # attributes must be immutable for SamplingResult - return MappingProxyType(attributes) + return MappingProxyType(new_attributes) def should_sample( self, From ffccf28aeb51328ff7a79d1811249eb72d776a76 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 14 Apr 2022 10:35:48 -0700 Subject: [PATCH 56/71] SWKeys KV added to root span, sw_keys always str, trigger_trace as int not bool --- opentelemetry_distro_solarwinds/sampler.py | 22 +++++++++---- .../traceoptions.py | 31 +++++++------------ 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index dd8e89b3..92171742 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -29,6 +29,7 @@ class _SwSampler(Sampler): """SolarWinds custom opentelemetry sampler which obeys configuration options provided by the NH/AO Backend.""" + _INTERNAL_SW_KEYS = "SWKeys" _SW_TRACESTATE_CAPTURE_KEY = "sw.w3c.tracestate" _SW_TRACESTATE_ROOT_KEY = "sw.tracestate_parent_id" _XTRACEOPTIONS_RESP_AUTH = "auth" @@ -65,7 +66,7 @@ def calculate_liboboe_decision( signature = None timestamp = None if xtraceoptions: - options = xtraceoptions.to_options_header() + options = xtraceoptions.options_header trigger_trace = xtraceoptions.trigger_trace signature = xtraceoptions.signature timestamp = xtraceoptions.ts @@ -271,7 +272,8 @@ def calculate_attributes( attributes: Attributes, decision: Dict, trace_state: TraceState, - parent_span_context: SpanContext + parent_span_context: SpanContext, + xtraceoptions: XTraceOptions ) -> Attributes or None: """Calculates Attributes or None based on trace decision, trace state, parent span context, and existing attributes. @@ -284,10 +286,17 @@ def calculate_attributes( logger.debug("Trace decision is to drop - not setting attributes") return None # Trace's root span has no valid traceparent nor tracestate - # so we don't set additional attributes + # so set SWKeys and service entry internal KVs here and only here if not parent_span_context.is_valid or not trace_state: - logger.debug("No valid traceparent or no tracestate - not setting attributes") - return None + new_attributes = { + self._INTERNAL_SW_KEYS: "{}".format(xtraceoptions.sw_keys), + } + logger.debug( + "No valid traceparent or no tracestate - only setting " + "service entry internal KVs attributes: {}".format(new_attributes) + ) + # attributes must be immutable for SamplingResult + return MappingProxyType(new_attributes) # Set attributes with self._SW_TRACESTATE_ROOT_KEY and self._SW_TRACESTATE_CAPTURE_KEY if not attributes: @@ -368,7 +377,8 @@ def should_sample( attributes, liboboe_decision, new_trace_state, - parent_span_context + parent_span_context, + xtraceoptions ) otel_decision = self.otel_decision_from_liboboe(liboboe_decision) diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index 9f044c52..5ad95f5d 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -44,15 +44,19 @@ def __init__(self, """ self.custom_kvs = {} self.signature = None - self.sw_keys = {} - self.trigger_trace = False + self.sw_keys = "" + self.trigger_trace = 0 self.ts = 0 self.ignored = [] + self.options_header = "" if not options_header: self.from_context(context) return + # store original header for sample decision later + self.options_header = options_header + # each of options delimited by semicolon traceoptions = re.split(r";+", options_header) for option in traceoptions: @@ -69,22 +73,10 @@ def __init__(self, self._XTRACEOPTIONS_HEADER_KEY_TRIGGER_TRACE ) else: - self.trigger_trace = True + self.trigger_trace = 1 elif option_key == self._XTRACEOPTIONS_HEADER_KEY_SW_KEYS: - # each of sw-keys KVs delimited by comma - sw_kvs = re.split(r",+", option_kv[1]) - for assignment in sw_kvs: - # each of sw-keys values assigned by colon - sw_kv = assignment.split(":", 2) - if not sw_kv[0]: - logger.debug( - "Could not parse sw-key assignment {}. Ignoring.".format( - assignment - )) - self.ignore.append(assignment) - else: - self.sw_keys.update({sw_kv[0]: sw_kv[1]}) + self.sw_keys = option_kv[1].strip() elif re.match(self._XTRACEOPTIONS_CUSTOM_RE, option_key): self.custom_kvs[option_key] = option_kv[1].strip() @@ -124,13 +116,10 @@ def to_options_header(self) -> str: options.append(self._XTRACEOPTIONS_HEADER_KEY_TRIGGER_TRACE) if len(self.sw_keys) > 0: - sw_keys = [] - for _, (k, v) in enumerate(self.sw_keys.items()): - sw_keys.append(":".join([k, v])) options.append( "=".join([ self._XTRACEOPTIONS_HEADER_KEY_SW_KEYS, - ",".join(sw_keys) + self.sw_keys ]) ) @@ -161,6 +150,8 @@ def from_context( for option_key in self._OPTION_KEYS: if context.get(option_key, None): setattr(self, option_key, context[option_key]) + # store header for sample decision + self.options_header = self.to_options_header() @classmethod def get_sw_xtraceoptions_response_key(cls) -> str: From 3a90bdb1d1d1d3c4c16acf35d7ba9f9782b1013f Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 18 Apr 2022 11:53:59 -0700 Subject: [PATCH 57/71] Simplify x-trace-options handling --- opentelemetry_distro_solarwinds/__init__.py | 4 +- opentelemetry_distro_solarwinds/propagator.py | 33 ++++--- opentelemetry_distro_solarwinds/sampler.py | 2 - .../traceoptions.py | 96 ++++--------------- 4 files changed, 41 insertions(+), 94 deletions(-) diff --git a/opentelemetry_distro_solarwinds/__init__.py b/opentelemetry_distro_solarwinds/__init__.py index d805f60a..f5341bd4 100644 --- a/opentelemetry_distro_solarwinds/__init__.py +++ b/opentelemetry_distro_solarwinds/__init__.py @@ -4,4 +4,6 @@ COMMA_W3C_SANITIZED = "...." EQUALS = "=" EQUALS_W3C_SANITIZED = "####" -SW_TRACESTATE_KEY = "sw" \ No newline at end of file +SW_TRACESTATE_KEY = "sw" +OTEL_CONTEXT_SW_OPTIONS_KEY = "sw_xtraceoptions" +OTEL_CONTEXT_SW_SIGNATURE_KEY = "sw_signature" \ No newline at end of file diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index 4baf0676..b11d50ee 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -6,7 +6,11 @@ from opentelemetry.propagators import textmap from opentelemetry.trace.span import TraceState -from opentelemetry_distro_solarwinds import SW_TRACESTATE_KEY +from opentelemetry_distro_solarwinds import ( + OTEL_CONTEXT_SW_OPTIONS_KEY, + OTEL_CONTEXT_SW_SIGNATURE_KEY, + SW_TRACESTATE_KEY +) from opentelemetry_distro_solarwinds.traceoptions import XTraceOptions from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer @@ -39,7 +43,13 @@ def extract( if not xtraceoptions_header: logger.debug("No xtraceoptions to extract; ignoring signature") return context - logger.debug("Extracted xtraceoptions_header: {}".format( + + context.update({ + OTEL_CONTEXT_SW_OPTIONS_KEY: xtraceoptions_header[0] + }) + logger.debug("Extracted {} as {}: {}".format( + self._XTRACEOPTIONS_HEADER_NAME, + OTEL_CONTEXT_SW_OPTIONS_KEY, xtraceoptions_header[0] )) @@ -48,17 +58,14 @@ def extract( self._XTRACEOPTIONS_SIGNATURE_HEADER_NAME ) if signature_header: - xtraceoptions = XTraceOptions( - context, - xtraceoptions_header[0], - signature_header[0], - ) - else: - xtraceoptions = XTraceOptions( - context, - xtraceoptions_header[0], - ) - context.update(dict(xtraceoptions)) + context.update({ + OTEL_CONTEXT_SW_SIGNATURE_KEY: signature_header[0] + }) + logger.debug("Extracted {} as {}: {}".format( + self._XTRACEOPTIONS_SIGNATURE_HEADER_NAME, + OTEL_CONTEXT_SW_SIGNATURE_KEY, + xtraceoptions_header[0] + )) return context def inject( diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 92171742..03e5835c 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -59,8 +59,6 @@ def calculate_liboboe_decision( sample_rate = -1 trigger_tracing_mode_disabled = -1 - logger.debug("xtraceoptions is {}".format(dict(xtraceoptions))) - options = None trigger_trace = 0 signature = None diff --git a/opentelemetry_distro_solarwinds/traceoptions.py b/opentelemetry_distro_solarwinds/traceoptions.py index 5ad95f5d..6d04a101 100644 --- a/opentelemetry_distro_solarwinds/traceoptions.py +++ b/opentelemetry_distro_solarwinds/traceoptions.py @@ -4,10 +4,15 @@ from opentelemetry.context.context import Context +from opentelemetry_distro_solarwinds import ( + OTEL_CONTEXT_SW_OPTIONS_KEY, + OTEL_CONTEXT_SW_SIGNATURE_KEY +) + logger = logging.getLogger(__file__) class XTraceOptions(): - """Formats X-Trace-Options for trigger tracing""" + """Formats X-Trace-Options and signature for trigger tracing""" _SW_XTRACEOPTIONS_RESPONSE_KEY = "xtrace_options_response" _XTRACEOPTIONS_CUSTOM = ("^custom-[^\s]*$") @@ -17,47 +22,29 @@ class XTraceOptions(): _XTRACEOPTIONS_HEADER_KEY_TRIGGER_TRACE = "trigger-trace" _XTRACEOPTIONS_HEADER_KEY_TS = "ts" - _OPTION_KEYS = [ - "custom_kvs", - "signature", - "sw_keys", - "trigger_trace", - "ts", - "ignored" - ] - def __init__(self, context: typing.Optional[Context] = None, - options_header: str = None, - signature_header: str = None ): """ Args: - context: OTel context that may contain x-trace-options - options_header: A string of x-trace-options - signature_header: A string required for signed trigger trace - - Examples of options_header: - "trigger-trace" - "trigger-trace;sw-keys=check-id:check-1013,website-id:booking-demo" - "trigger-trace;custom-key1=value1" + context: OTel context that may contain OTEL_CONTEXT_SW_OPTIONS_KEY,OTEL_CONTEXT_SW_SIGNATURE_KEY """ - self.custom_kvs = {} + self.ignored = [] + self.options_header = "" self.signature = None self.sw_keys = "" self.trigger_trace = 0 self.ts = 0 - self.ignored = [] - self.options_header = "" - + + if not context: + return + options_header = context.get(OTEL_CONTEXT_SW_OPTIONS_KEY, None) if not options_header: - self.from_context(context) return # store original header for sample decision later self.options_header = options_header - # each of options delimited by semicolon traceoptions = re.split(r";+", options_header) for option in traceoptions: # KVs (e.g. sw-keys or custom-key1) are assigned by equals @@ -79,7 +66,8 @@ def __init__(self, self.sw_keys = option_kv[1].strip() elif re.match(self._XTRACEOPTIONS_CUSTOM_RE, option_key): - self.custom_kvs[option_key] = option_kv[1].strip() + # custom keys are valid but do not need parsing + pass elif option_key == self._XTRACEOPTIONS_HEADER_KEY_TS: try: @@ -101,57 +89,9 @@ def __init__(self, ", ".join(self.ignored) )) - if signature_header: - self.signature = signature_header - - def __iter__(self) -> typing.Iterator: - """Iterable representation of XTraceOptions""" - yield from self.__dict__.items() - - def to_options_header(self) -> str: - """String representation of XTraceOptions, without signature.""" - options = [] - - if self.trigger_trace: - options.append(self._XTRACEOPTIONS_HEADER_KEY_TRIGGER_TRACE) - - if len(self.sw_keys) > 0: - options.append( - "=".join([ - self._XTRACEOPTIONS_HEADER_KEY_SW_KEYS, - self.sw_keys - ]) - ) - - if len(self.custom_kvs) > 0: - for _, (k, v) in enumerate(self.custom_kvs.items()): - options.append("=".join([k, v])) - - if self.ts > 0: - options.append( - "=".join([ - self._XTRACEOPTIONS_HEADER_KEY_TS, str(self.ts) - ]) - ) - - return ";".join(options) - - def from_context( - self, - context: typing.Optional[Context] - ) -> None: - """ - Args: - context: OTel context that may contain x-trace-options - """ - logger.debug("Setting XTraceOptions from_context with {}".format(context)) - if not context: - return - for option_key in self._OPTION_KEYS: - if context.get(option_key, None): - setattr(self, option_key, context[option_key]) - # store header for sample decision - self.options_header = self.to_options_header() + options_signature = context.get(OTEL_CONTEXT_SW_SIGNATURE_KEY, None) + if options_signature: + self.signature = options_signature @classmethod def get_sw_xtraceoptions_response_key(cls) -> str: From 118c1bceb54fab2109801731b9ab7f4fd6229486 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 18 Apr 2022 13:43:56 -0700 Subject: [PATCH 58/71] Sampler always sets SWKeys KV if available; calculate_attributes refactor --- opentelemetry_distro_solarwinds/sampler.py | 76 ++++++++++------------ 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 03e5835c..b460e7a3 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -274,72 +274,62 @@ def calculate_attributes( xtraceoptions: XTraceOptions ) -> Attributes or None: """Calculates Attributes or None based on trace decision, trace state, - parent span context, and existing attributes. - - For non-existent or remote parent spans only.""" + parent span context, xtraceoptions, and existing attributes.""" logger.debug("Received attributes: {}".format(attributes)) - # Don't set attributes if not tracing if self.otel_decision_from_liboboe(decision) == Decision.DROP: logger.debug("Trace decision is to drop - not setting attributes") return None + + new_attributes = {} + # Always (root or is_remote) set _INTERNAL_SW_KEYS if injected + internal_sw_keys = xtraceoptions.sw_keys + if internal_sw_keys: + new_attributes[self._INTERNAL_SW_KEYS] = internal_sw_keys + # Trace's root span has no valid traceparent nor tracestate - # so set SWKeys and service entry internal KVs here and only here + # so we can't calculate remaining attributes if not parent_span_context.is_valid or not trace_state: - new_attributes = { - self._INTERNAL_SW_KEYS: "{}".format(xtraceoptions.sw_keys), - } logger.debug( - "No valid traceparent or no tracestate - only setting " - "service entry internal KVs attributes: {}".format(new_attributes) + "No valid traceparent or no tracestate - returning attributes: {}" + .format(new_attributes) ) # attributes must be immutable for SamplingResult return MappingProxyType(new_attributes) - # Set attributes with self._SW_TRACESTATE_ROOT_KEY and self._SW_TRACESTATE_CAPTURE_KEY if not attributes: - trace_state_no_response = self.remove_response_from_sw(trace_state) - attributes = { - self._SW_TRACESTATE_CAPTURE_KEY: trace_state_no_response.to_header() - } - # Only set self._SW_TRACESTATE_ROOT_KEY on the entry (root) span for this service + # _SW_TRACESTATE_ROOT_KEY is set once per trace, if possible sw_value = parent_span_context.trace_state.get(SW_TRACESTATE_KEY, None) if sw_value: - attributes[self._SW_TRACESTATE_ROOT_KEY] \ - = W3CTransformer.span_id_from_sw(sw_value) - - logger.debug("Created new attributes: {}".format(attributes)) + new_attributes[self._SW_TRACESTATE_ROOT_KEY]= W3CTransformer.span_id_from_sw(sw_value) else: - # Copy existing MappingProxyType KV into new_attributes for modification + # Copy existing MappingProxyType KV into new_attributes for modification. # attributes may have other vendor info etc - new_attributes = {} for k,v in attributes.items(): new_attributes[k] = v - if not new_attributes.get(self._SW_TRACESTATE_CAPTURE_KEY, None): - # Add new self._SW_TRACESTATE_CAPTURE_KEY KV - trace_state_no_response = self.remove_response_from_sw(trace_state) - new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = trace_state_no_response.to_header() - else: - # Update existing self._SW_TRACESTATE_CAPTURE_KEY KV - attr_trace_state = TraceState.from_header( - new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] - ) - attr_trace_state.update( - SW_TRACESTATE_KEY, - W3CTransformer.sw_from_span_and_decision( - parent_span_context.span_id, - W3CTransformer.trace_flags_from_int(decision["do_sample"]) - ) + tracestate_capture = new_attributes.get(self._SW_TRACESTATE_CAPTURE_KEY, None) + if not tracestate_capture: + trace_state_no_response = self.remove_response_from_sw(trace_state) + else: + # Must retain all potential tracestate pairs for attributes + attr_trace_state = TraceState.from_header( + tracestate_capture + ) + new_attr_trace_state = attr_trace_state.update( + SW_TRACESTATE_KEY, + W3CTransformer.sw_from_span_and_decision( + parent_span_context.span_id, + W3CTransformer.trace_flags_from_int(decision["do_sample"]) ) - trace_state_no_response = self.remove_response_from_sw(attr_trace_state) - new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = trace_state_no_response.to_header() + ) + trace_state_no_response = self.remove_response_from_sw(new_attr_trace_state) + new_attributes[self._SW_TRACESTATE_CAPTURE_KEY] = trace_state_no_response.to_header() + + logger.debug("Setting attributes: {}".format(new_attributes)) - attributes = new_attributes - logger.debug("Set updated attributes: {}".format(attributes)) - # attributes must be immutable for SamplingResult - return MappingProxyType(attributes) + return MappingProxyType(new_attributes) def should_sample( self, From 23a12a255f8d0e159b5726cc3da7bb710756b216 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Mon, 18 Apr 2022 15:12:20 -0700 Subject: [PATCH 59/71] Sampler sets bucket/sample KVs if liboboe decision not cont'd --- opentelemetry_distro_solarwinds/sampler.py | 50 ++++++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index 6e32cc47..8238034d 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -33,6 +33,7 @@ class _SwSampler(Sampler): _INTERNAL_BUCKET_RATE = "BucketRate" _INTERNAL_SAMPLE_RATE = "SampleRate" _INTERNAL_SAMPLE_SOURCE = "SampleSource" + _LIBOBOE_CONTINUED = -1 _SW_TRACESTATE_CAPTURE_KEY = "sw.w3c.tracestate" _SW_TRACESTATE_ROOT_KEY = "sw.tracestate_parent_id" _XTRACEOPTIONS_RESP_AUTH = "auth" @@ -124,6 +125,21 @@ def calculate_liboboe_decision( logger.debug("Got liboboe decision outputs: {}".format(decision)) return decision + def is_decision_continued( + self, + liboboe_decision: Dict + ) -> bool: + """Returns True if liboboe decision is a continued one, else False""" + for val in [ + liboboe_decision["rate"], + liboboe_decision["source"], + liboboe_decision["bucket_rate"], + liboboe_decision["bucket_cap"] + ]: + if val != self._LIBOBOE_CONTINUED: + return False + return True + def otel_decision_from_liboboe( self, liboboe_decision: Dict @@ -290,28 +306,35 @@ def calculate_attributes( new_attributes = {} + # If not a liboboe continued trace, set service entry internal KVs + if not self.is_decision_continued(decision): + new_attributes[self._INTERNAL_BUCKET_CAPACITY] = "{}".format(decision["bucket_cap"]) + new_attributes[self._INTERNAL_BUCKET_RATE] = "{}".format(decision["bucket_rate"]) + new_attributes[self._INTERNAL_SAMPLE_RATE] = decision["rate"] + new_attributes[self._INTERNAL_SAMPLE_SOURCE] = decision["source"] + logger.debug( + "Set attributes with service entry internal KVs: {}".format(new_attributes) + ) + # Trace's root span has no valid traceparent nor tracestate - # so set service entry internal KVs here and only here + # so we can't calculate remaining attributes if not parent_span_context.is_valid or not trace_state: - new_attributes = { - self._INTERNAL_BUCKET_CAPACITY: "{}".format(decision["bucket_cap"]), - self._INTERNAL_BUCKET_RATE: "{}".format(decision["bucket_rate"]), - self._INTERNAL_SAMPLE_RATE: decision["rate"], - self._INTERNAL_SAMPLE_SOURCE: decision["source"] - } logger.debug( - "No valid traceparent or no tracestate - only setting " - "service entry internal KVs attributes: {}".format(new_attributes) + "No valid traceparent or no tracestate - returning attributes: {}" + .format(new_attributes) ) - # attributes must be immutable for SamplingResult - return MappingProxyType(new_attributes) + if new_attributes: + # attributes must be immutable for SamplingResult + return MappingProxyType(new_attributes) + else: + return None # Set attributes with self._SW_TRACESTATE_ROOT_KEY and self._SW_TRACESTATE_CAPTURE_KEY if not attributes: trace_state_no_response = self.remove_response_from_sw(trace_state) - new_attributes = { + new_attributes.update({ self._SW_TRACESTATE_CAPTURE_KEY: trace_state_no_response.to_header() - } + }) # Only set self._SW_TRACESTATE_ROOT_KEY on the entry (root) span for this service sw_value = parent_span_context.trace_state.get(SW_TRACESTATE_KEY, None) if sw_value: @@ -322,7 +345,6 @@ def calculate_attributes( else: # Copy existing MappingProxyType KV into new_attributes for modification # attributes may have other vendor info etc - new_attributes = {} for k,v in attributes.items(): new_attributes[k] = v From 7077e6a4ff396ac3ef84977b82777b0838a86762 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Tue, 19 Apr 2022 16:20:20 -0700 Subject: [PATCH 60/71] Latest otel 1.11.0/0.30b0, rename entry point to solarwinds_propagator, add exporter entry point --- setup.cfg | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7d13bf18..3ee83a83 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,12 +18,14 @@ classifiers = [options] python_requires = >=3.6 install_requires = - opentelemetry-api == 1.10.0 - opentelemetry-sdk == 1.10.0 - opentelemetry-instrumentation == 0.29b0 + opentelemetry-api == 1.11.0 + opentelemetry-sdk == 1.11.0 + opentelemetry-instrumentation == 0.30b0 [options.entry_points] opentelemetry_distro = solarwinds_distro = opentelemetry_distro_solarwinds.distro:SolarWindsDistro opentelemetry_propagator = - solarwinds = opentelemetry_distro_solarwinds.propagator:SolarWindsPropagator + solarwinds_propagator = opentelemetry_distro_solarwinds.propagator:SolarWindsPropagator +opentelemetry_traces_exporter = + solarwinds_exporter = opentelemetry_distro_solarwinds.exporter:SolarWindsSpanExporter \ No newline at end of file From f9171c8134942316a6362e86802e248020cfa6b9 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Tue, 19 Apr 2022 16:21:07 -0700 Subject: [PATCH 61/71] Exporter and Propagator(s) are env var configurable --- opentelemetry_distro_solarwinds/distro.py | 86 +++++++++++++++++------ 1 file changed, 65 insertions(+), 21 deletions(-) diff --git a/opentelemetry_distro_solarwinds/distro.py b/opentelemetry_distro_solarwinds/distro.py index 628e9a42..cbd35c55 100644 --- a/opentelemetry_distro_solarwinds/distro.py +++ b/opentelemetry_distro_solarwinds/distro.py @@ -1,43 +1,87 @@ """Module to configure OpenTelemetry agent to work with SolarWinds backend""" +import logging +from os import environ +from pkg_resources import iter_entry_points + from opentelemetry import trace -from opentelemetry.baggage.propagation import W3CBaggagePropagator +from opentelemetry.environment_variables import OTEL_PROPAGATORS, OTEL_TRACES_EXPORTER from opentelemetry.instrumentation.distro import BaseDistro from opentelemetry.instrumentation.propagators import set_global_response_propagator from opentelemetry.propagate import set_global_textmap from opentelemetry.propagators.composite import CompositePropagator from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.trace.propagation.tracecontext import ( - TraceContextTextMapPropagator, -) -from opentelemetry_distro_solarwinds.exporter import SolarWindsSpanExporter -from opentelemetry_distro_solarwinds.propagator import SolarWindsPropagator from opentelemetry_distro_solarwinds.response_propagator import SolarWindsTraceResponsePropagator from opentelemetry_distro_solarwinds.sampler import ParentBasedSwSampler +logger = logging.getLogger(__name__) class SolarWindsDistro(BaseDistro): - """SolarWinds custom distro for OpenTelemetry agents. + """SolarWinds custom distro for OpenTelemetry agents.""" + + _DEFAULT_OTEL_EXPORTER = "solarwinds_exporter" + _DEFAULT_OTEL_PROPAGATORS = [ + "tracecontext", + "baggage", + "solarwinds_propagator", + ] - With this custom distro, the following functionality is introduced: - - no functionality added at this time - """ def _configure(self, **kwargs): - # Automatically make use of custom SolarWinds sampler + # Automatically use custom SolarWinds sampler trace.set_tracer_provider( TracerProvider(sampler=ParentBasedSwSampler())) - # Automatically configure the SolarWinds Span exporter - span_exporter = BatchSpanProcessor(SolarWindsSpanExporter()) - trace.get_tracer_provider().add_span_processor(span_exporter) - # Configure a CompositePropagator including SolarWinds - set_global_textmap( - CompositePropagator([ - TraceContextTextMapPropagator(), - W3CBaggagePropagator(), - SolarWindsPropagator() - ]) + + # Customize Exporter else default to SolarWindsSpanExporter + environ_exporter = environ.get( + OTEL_TRACES_EXPORTER, + self._DEFAULT_OTEL_EXPORTER ) + try: + exporter = next( + iter_entry_points( + "opentelemetry_traces_exporter", + environ_exporter + )).load()() + except: + logger.exception( + "Failed to load configured exporter `%s`", environ_exporter + ) + raise + + span_exporter = BatchSpanProcessor(exporter) + trace.get_tracer_provider().add_span_processor(span_exporter) + + # Configure context propagators to always include + # tracecontext,baggage,solarwinds -- first and in that order + # -- plus any others specified by env var + environ_propagators = environ.get( + OTEL_PROPAGATORS, + ",".join(self._DEFAULT_OTEL_PROPAGATORS) + ).split(",") + if environ_propagators != self._DEFAULT_OTEL_PROPAGATORS: + for default in self._DEFAULT_OTEL_PROPAGATORS: + while default in environ_propagators: + environ_propagators.remove(default) + environ_propagators = self._DEFAULT_OTEL_PROPAGATORS + environ_propagators + environ[OTEL_PROPAGATORS] = ",".join(environ_propagators) + + # Init and set CompositePropagator globally, like OTel API + propagators = [] + for propagator in environ_propagators: + try: + propagators.append( + next( + iter_entry_points("opentelemetry_propagator", propagator) + ).load()() + ) + except Exception: + logger.exception( + "Failed to load configured propagator `%s`", propagator + ) + raise + set_global_textmap(CompositePropagator(propagators)) + # Set global HTTP response propagator set_global_response_propagator(SolarWindsTraceResponsePropagator()) \ No newline at end of file From 0aef14c867867720fa05322e205a8a48b72f6ca6 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 21 Apr 2022 14:00:56 -0700 Subject: [PATCH 62/71] NH-9150 Use oboe_api fns for no default timestamp --- opentelemetry_distro_solarwinds/exporter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/opentelemetry_distro_solarwinds/exporter.py b/opentelemetry_distro_solarwinds/exporter.py index 8c1ed70d..ce23637e 100644 --- a/opentelemetry_distro_solarwinds/exporter.py +++ b/opentelemetry_distro_solarwinds/exporter.py @@ -38,19 +38,19 @@ def export(self, spans): # If there is a parent, we need to add an edge to this parent to this entry event logger.debug("Continue trace from {}".format(md.toString())) parent_md = self._build_metadata(span.parent) - evt = Context.startTrace(md, int(span.start_time / 1000), + evt = Context.createEntry(md, int(span.start_time / 1000), parent_md) else: # In OpenTelemrtry, there are no events with individual IDs, but only a span ID # and trace ID. Thus, the entry event needs to be generated such that it has the # same op ID as the span ID of the OTel span. logger.debug("Start a new trace {}".format(md.toString())) - evt = Context.startTrace(md, int(span.start_time / 1000)) + evt = Context.createEntry(md, int(span.start_time / 1000)) evt.addInfo('Layer', span.name) evt.addInfo('Language', 'Python') for k, v in span.attributes.items(): evt.addInfo(k, v) - self.reporter.sendReport(evt) + self.reporter.sendReport(evt, False) for event in span.events: if event.name == 'exception': @@ -58,9 +58,9 @@ def export(self, spans): else: self._report_info_event(event) - evt = Context.stopTrace(int(span.end_time / 1000)) + evt = Context.createExit(int(span.end_time / 1000)) evt.addInfo('Layer', span.name) - self.reporter.sendReport(evt) + self.reporter.sendReport(evt, False) def _report_exception_event(self, event): evt = Context.createEvent(int(event.timestamp / 1000)) @@ -76,7 +76,7 @@ def _report_exception_event(self, event): if k not in ('exception.type', 'exception.message', 'exception.stacktrace'): evt.addInfo(k, v) - self.reporter.sendReport(evt) + self.reporter.sendReport(evt, False) def _report_info_event(self, event): print("Found info event") @@ -86,7 +86,7 @@ def _report_info_event(self, event): evt.addInfo('Label', 'info') for k, v in event.attributes.items(): evt.addInfo(k, v) - self.reporter.sendReport(evt) + self.reporter.sendReport(evt, False) def _initialize_solarwinds_reporter(self): """Initialize liboboe.""" From 7b75e65f4df0a784060b072472c0f75fa0fa9081 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 22 Apr 2022 12:15:39 -0700 Subject: [PATCH 63/71] Separate concerns: env vars in distro, component init in configurator --- .../configurator.py | 82 +++++++++++++++++ opentelemetry_distro_solarwinds/distro.py | 87 ++++++------------- setup.cfg | 6 +- 3 files changed, 113 insertions(+), 62 deletions(-) create mode 100644 opentelemetry_distro_solarwinds/configurator.py diff --git a/opentelemetry_distro_solarwinds/configurator.py b/opentelemetry_distro_solarwinds/configurator.py new file mode 100644 index 00000000..0face11f --- /dev/null +++ b/opentelemetry_distro_solarwinds/configurator.py @@ -0,0 +1,82 @@ +"""Module to initialize OpenTelemetry SDK components to work with SolarWinds backend""" + +import logging +from os import environ +from pkg_resources import iter_entry_points + +from opentelemetry import trace +from opentelemetry.environment_variables import ( + OTEL_PROPAGATORS, + OTEL_TRACES_EXPORTER +) +from opentelemetry.instrumentation.propagators import set_global_response_propagator +from opentelemetry.propagate import set_global_textmap +from opentelemetry.propagators.composite import CompositePropagator +from opentelemetry.sdk._configuration import _OTelSDKConfigurator +from opentelemetry.sdk.environment_variables import OTEL_TRACES_SAMPLER +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from opentelemetry_distro_solarwinds.distro import SolarWindsDistro +from opentelemetry_distro_solarwinds.response_propagator import SolarWindsTraceResponsePropagator + +logger = logging.getLogger(__name__) + +class SolarWindsConfigurator(_OTelSDKConfigurator): + """OpenTelemetry Configurator for initializing SolarWinds-reporting SDK components""" + + def _configure(self, **kwargs): + # If default_traces_sampler is configured then hook up + # Else let OTel Python get_from_env_or_default + environ_sampler = environ.get(OTEL_TRACES_SAMPLER) + if environ_sampler == SolarWindsDistro.default_sw_traces_sampler(): + try: + sampler = next( + iter_entry_points( + "opentelemetry_traces_sampler", + environ_sampler + )).load()() + except: + logger.exception( + "Failed to load configured sampler `%s`", environ_sampler + ) + raise + trace.set_tracer_provider( + TracerProvider(sampler=sampler)) + else: + trace.set_tracer_provider() + + environ_exporter = environ.get(OTEL_TRACES_EXPORTER) + try: + exporter = next( + iter_entry_points( + "opentelemetry_traces_exporter", + environ_exporter + )).load()() + except: + logger.exception( + "Failed to load configured exporter `%s`", environ_exporter + ) + raise + span_exporter = BatchSpanProcessor(exporter) + trace.get_tracer_provider().add_span_processor(span_exporter) + + # Init and set CompositePropagator globally, like OTel API + environ_propagators = environ.get(OTEL_PROPAGATORS).split(",") + propagators = [] + for propagator in environ_propagators: + try: + propagators.append( + next( + iter_entry_points("opentelemetry_propagator", propagator) + ).load()() + ) + except Exception: + logger.exception( + "Failed to load configured propagator `%s`", propagator + ) + raise + set_global_textmap(CompositePropagator(propagators)) + + # Set global HTTP response propagator + set_global_response_propagator(SolarWindsTraceResponsePropagator()) diff --git a/opentelemetry_distro_solarwinds/distro.py b/opentelemetry_distro_solarwinds/distro.py index cbd35c55..f48f8672 100644 --- a/opentelemetry_distro_solarwinds/distro.py +++ b/opentelemetry_distro_solarwinds/distro.py @@ -1,87 +1,52 @@ -"""Module to configure OpenTelemetry agent to work with SolarWinds backend""" +"""Module to configure OpenTelemetry to work with SolarWinds backend""" import logging from os import environ -from pkg_resources import iter_entry_points -from opentelemetry import trace -from opentelemetry.environment_variables import OTEL_PROPAGATORS, OTEL_TRACES_EXPORTER +from opentelemetry.environment_variables import ( + OTEL_PROPAGATORS, + OTEL_TRACES_EXPORTER +) from opentelemetry.instrumentation.distro import BaseDistro -from opentelemetry.instrumentation.propagators import set_global_response_propagator -from opentelemetry.propagate import set_global_textmap -from opentelemetry.propagators.composite import CompositePropagator -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor - -from opentelemetry_distro_solarwinds.response_propagator import SolarWindsTraceResponsePropagator -from opentelemetry_distro_solarwinds.sampler import ParentBasedSwSampler +from opentelemetry.sdk.environment_variables import OTEL_TRACES_SAMPLER logger = logging.getLogger(__name__) class SolarWindsDistro(BaseDistro): - """SolarWinds custom distro for OpenTelemetry agents.""" + """OpenTelemetry Distro for SolarWinds reporting environment""" - _DEFAULT_OTEL_EXPORTER = "solarwinds_exporter" - _DEFAULT_OTEL_PROPAGATORS = [ + _DEFAULT_SW_PROPAGATORS = [ "tracecontext", "baggage", "solarwinds_propagator", ] + _DEFAULT_SW_TRACES_EXPORTER = "solarwinds_exporter" + _DEFAULT_SW_TRACES_SAMPLER = "solarwinds_sampler" def _configure(self, **kwargs): - # Automatically use custom SolarWinds sampler - trace.set_tracer_provider( - TracerProvider(sampler=ParentBasedSwSampler())) - - # Customize Exporter else default to SolarWindsSpanExporter - environ_exporter = environ.get( - OTEL_TRACES_EXPORTER, - self._DEFAULT_OTEL_EXPORTER - ) - try: - exporter = next( - iter_entry_points( - "opentelemetry_traces_exporter", - environ_exporter - )).load()() - except: - logger.exception( - "Failed to load configured exporter `%s`", environ_exporter - ) - raise - - span_exporter = BatchSpanProcessor(exporter) - trace.get_tracer_provider().add_span_processor(span_exporter) - + environ.setdefault(OTEL_TRACES_SAMPLER, self._DEFAULT_SW_TRACES_SAMPLER) + environ.setdefault(OTEL_TRACES_EXPORTER, self._DEFAULT_SW_TRACES_EXPORTER) + # Configure context propagators to always include # tracecontext,baggage,solarwinds -- first and in that order # -- plus any others specified by env var environ_propagators = environ.get( OTEL_PROPAGATORS, - ",".join(self._DEFAULT_OTEL_PROPAGATORS) + ",".join(self._DEFAULT_SW_PROPAGATORS) ).split(",") - if environ_propagators != self._DEFAULT_OTEL_PROPAGATORS: - for default in self._DEFAULT_OTEL_PROPAGATORS: + if environ_propagators != self._DEFAULT_SW_PROPAGATORS: + for default in self._DEFAULT_SW_PROPAGATORS: while default in environ_propagators: environ_propagators.remove(default) - environ_propagators = self._DEFAULT_OTEL_PROPAGATORS + environ_propagators + environ_propagators = self._DEFAULT_SW_PROPAGATORS + environ_propagators environ[OTEL_PROPAGATORS] = ",".join(environ_propagators) - # Init and set CompositePropagator globally, like OTel API - propagators = [] - for propagator in environ_propagators: - try: - propagators.append( - next( - iter_entry_points("opentelemetry_propagator", propagator) - ).load()() - ) - except Exception: - logger.exception( - "Failed to load configured propagator `%s`", propagator - ) - raise - set_global_textmap(CompositePropagator(propagators)) - - # Set global HTTP response propagator - set_global_response_propagator(SolarWindsTraceResponsePropagator()) \ No newline at end of file + logger.debug("Configured SolarWindsDistro: {}, {}, {}".format( + environ.get(OTEL_TRACES_SAMPLER), + environ.get(OTEL_TRACES_EXPORTER), + environ.get(OTEL_PROPAGATORS) + )) + + @classmethod + def default_sw_traces_sampler(cls): + return cls._DEFAULT_SW_TRACES_SAMPLER diff --git a/setup.cfg b/setup.cfg index 3ee83a83..50733850 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,11 @@ install_requires = [options.entry_points] opentelemetry_distro = solarwinds_distro = opentelemetry_distro_solarwinds.distro:SolarWindsDistro +opentelemetry_configurator = + solarwinds_configurator = opentelemetry_distro_solarwinds.configurator:SolarWindsConfigurator opentelemetry_propagator = solarwinds_propagator = opentelemetry_distro_solarwinds.propagator:SolarWindsPropagator opentelemetry_traces_exporter = - solarwinds_exporter = opentelemetry_distro_solarwinds.exporter:SolarWindsSpanExporter \ No newline at end of file + solarwinds_exporter = opentelemetry_distro_solarwinds.exporter:SolarWindsSpanExporter +opentelemetry_traces_sampler = + solarwinds_sampler = opentelemetry_distro_solarwinds.sampler:ParentBasedSwSampler \ No newline at end of file From 600e5b2d210c49daa67caf62eadd4f4dd523c14a Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 27 Apr 2022 10:37:06 -0700 Subject: [PATCH 64/71] Custom Propagator gets tracestate from carrier in case of upstream propagators --- opentelemetry_distro_solarwinds/propagator.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index b11d50ee..b4054a88 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -74,14 +74,16 @@ def inject( context: typing.Optional[Context] = None, setter: textmap.Setter = textmap.default_setter, ) -> None: - """Injects sw tracestate and trace options from SpanContext into carrier for HTTP request""" + """Injects sw tracestate from carrier into carrier for HTTP request, to get + tracestate injected by previous propagators""" span = trace.get_current_span(context) span_context = span.get_span_context() - trace_state = span_context.trace_state sw_value = W3CTransformer.sw_from_context(span_context) + trace_state_header = carrier.get(self._TRACESTATE_HEADER_NAME, None) - # Prepare carrier with context's or new tracestate - if trace_state: + # Prepare carrier with carrier's or new tracestate + if trace_state_header: + trace_state = TraceState.from_header(trace_state_header) # Check if trace_state already contains sw KV if SW_TRACESTATE_KEY in trace_state.keys(): # If so, modify current span_id and trace_flags, and move to beginning of list From ede5686fa2ef84a59cc8ab68f8c8fda9d089b3de Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 27 Apr 2022 17:42:07 -0700 Subject: [PATCH 65/71] Call TraceState.from_header correctly --- opentelemetry_distro_solarwinds/propagator.py | 2 +- opentelemetry_distro_solarwinds/sampler.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/opentelemetry_distro_solarwinds/propagator.py b/opentelemetry_distro_solarwinds/propagator.py index b4054a88..46b578dd 100644 --- a/opentelemetry_distro_solarwinds/propagator.py +++ b/opentelemetry_distro_solarwinds/propagator.py @@ -83,7 +83,7 @@ def inject( # Prepare carrier with carrier's or new tracestate if trace_state_header: - trace_state = TraceState.from_header(trace_state_header) + trace_state = TraceState.from_header([trace_state_header]) # Check if trace_state already contains sw KV if SW_TRACESTATE_KEY in trace_state.keys(): # If so, modify current span_id and trace_flags, and move to beginning of list diff --git a/opentelemetry_distro_solarwinds/sampler.py b/opentelemetry_distro_solarwinds/sampler.py index a4610d28..378e2ab4 100644 --- a/opentelemetry_distro_solarwinds/sampler.py +++ b/opentelemetry_distro_solarwinds/sampler.py @@ -348,9 +348,9 @@ def calculate_attributes( trace_state_no_response = self.remove_response_from_sw(trace_state) else: # Must retain all potential tracestate pairs for attributes - attr_trace_state = TraceState.from_header( + attr_trace_state = TraceState.from_header([ tracestate_capture - ) + ]) new_attr_trace_state = attr_trace_state.update( SW_TRACESTATE_KEY, W3CTransformer.sw_from_span_and_decision( From 2842209ef754e51cc38eb069336e09f7d7f8d1b3 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 28 Apr 2022 12:38:43 -0700 Subject: [PATCH 66/71] NH-2313 adjust CompositePropagator config with OTEL_PROPAGATORS --- opentelemetry_distro_solarwinds/distro.py | 24 ++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/opentelemetry_distro_solarwinds/distro.py b/opentelemetry_distro_solarwinds/distro.py index f48f8672..89073868 100644 --- a/opentelemetry_distro_solarwinds/distro.py +++ b/opentelemetry_distro_solarwinds/distro.py @@ -15,10 +15,12 @@ class SolarWindsDistro(BaseDistro): """OpenTelemetry Distro for SolarWinds reporting environment""" + _TRACECONTEXT_PROPAGATOR = "tracecontext" + _SW_PROPAGATOR = "solarwinds_propagator" _DEFAULT_SW_PROPAGATORS = [ - "tracecontext", + _TRACECONTEXT_PROPAGATOR, "baggage", - "solarwinds_propagator", + _SW_PROPAGATOR, ] _DEFAULT_SW_TRACES_EXPORTER = "solarwinds_exporter" _DEFAULT_SW_TRACES_SAMPLER = "solarwinds_sampler" @@ -27,18 +29,22 @@ def _configure(self, **kwargs): environ.setdefault(OTEL_TRACES_SAMPLER, self._DEFAULT_SW_TRACES_SAMPLER) environ.setdefault(OTEL_TRACES_EXPORTER, self._DEFAULT_SW_TRACES_EXPORTER) - # Configure context propagators to always include - # tracecontext,baggage,solarwinds -- first and in that order - # -- plus any others specified by env var environ_propagators = environ.get( OTEL_PROPAGATORS, ",".join(self._DEFAULT_SW_PROPAGATORS) ).split(",") + # If not using the default propagators, + # can any arbitrary list BUT + # (1) must include tracecontext and solarwinds_propagator + # (2) tracecontext must be before solarwinds_propagator if environ_propagators != self._DEFAULT_SW_PROPAGATORS: - for default in self._DEFAULT_SW_PROPAGATORS: - while default in environ_propagators: - environ_propagators.remove(default) - environ_propagators = self._DEFAULT_SW_PROPAGATORS + environ_propagators + if not self._TRACECONTEXT_PROPAGATOR in environ_propagators or \ + not self._SW_PROPAGATOR in environ_propagators: + raise ValueError("Must include tracecontext and solarwinds_propagator in OTEL_PROPAGATORS to use SolarWinds Observability.") + + if environ_propagators.index(self._SW_PROPAGATOR) \ + < environ_propagators.index(self._TRACECONTEXT_PROPAGATOR): + raise ValueError("tracecontext must be before solarwinds_propagator in OTEL_PROPAGATORS to use SolarWinds Observability.") environ[OTEL_PROPAGATORS] = ",".join(environ_propagators) logger.debug("Configured SolarWindsDistro: {}, {}, {}".format( From 7ffb1a9bee831c23fba0ddee4c5116e63de021c4 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 28 Apr 2022 16:16:01 -0700 Subject: [PATCH 67/71] solarwinds_sampler cannot be env default because not in OTel _KNOWN_SAMPLERS --- .../configurator.py | 41 ++++++++++--------- opentelemetry_distro_solarwinds/distro.py | 6 --- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/opentelemetry_distro_solarwinds/configurator.py b/opentelemetry_distro_solarwinds/configurator.py index 0face11f..7a5514e7 100644 --- a/opentelemetry_distro_solarwinds/configurator.py +++ b/opentelemetry_distro_solarwinds/configurator.py @@ -17,7 +17,6 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry_distro_solarwinds.distro import SolarWindsDistro from opentelemetry_distro_solarwinds.response_propagator import SolarWindsTraceResponsePropagator logger = logging.getLogger(__name__) @@ -25,26 +24,28 @@ class SolarWindsConfigurator(_OTelSDKConfigurator): """OpenTelemetry Configurator for initializing SolarWinds-reporting SDK components""" + # Cannot set as env default because not part of OTel Python _KNOWN_SAMPLERS + # https://github.com/open-telemetry/opentelemetry-python/blob/main/opentelemetry-sdk/src/opentelemetry/sdk/trace/sampling.py#L364-L380 + _DEFAULT_SW_TRACES_SAMPLER = "solarwinds_sampler" + def _configure(self, **kwargs): - # If default_traces_sampler is configured then hook up - # Else let OTel Python get_from_env_or_default - environ_sampler = environ.get(OTEL_TRACES_SAMPLER) - if environ_sampler == SolarWindsDistro.default_sw_traces_sampler(): - try: - sampler = next( - iter_entry_points( - "opentelemetry_traces_sampler", - environ_sampler - )).load()() - except: - logger.exception( - "Failed to load configured sampler `%s`", environ_sampler - ) - raise - trace.set_tracer_provider( - TracerProvider(sampler=sampler)) - else: - trace.set_tracer_provider() + environ_sampler = environ.get( + OTEL_TRACES_SAMPLER, + self._DEFAULT_SW_TRACES_SAMPLER, + ) + try: + sampler = next( + iter_entry_points( + "opentelemetry_traces_sampler", + environ_sampler + )).load()() + except: + logger.exception( + "Failed to load configured sampler `%s`", environ_sampler + ) + raise + trace.set_tracer_provider( + TracerProvider(sampler=sampler)) environ_exporter = environ.get(OTEL_TRACES_EXPORTER) try: diff --git a/opentelemetry_distro_solarwinds/distro.py b/opentelemetry_distro_solarwinds/distro.py index f48f8672..0b3371b4 100644 --- a/opentelemetry_distro_solarwinds/distro.py +++ b/opentelemetry_distro_solarwinds/distro.py @@ -21,10 +21,8 @@ class SolarWindsDistro(BaseDistro): "solarwinds_propagator", ] _DEFAULT_SW_TRACES_EXPORTER = "solarwinds_exporter" - _DEFAULT_SW_TRACES_SAMPLER = "solarwinds_sampler" def _configure(self, **kwargs): - environ.setdefault(OTEL_TRACES_SAMPLER, self._DEFAULT_SW_TRACES_SAMPLER) environ.setdefault(OTEL_TRACES_EXPORTER, self._DEFAULT_SW_TRACES_EXPORTER) # Configure context propagators to always include @@ -46,7 +44,3 @@ def _configure(self, **kwargs): environ.get(OTEL_TRACES_EXPORTER), environ.get(OTEL_PROPAGATORS) )) - - @classmethod - def default_sw_traces_sampler(cls): - return cls._DEFAULT_SW_TRACES_SAMPLER From 1241e7a149facd9f8b84f274e8d3e760ffb638e5 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Tue, 3 May 2022 12:19:23 -0700 Subject: [PATCH 68/71] load_entry_point instead of iter for singulars --- .../configurator.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/opentelemetry_distro_solarwinds/configurator.py b/opentelemetry_distro_solarwinds/configurator.py index 7a5514e7..e795fd0f 100644 --- a/opentelemetry_distro_solarwinds/configurator.py +++ b/opentelemetry_distro_solarwinds/configurator.py @@ -2,7 +2,10 @@ import logging from os import environ -from pkg_resources import iter_entry_points +from pkg_resources import ( + iter_entry_points, + load_entry_point +) from opentelemetry import trace from opentelemetry.environment_variables import ( @@ -34,11 +37,11 @@ def _configure(self, **kwargs): self._DEFAULT_SW_TRACES_SAMPLER, ) try: - sampler = next( - iter_entry_points( - "opentelemetry_traces_sampler", - environ_sampler - )).load()() + sampler = load_entry_point( + "opentelemetry_distro_solarwinds", + "opentelemetry_traces_sampler", + environ_sampler + )() except: logger.exception( "Failed to load configured sampler `%s`", environ_sampler @@ -49,11 +52,11 @@ def _configure(self, **kwargs): environ_exporter = environ.get(OTEL_TRACES_EXPORTER) try: - exporter = next( - iter_entry_points( - "opentelemetry_traces_exporter", - environ_exporter - )).load()() + exporter = load_entry_point( + "opentelemetry_distro_solarwinds", + "opentelemetry_traces_exporter", + environ_exporter + )() except: logger.exception( "Failed to load configured exporter `%s`", environ_exporter From 73be566c980bc0b9152cddb3fd79ae687f8bfed9 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 May 2022 11:49:20 -0700 Subject: [PATCH 69/71] Move _initialize_solarwinds_reporter from Exporter to Configurator --- opentelemetry_distro_solarwinds/__init__.py | 3 +- .../configurator.py | 116 ++++++++++++++---- opentelemetry_distro_solarwinds/distro.py | 10 +- opentelemetry_distro_solarwinds/exporter.py | 49 ++------ 4 files changed, 108 insertions(+), 70 deletions(-) diff --git a/opentelemetry_distro_solarwinds/__init__.py b/opentelemetry_distro_solarwinds/__init__.py index f5341bd4..8c04e25f 100644 --- a/opentelemetry_distro_solarwinds/__init__.py +++ b/opentelemetry_distro_solarwinds/__init__.py @@ -6,4 +6,5 @@ EQUALS_W3C_SANITIZED = "####" SW_TRACESTATE_KEY = "sw" OTEL_CONTEXT_SW_OPTIONS_KEY = "sw_xtraceoptions" -OTEL_CONTEXT_SW_SIGNATURE_KEY = "sw_signature" \ No newline at end of file +OTEL_CONTEXT_SW_SIGNATURE_KEY = "sw_signature" +DEFAULT_SW_TRACES_EXPORTER = "solarwinds_exporter" \ No newline at end of file diff --git a/opentelemetry_distro_solarwinds/configurator.py b/opentelemetry_distro_solarwinds/configurator.py index e795fd0f..7d8d1df4 100644 --- a/opentelemetry_distro_solarwinds/configurator.py +++ b/opentelemetry_distro_solarwinds/configurator.py @@ -20,6 +20,8 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry_distro_solarwinds import DEFAULT_SW_TRACES_EXPORTER +from opentelemetry_distro_solarwinds.extension.oboe import Reporter from opentelemetry_distro_solarwinds.response_propagator import SolarWindsTraceResponsePropagator logger = logging.getLogger(__name__) @@ -32,7 +34,18 @@ class SolarWindsConfigurator(_OTelSDKConfigurator): _DEFAULT_SW_TRACES_SAMPLER = "solarwinds_sampler" def _configure(self, **kwargs): - environ_sampler = environ.get( + """Configure OTel sampler, exporter, propagator, response propagator""" + reporter = self._initialize_solarwinds_reporter() + self._configure_sampler() + self._configure_exporter(reporter) + self._configure_propagator() + # Set global HTTP response propagator + set_global_response_propagator(SolarWindsTraceResponsePropagator()) + + def _configure_sampler(self): + """Configure SolarWinds or env-specified OTel sampler""" + sampler = None + environ_sampler_name = environ.get( OTEL_TRACES_SAMPLER, self._DEFAULT_SW_TRACES_SAMPLER, ) @@ -40,35 +53,60 @@ def _configure(self, **kwargs): sampler = load_entry_point( "opentelemetry_distro_solarwinds", "opentelemetry_traces_sampler", - environ_sampler + environ_sampler_name )() except: logger.exception( - "Failed to load configured sampler `%s`", environ_sampler + "Failed to load configured sampler {}".format( + environ_sampler_name + ) ) raise trace.set_tracer_provider( TracerProvider(sampler=sampler)) - environ_exporter = environ.get(OTEL_TRACES_EXPORTER) - try: - exporter = load_entry_point( - "opentelemetry_distro_solarwinds", - "opentelemetry_traces_exporter", - environ_exporter - )() - except: - logger.exception( - "Failed to load configured exporter `%s`", environ_exporter - ) - raise - span_exporter = BatchSpanProcessor(exporter) - trace.get_tracer_provider().add_span_processor(span_exporter) + def _configure_exporter(self, reporter): + """Configure SolarWinds or env-specified OTel span exporter. + Initialization of SolarWinds exporter requires a liboboe reporter.""" + exporter = None + environ_exporter_name = environ.get(OTEL_TRACES_EXPORTER) + + if environ_exporter_name == DEFAULT_SW_TRACES_EXPORTER: + try: + exporter = load_entry_point( + "opentelemetry_distro_solarwinds", + "opentelemetry_traces_exporter", + environ_exporter_name + )(reporter) + except: + logger.exception( + "Failed to load configured exporter {} with reporter".format( + environ_exporter_name + ) + ) + raise + else: + try: + exporter = load_entry_point( + "opentelemetry_distro_solarwinds", + "opentelemetry_traces_exporter", + environ_exporter_name + )() + except: + logger.exception( + "Failed to load configured exporter {}".format( + environ_exporter_name + ) + ) + raise + span_processor = BatchSpanProcessor(exporter) + trace.get_tracer_provider().add_span_processor(span_processor) - # Init and set CompositePropagator globally, like OTel API - environ_propagators = environ.get(OTEL_PROPAGATORS).split(",") + def _configure_propagator(self): + """Configure CompositePropagator with SolarWinds and other propagators""" propagators = [] - for propagator in environ_propagators: + environ_propagators_names = environ.get(OTEL_PROPAGATORS).split(",") + for propagator in environ_propagators_names: try: propagators.append( next( @@ -77,10 +115,42 @@ def _configure(self, **kwargs): ) except Exception: logger.exception( - "Failed to load configured propagator `%s`", propagator + "Failed to load configured propagator {}".format( + propagator + ) ) raise set_global_textmap(CompositePropagator(propagators)) - # Set global HTTP response propagator - set_global_response_propagator(SolarWindsTraceResponsePropagator()) + def _initialize_solarwinds_reporter(self) -> Reporter: + """Initialize SolarWinds reporter used by sampler and exporter. This establishes collector and sampling settings in a background thread.""" + log_level = environ.get('SOLARWINDS_DEBUG_LEVEL', 3) + try: + log_level = int(log_level) + except ValueError: + log_level = 3 + # TODO make some of these customizable + return Reporter( + hostname_alias='', + log_level=log_level, + log_file_path='', + max_transactions=-1, + max_flush_wait_time=-1, + events_flush_interval=-1, + max_request_size_bytes=-1, + reporter='ssl', + host=environ.get('SOLARWINDS_COLLECTOR', ''), + service_key=environ.get('SOLARWINDS_SERVICE_KEY', ''), + trusted_path='', + buffer_size=-1, + trace_metrics=-1, + histogram_precision=-1, + token_bucket_capacity=-1, + token_bucket_rate=-1, + file_single=0, + ec2_metadata_timeout=1000, + grpc_proxy='', + stdout_clear_nonblocking=0, + is_grpc_clean_hack_enabled=False, + w3c_trace_format=1, + ) diff --git a/opentelemetry_distro_solarwinds/distro.py b/opentelemetry_distro_solarwinds/distro.py index 6ffab6cf..be0a2b27 100644 --- a/opentelemetry_distro_solarwinds/distro.py +++ b/opentelemetry_distro_solarwinds/distro.py @@ -8,7 +8,8 @@ OTEL_TRACES_EXPORTER ) from opentelemetry.instrumentation.distro import BaseDistro -from opentelemetry.sdk.environment_variables import OTEL_TRACES_SAMPLER + +from opentelemetry_distro_solarwinds import DEFAULT_SW_TRACES_EXPORTER logger = logging.getLogger(__name__) @@ -22,10 +23,10 @@ class SolarWindsDistro(BaseDistro): "baggage", _SW_PROPAGATOR, ] - _DEFAULT_SW_TRACES_EXPORTER = "solarwinds_exporter" def _configure(self, **kwargs): - environ.setdefault(OTEL_TRACES_EXPORTER, self._DEFAULT_SW_TRACES_EXPORTER) + """Configure OTel exporter and propagators""" + environ.setdefault(OTEL_TRACES_EXPORTER, DEFAULT_SW_TRACES_EXPORTER) environ_propagators = environ.get( OTEL_PROPAGATORS, @@ -45,8 +46,7 @@ def _configure(self, **kwargs): raise ValueError("tracecontext must be before solarwinds_propagator in OTEL_PROPAGATORS to use SolarWinds Observability.") environ[OTEL_PROPAGATORS] = ",".join(environ_propagators) - logger.debug("Configured SolarWindsDistro: {}, {}, {}".format( - environ.get(OTEL_TRACES_SAMPLER), + logger.debug("Configured SolarWindsDistro: {}, {}".format( environ.get(OTEL_TRACES_EXPORTER), environ.get(OTEL_PROPAGATORS) )) diff --git a/opentelemetry_distro_solarwinds/exporter.py b/opentelemetry_distro_solarwinds/exporter.py index ce23637e..3a03355e 100644 --- a/opentelemetry_distro_solarwinds/exporter.py +++ b/opentelemetry_distro_solarwinds/exporter.py @@ -5,26 +5,25 @@ """ import logging -import os from opentelemetry.sdk.trace.export import SpanExporter -from opentelemetry_distro_solarwinds.extension.oboe import (Context, Metadata, - Reporter) +from opentelemetry_distro_solarwinds.extension.oboe import ( + Context, + Metadata +) from opentelemetry_distro_solarwinds.w3c_transformer import W3CTransformer logger = logging.getLogger(__file__) class SolarWindsSpanExporter(SpanExporter): - """SolarWinds span exporter. - - Reports instrumentation data to the SolarWinds backend. + """SolarWinds custom span exporter for the SolarWinds backend. + Initialization requires a liboboe reporter. """ - def __init__(self, *args, **kw_args): + def __init__(self, reporter, *args, **kw_args): super().__init__(*args, **kw_args) - self.reporter = None - self._initialize_solarwinds_reporter() + self.reporter = reporter def export(self, spans): """Export to AO events and report via liboboe. @@ -88,38 +87,6 @@ def _report_info_event(self, event): evt.addInfo(k, v) self.reporter.sendReport(evt, False) - def _initialize_solarwinds_reporter(self): - """Initialize liboboe.""" - log_level = os.environ.get('SOLARWINDS_DEBUG_LEVEL', 3) - try: - log_level = int(log_level) - except ValueError: - log_level = 3 - self.reporter = Reporter( - hostname_alias='', - log_level=log_level, - log_file_path='', - max_transactions=-1, - max_flush_wait_time=-1, - events_flush_interval=-1, - max_request_size_bytes=-1, - reporter='ssl', - host=os.environ.get('SOLARWINDS_COLLECTOR', ''), - service_key=os.environ.get('SOLARWINDS_SERVICE_KEY', ''), - trusted_path='', - buffer_size=-1, - trace_metrics=-1, - histogram_precision=-1, - token_bucket_capacity=-1, - token_bucket_rate=-1, - file_single=0, - ec2_metadata_timeout=1000, - grpc_proxy='', - stdout_clear_nonblocking=0, - is_grpc_clean_hack_enabled=False, - w3c_trace_format=1, - ) - @staticmethod def _build_metadata(span_context): return Metadata.fromString( From 126e03066fb09f1cd33d4911af70cdeda8c6ba8f Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 May 2022 14:42:25 -0700 Subject: [PATCH 70/71] Fix non-SW sampler,exporter configure --- .../configurator.py | 53 +++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/opentelemetry_distro_solarwinds/configurator.py b/opentelemetry_distro_solarwinds/configurator.py index 7d8d1df4..a7abf08d 100644 --- a/opentelemetry_distro_solarwinds/configurator.py +++ b/opentelemetry_distro_solarwinds/configurator.py @@ -17,7 +17,10 @@ from opentelemetry.propagators.composite import CompositePropagator from opentelemetry.sdk._configuration import _OTelSDKConfigurator from opentelemetry.sdk.environment_variables import OTEL_TRACES_SAMPLER -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace import ( + sampling, + TracerProvider +) from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry_distro_solarwinds import DEFAULT_SW_TRACES_EXPORTER @@ -49,21 +52,28 @@ def _configure_sampler(self): OTEL_TRACES_SAMPLER, self._DEFAULT_SW_TRACES_SAMPLER, ) - try: - sampler = load_entry_point( - "opentelemetry_distro_solarwinds", - "opentelemetry_traces_sampler", - environ_sampler_name - )() - except: - logger.exception( - "Failed to load configured sampler {}".format( + + if environ_sampler_name == self._DEFAULT_SW_TRACES_SAMPLER: + try: + sampler = load_entry_point( + "opentelemetry_distro_solarwinds", + "opentelemetry_traces_sampler", environ_sampler_name + )() + except: + logger.exception( + "Failed to load configured sampler {}".format( + environ_sampler_name + ) ) - ) - raise + raise + else: + # OTel SDK uses _get_from_env_or_default, not entrypoints + sampler = sampling._get_from_env_or_default() + trace.set_tracer_provider( - TracerProvider(sampler=sampler)) + TracerProvider(sampler=sampler) + ) def _configure_exporter(self, reporter): """Configure SolarWinds or env-specified OTel span exporter. @@ -87,11 +97,12 @@ def _configure_exporter(self, reporter): raise else: try: - exporter = load_entry_point( - "opentelemetry_distro_solarwinds", - "opentelemetry_traces_exporter", - environ_exporter_name - )() + exporter = next( + iter_entry_points( + "opentelemetry_traces_exporter", + environ_exporter_name + ) + ).load()() except: logger.exception( "Failed to load configured exporter {}".format( @@ -106,17 +117,17 @@ def _configure_propagator(self): """Configure CompositePropagator with SolarWinds and other propagators""" propagators = [] environ_propagators_names = environ.get(OTEL_PROPAGATORS).split(",") - for propagator in environ_propagators_names: + for propagator_name in environ_propagators_names: try: propagators.append( next( - iter_entry_points("opentelemetry_propagator", propagator) + iter_entry_points("opentelemetry_propagator", propagator_name) ).load()() ) except Exception: logger.exception( "Failed to load configured propagator {}".format( - propagator + propagator_name ) ) raise From 01f1e985b0e16a398679af095eaf5dae7fa7f5d2 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 5 May 2022 09:33:02 -0700 Subject: [PATCH 71/71] Always use SW Sampler --- .../configurator.py | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/opentelemetry_distro_solarwinds/configurator.py b/opentelemetry_distro_solarwinds/configurator.py index a7abf08d..2a06f7e2 100644 --- a/opentelemetry_distro_solarwinds/configurator.py +++ b/opentelemetry_distro_solarwinds/configurator.py @@ -46,31 +46,20 @@ def _configure(self, **kwargs): set_global_response_propagator(SolarWindsTraceResponsePropagator()) def _configure_sampler(self): - """Configure SolarWinds or env-specified OTel sampler""" - sampler = None - environ_sampler_name = environ.get( - OTEL_TRACES_SAMPLER, - self._DEFAULT_SW_TRACES_SAMPLER, - ) - - if environ_sampler_name == self._DEFAULT_SW_TRACES_SAMPLER: - try: - sampler = load_entry_point( - "opentelemetry_distro_solarwinds", - "opentelemetry_traces_sampler", - environ_sampler_name - )() - except: - logger.exception( - "Failed to load configured sampler {}".format( - environ_sampler_name - ) + """Always configure SolarWinds OTel sampler""" + try: + sampler = load_entry_point( + "opentelemetry_distro_solarwinds", + "opentelemetry_traces_sampler", + self._DEFAULT_SW_TRACES_SAMPLER + )() + except: + logger.exception( + "Failed to load configured sampler {}".format( + self._DEFAULT_SW_TRACES_SAMPLER ) - raise - else: - # OTel SDK uses _get_from_env_or_default, not entrypoints - sampler = sampling._get_from_env_or_default() - + ) + raise trace.set_tracer_provider( TracerProvider(sampler=sampler) )