From f5eb6655ea5fa2d36c6d4146f42cfe636421cd47 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Mon, 8 Aug 2022 20:10:46 -0500 Subject: [PATCH 01/44] Handle optional sampling priority as nil; handle optional origin header --- lib/spandex_datadog/adapter.ex | 150 ++++++++++++++++++++------------- test/adapter_test.exs | 35 ++++++-- 2 files changed, 121 insertions(+), 64 deletions(-) diff --git a/lib/spandex_datadog/adapter.ex b/lib/spandex_datadog/adapter.ex index 00a64ec..98a138f 100644 --- a/lib/spandex_datadog/adapter.ex +++ b/lib/spandex_datadog/adapter.ex @@ -33,91 +33,123 @@ defmodule SpandexDatadog.Adapter do Fetches the Datadog-specific conn request headers if they are present. """ @impl Spandex.Adapter - @spec distributed_context(conn :: Plug.Conn.t(), Tracer.opts()) :: - {:ok, SpanContext.t()} - | {:error, :no_distributed_trace} - def distributed_context(%Plug.Conn{} = conn, _opts) do - trace_id = get_first_header(conn, "x-datadog-trace-id") - parent_id = get_first_header(conn, "x-datadog-parent-id") - priority = get_first_header(conn, "x-datadog-sampling-priority") || 1 - - if is_nil(trace_id) || is_nil(parent_id) do - {:error, :no_distributed_trace} - else - {:ok, %SpanContext{trace_id: trace_id, parent_id: parent_id, priority: priority}} - end + @spec distributed_context(Plug.Conn.t(), Tracer.opts()) :: + {:ok, SpanContext.t()} | {:error, :no_distributed_trace} + def distributed_context(%Plug.Conn{req_headers: headers}, opts) do + distributed_context(headers, opts) end @impl Spandex.Adapter - @spec distributed_context(headers :: Spandex.headers(), Tracer.opts()) :: - {:ok, SpanContext.t()} - | {:error, :no_distributed_trace} + @spec distributed_context(Spandex.headers(), Tracer.opts()) :: + {:ok, SpanContext.t()} | {:error, :no_distributed_trace} def distributed_context(headers, _opts) do - trace_id = get_header(headers, "x-datadog-trace-id") - parent_id = get_header(headers, "x-datadog-parent-id") - priority = get_header(headers, "x-datadog-sampling-priority") || 1 - - if is_nil(trace_id) || is_nil(parent_id) do - {:error, :no_distributed_trace} - else - {:ok, %SpanContext{trace_id: trace_id, parent_id: parent_id, priority: priority}} - end + headers + |> Enum.reduce(%SpanContext{}, &extract_header/2) + |> validate_context() end @doc """ Injects Datadog-specific HTTP headers to represent the specified SpanContext """ @impl Spandex.Adapter - @spec inject_context([{term(), term()}], SpanContext.t(), Tracer.opts()) :: [{term(), term()}] - def inject_context(headers, %SpanContext{} = span_context, _opts) when is_list(headers) do - span_context - |> tracing_headers() - |> Kernel.++(headers) + @spec inject_context(Spandex.headers(), SpanContext.t(), Tracer.opts()) :: Spandex.headers() + def inject_context(headers, %SpanContext{} = span_context, opts) when is_map(headers) do + inject_context(Map.to_list(headers), span_context, opts) |> Enum.into(%{}) end - def inject_context(headers, %SpanContext{} = span_context, _opts) when is_map(headers) do + def inject_context(headers, %SpanContext{} = span_context, _opts) when is_list(headers) do span_context - |> tracing_headers() - |> Enum.into(%{}) - |> Map.merge(headers) + |> Map.from_struct() + |> Enum.reject(fn {_, value} -> is_nil(value) end) + |> Enum.reduce([], &inject_header/2) + |> Enum.concat(headers) end # Private Helpers - @spec get_first_header(Plug.Conn.t(), String.t()) :: integer() | nil - defp get_first_header(conn, header_name) do - conn - |> Plug.Conn.get_req_header(header_name) - |> List.first() - |> parse_header() + # Reduce function that sets values in SpanContext for headers + @spec extract_header({binary(), binary()}, SpanContext.t()) :: SpanContext.t() + defp extract_header({"x-datadog-trace-id", value}, span_context) do + put_context_int(span_context, :trace_id, value) + end + + defp extract_header({"x-datadog-parent-id", value}, span_context) do + put_context_int(span_context, :parent_id, value) end - @spec get_header(%{}, String.t()) :: integer() | nil - defp get_header(headers, key) when is_map(headers) do - Map.get(headers, key, nil) - |> parse_header() + defp extract_header({"x-datadog-sampling-priority", value}, span_context) do + put_context_int(span_context, :priority, value) end - @spec get_header([], String.t()) :: integer() | nil - defp get_header(headers, key) when is_list(headers) do - Enum.find_value(headers, fn {k, v} -> if k == key, do: v end) - |> parse_header() + defp extract_header({"x-datadog-origin", value}, span_context) do + %{span_context | baggage: span_context.baggage ++ [{"_dd.origin", value}]} end - defp parse_header(header) when is_bitstring(header) do - case Integer.parse(header) do - {int, _} -> int - _ -> nil + # Update context with value if it is an integer + @spec put_context_int(Spandex.SpanContext.t(), atom(), binary()) :: Spandex.SpanContext.t() + defp put_context_int(span_context, key, value) do + case Integer.parse(value) do + {int_value, _} -> + Map.put(span_context, key, int_value) + + _ -> + span_context + end + end + + # Determine if SpanContext is valid + @spec validate_context(SpanContext.t()) :: + {:ok, SpanContext.t()} | {:error, :no_distributed_trace} + defp validate_context(%{trace_id: nil}) do + {:error, :no_distributed_trace} + end + + # Based on the code of the Ruby libary, it seems valid + # to have onlly the trace_id and origin headers set, but not parent_id. + # This might happen with RUM or synthetic traces. + defp validate_context(%{parent_id: nil} = span_context) do + case :proplists.get_value("_dd.origin", span_context.baggage) do + :undefined -> + {:error, :no_distributed_trace} + + _ -> + {:ok, span_context} end end - defp parse_header(_header), do: nil + defp validate_context(span_context) do + {:ok, span_context} + end + + # Add SpanContext value to headers + @spec inject_header({atom(), term()}, [{binary(), binary()}]) :: [{binary(), binary()}] + defp inject_header({:baggage, []}, headers) do + headers + end + + defp inject_header({:baggage, baggage}, headers) do + case :proplists.get_value("_dd.origin", baggage) do + :undefined -> + headers + + value -> + [{"x-datadog-origin", value} | headers] + end + end + + defp inject_header({:trace_id, value}, headers) do + [{"x-datadog-trace-id", to_string(value)} | headers] + end + + defp inject_header({:parent_id, value}, headers) do + [{"x-datadog-parent-id", to_string(value)} | headers] + end + + defp inject_header({:priority, value}, headers) do + [{"x-datadog-sampling-priority", to_string(value)} | headers] + end - defp tracing_headers(%SpanContext{trace_id: trace_id, parent_id: parent_id, priority: priority}) do - [ - {"x-datadog-trace-id", to_string(trace_id)}, - {"x-datadog-parent-id", to_string(parent_id)}, - {"x-datadog-sampling-priority", to_string(priority)} - ] + defp inject_header(_, headers) do + headers end end diff --git a/test/adapter_test.exs b/test/adapter_test.exs index 93fde23..36d64fe 100644 --- a/test/adapter_test.exs +++ b/test/adapter_test.exs @@ -88,14 +88,26 @@ defmodule SpandexDatadog.Test.AdapterTest do assert span_context.priority == 2 end - test "priority defaults to 1 (i.e. we currently assume all distributed traces should be kept)" do + test "returns a SpanContext struct when only x-datadog-trace-id and x-datadog-origin are set" do + conn = + :get + |> Plug.Test.conn("/") + |> Plug.Conn.put_req_header("x-datadog-trace-id", "123") + |> Plug.Conn.put_req_header("x-datadog-origin", "rum") + + assert {:ok, %SpanContext{} = span_context} = Adapter.distributed_context(conn, []) + assert span_context.trace_id == 123 + assert :proplists.get_value("_dd.origin", span_context.baggage) == "rum" + end + + test "priority defaults to nil if there is no x-datadog-sampling-priority header" do conn = :get |> Plug.Test.conn("/") |> Plug.Conn.put_req_header("x-datadog-trace-id", "123") |> Plug.Conn.put_req_header("x-datadog-parent-id", "456") - assert {:ok, %SpanContext{priority: 1}} = Adapter.distributed_context(conn, []) + assert {:ok, %SpanContext{priority: nil}} = Adapter.distributed_context(conn, []) end test "returns an error when it cannot detect both a Trace ID and a Span ID" do @@ -127,13 +139,13 @@ defmodule SpandexDatadog.Test.AdapterTest do assert span_context.priority == 2 end - test "priority defaults to 1 (i.e. we currently assume all distributed traces should be kept)" do + test "priority is nil if x-datadog-sampling-priority header is not present" do headers = %{ "x-datadog-trace-id" => "123", "x-datadog-parent-id" => "456" } - assert {:ok, %SpanContext{priority: 1}} = Adapter.distributed_context(headers, []) + assert {:ok, %SpanContext{priority: nil}} = Adapter.distributed_context(headers, []) end test "returns an error when it cannot detect both a Trace ID and a Span ID" do @@ -151,8 +163,8 @@ defmodule SpandexDatadog.Test.AdapterTest do assert result == [ {"x-datadog-trace-id", "123"}, - {"x-datadog-parent-id", "456"}, {"x-datadog-sampling-priority", "10"}, + {"x-datadog-parent-id", "456"}, {"header1", "value1"}, {"header2", "value2"} ] @@ -172,5 +184,18 @@ defmodule SpandexDatadog.Test.AdapterTest do "header2" => "value2" } end + + test "Sets x-datadog-origin from baggage" do + span_context = %SpanContext{trace_id: 123, parent_id: 456, priority: 10, baggage: [{"_dd.origin", "rum"}]} + + result = Adapter.inject_context(%{}, span_context, []) + + assert result == %{ + "x-datadog-trace-id" => "123", + "x-datadog-parent-id" => "456", + "x-datadog-sampling-priority" => "10", + "x-datadog-origin" => "rum" + } + end end end From 322aa33d9a579a30fa05741f23896b091a40b509 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 14:27:22 -0500 Subject: [PATCH 02/44] Baggage keys are atoms --- lib/spandex_datadog/adapter.ex | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/spandex_datadog/adapter.ex b/lib/spandex_datadog/adapter.ex index 98a138f..1c6953f 100644 --- a/lib/spandex_datadog/adapter.ex +++ b/lib/spandex_datadog/adapter.ex @@ -82,7 +82,7 @@ defmodule SpandexDatadog.Adapter do end defp extract_header({"x-datadog-origin", value}, span_context) do - %{span_context | baggage: span_context.baggage ++ [{"_dd.origin", value}]} + %{span_context | baggage: span_context.baggage ++ [{:"_dd.origin", value}]} end # Update context with value if it is an integer @@ -108,7 +108,7 @@ defmodule SpandexDatadog.Adapter do # to have onlly the trace_id and origin headers set, but not parent_id. # This might happen with RUM or synthetic traces. defp validate_context(%{parent_id: nil} = span_context) do - case :proplists.get_value("_dd.origin", span_context.baggage) do + case :proplists.get_value(:"_dd.origin", span_context.baggage) do :undefined -> {:error, :no_distributed_trace} @@ -128,12 +128,11 @@ defmodule SpandexDatadog.Adapter do end defp inject_header({:baggage, baggage}, headers) do - case :proplists.get_value("_dd.origin", baggage) do - :undefined -> - headers - - value -> + case Keyword.fetch(baggage, :"_dd.origin") do + {:ok, value} -> [{"x-datadog-origin", value} | headers] + :error -> + headers end end From fbbebcd043bdb01d4e47026ad11bd28fd3670ac5 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 14:28:51 -0500 Subject: [PATCH 03/44] Baggage keys are atoms. --- test/adapter_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/adapter_test.exs b/test/adapter_test.exs index 36d64fe..3837eef 100644 --- a/test/adapter_test.exs +++ b/test/adapter_test.exs @@ -97,7 +97,7 @@ defmodule SpandexDatadog.Test.AdapterTest do assert {:ok, %SpanContext{} = span_context} = Adapter.distributed_context(conn, []) assert span_context.trace_id == 123 - assert :proplists.get_value("_dd.origin", span_context.baggage) == "rum" + assert span_context.baggage[:"_dd.origin"] == "rum" end test "priority defaults to nil if there is no x-datadog-sampling-priority header" do @@ -186,7 +186,7 @@ defmodule SpandexDatadog.Test.AdapterTest do end test "Sets x-datadog-origin from baggage" do - span_context = %SpanContext{trace_id: 123, parent_id: 456, priority: 10, baggage: [{"_dd.origin", "rum"}]} + span_context = %SpanContext{trace_id: 123, parent_id: 456, priority: 10, baggage: [{:"_dd.origin", "rum"}]} result = Adapter.inject_context(%{}, span_context, []) From b7cb9e8c005e4297501b24f0aa5472a5cb20710c Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 14:29:22 -0500 Subject: [PATCH 04/44] Format test names to start lowercase --- test/adapter_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/adapter_test.exs b/test/adapter_test.exs index 3837eef..b14de1d 100644 --- a/test/adapter_test.exs +++ b/test/adapter_test.exs @@ -155,7 +155,7 @@ defmodule SpandexDatadog.Test.AdapterTest do end describe "inject_context/3" do - test "Prepends distributed tracing headers to an existing list of headers" do + test "prepends distributed tracing headers to an existing list of headers" do span_context = %SpanContext{trace_id: 123, parent_id: 456, priority: 10} headers = [{"header1", "value1"}, {"header2", "value2"}] @@ -170,7 +170,7 @@ defmodule SpandexDatadog.Test.AdapterTest do ] end - test "Merges distributed tracing headers with an existing map of headers" do + test "merges distributed tracing headers with an existing map of headers" do span_context = %SpanContext{trace_id: 123, parent_id: 456, priority: 10} headers = %{"header1" => "value1", "header2" => "value2"} @@ -185,7 +185,7 @@ defmodule SpandexDatadog.Test.AdapterTest do } end - test "Sets x-datadog-origin from baggage" do + test "sets x-datadog-origin from baggage" do span_context = %SpanContext{trace_id: 123, parent_id: 456, priority: 10, baggage: [{:"_dd.origin", "rum"}]} result = Adapter.inject_context(%{}, span_context, []) From 63bec7402aaf233137affee2601f109e7ead9b1f Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 14:30:56 -0500 Subject: [PATCH 05/44] Set sampling via tags --- lib/spandex_datadog/api_server.ex | 150 ++++++++++++++++++++++-------- test/api_server_test.exs | 56 ++++++----- 2 files changed, 143 insertions(+), 63 deletions(-) diff --git a/lib/spandex_datadog/api_server.ex b/lib/spandex_datadog/api_server.ex index 416698f..2ce8f19 100644 --- a/lib/spandex_datadog/api_server.ex +++ b/lib/spandex_datadog/api_server.ex @@ -161,8 +161,8 @@ defmodule SpandexDatadog.ApiServer do @spec format(Span.t()) :: map() def format(%Span{} = span), do: format(span, 1, []) - @spec format(Span.t(), integer(), Keyword.t()) :: map() - def format(%Span{} = span, priority, _baggage) do + @spec format(Span.t(), integer() | nil, Keyword.t()) :: map() + def format(%Span{} = span, priority, baggage) do %{ trace_id: span.trace_id, span_id: span.id, @@ -174,13 +174,8 @@ defmodule SpandexDatadog.ApiServer do resource: span.resource || span.name, service: span.service, type: span.type, - meta: meta(span), - metrics: - metrics(span, %{ - _sampling_priority_v1: priority, - "_dd.rule_psr": 1.0, - "_dd.limit_psr": 1.0 - }) + meta: meta(span, priority, baggage), + metrics: metrics(span, priority, baggage) } end @@ -243,18 +238,32 @@ defmodule SpandexDatadog.ApiServer do end) end - @spec meta(Span.t()) :: map - defp meta(span) do + @spec meta(Span.t(), integer() | nil, Keyword.t()) :: map() + defp meta(span, _priority, baggage) do %{} + |> add_baggage_meta(baggage) |> add_datadog_meta(span) |> add_error_data(span) |> add_http_data(span) |> add_sql_data(span) - |> add_tags(span) + |> add_tags_meta(span) |> Enum.reject(fn {_k, v} -> is_nil(v) end) |> Enum.into(%{}) end + @spec add_baggage_meta(map(), Keyword.t()) :: map() + defp add_baggage_meta(meta, []), do: meta + + defp add_baggage_meta(meta, baggage) do + # distributed tracing + case Keyword.fetch(baggage, :"_dd.origin") do + {:ok, value} -> + Map.put(meta, "_dd.origin", value) + :error -> + meta + end + end + @spec add_datadog_meta(map, Span.t()) :: map defp add_datadog_meta(meta, %Span{env: nil}), do: meta @@ -313,44 +322,107 @@ defmodule SpandexDatadog.ApiServer do |> Map.put("sql.db", sql[:db]) end - @spec add_tags(map, Span.t()) :: map - defp add_tags(meta, %{tags: nil}), do: meta + @spec add_tags_meta(map(), Span.t()) :: map() + defp add_tags_meta(meta, %{tags: nil}), do: meta + + defp add_tags_meta(meta, %{tags: tags}) do + Enum.reduce(tags, meta, &add_tag_meta/2) + end - defp add_tags(meta, %{tags: tags}) do - tags = tags |> Keyword.delete(:analytics_event) + defp add_tag_meta({:hostname, value}, meta) do + Map.put(meta, "_dd.hostname", value) + end - Map.merge( - meta, - tags - |> Enum.map(fn {k, v} -> {k, term_to_string(v)} end) - |> Enum.into(%{}) - ) + defp add_tag_meta({:origin, value}, meta) do + Map.put(meta, "_dd.origin", value) end - @spec metrics(Span.t(), map) :: map - defp metrics(span, initial_value = %{}) do - initial_value - |> add_metrics(span) + # Handled in metrics + defp add_tag_meta({:analytics_event, _value}, meta), do: meta + + # Tags with numeric values go in metrics + defp add_tag_meta({_key, value}, meta) when is_integer(value) or is_float(value) do + meta + end + + defp add_tag_meta({key, value}, meta) do + Map.put(meta, to_string(key), term_to_string(value)) + end + + # Some tags understood by Datadog: + # runtime-id + # language: "elixir" + # system.pid: OS pid + # peer.hostname: Hostname of external service interacted with + # peer.service: Name of external service that performed the work + + @spec metrics(Span.t(), integer() | nil, Keyword.t()) :: map() + defp metrics(span, priority, _baggage) do + %{} + |> add_priority_metrics(priority) + |> add_tags_metrics(span) |> Enum.reject(fn {_k, v} -> is_nil(v) end) |> Enum.into(%{}) end - @spec add_metrics(map, Span.t()) :: map - defp add_metrics(metrics, %{tags: nil}), do: metrics + # Special tag names come mostly from the Ruby Datadog library + # ddtrace-1.1.0/lib/ddtrace/transport/trace_formatter.rb - defp add_metrics(metrics, %{tags: tags}) do - with analytics_event <- tags |> Keyword.get(:analytics_event), - true <- analytics_event != nil do - Map.merge( - metrics, - %{"_dd1.sr.eausr" => 1} - ) - else - _ -> - metrics - end + @spec add_tags_metrics(map(), Span.t()) :: map() + defp add_tags_metrics(meta, %{tags: nil}), do: meta + + defp add_tags_metrics(meta, %{tags: tags}) do + Enum.reduce(tags, meta, &add_tag_metric/2) end + # Sampling + defp add_tag_metric({:agent_sample_rate, value}, metrics) do + Map.put(metrics, "_dd.agent_psr", value) + end + + defp add_tag_metric({:sample_rate, value}, metrics) do + Map.put(metrics, "_dd1.sr.eausr", value) + end + + # Backwards compatibility + defp add_tag_metric({:analytics_event, value}, metrics) when value in [nil, false], do: metrics + + defp add_tag_metric({:analytics_event, _value}, metrics) do + Map.put(metrics, "_dd1.sr.eausr", 1) + end + + # If rule sampling is applied to a span, set metric for the sample rate + # configured for that rule. This should be done regardless of sampling + # outcome. + defp add_tag_metric({:rule_sample_rate, value}, metrics) do + Map.put(metrics, "_dd.rule_psr", value) + end + + # If rate limiting is checked on a span, set metric for the effective rate + # limiting rate applied. This should be done regardless of rate limiting + # outcome. + defp add_tag_metric({:rate_limiter_rate, value}, metrics) do + Map.put(metrics, "_dd.limit_psr", value) + end + + defp add_tag_metric({key, value}, metrics) when is_integer(value) or is_float(value) do + Map.put(metrics, to_string(key), value) + end + + defp add_tag_metric(_, metrics) do + metrics + end + + # Set priority from context + @spec add_priority_metrics(map(), integer() | nil) :: map() + defp add_priority_metrics(metrics, nil), do: metrics + + # Distributed tracing + defp add_priority_metrics(metrics, priority) do + Map.put(metrics, "_sampling_priority_v1", priority) + end + + @spec error(nil | Keyword.t()) :: integer defp error(nil), do: 0 diff --git a/test/api_server_test.exs b/test/api_server_test.exs index 1ed7540..df54365 100644 --- a/test/api_server_test.exs +++ b/test/api_server_test.exs @@ -61,7 +61,8 @@ defmodule SpandexDatadog.ApiServerTest do service: :bar, env: "local", name: "bar", - trace_id: trace_id + trace_id: trace_id, + tags: [analytics_event: true] ) {:ok, span_3} = @@ -73,10 +74,17 @@ defmodule SpandexDatadog.ApiServerTest do env: "local", name: "bar", trace_id: trace_id, - tags: [analytics_event: true] + tags: [ + hostname: "moon", + origin: "web", + agent_sample_rate: 1.0, + sample_rate: 1.0, + rule_sample_rate: 1.0, + rate_limiter_rate: 1.0, + ] ) - trace = %Trace{spans: [span_1, span_2, span_3]} + trace = %Trace{spans: [span_1, span_2, span_3], priority: 2} { :ok, @@ -170,7 +178,6 @@ defmodule SpandexDatadog.ApiServerTest do "duration" => 100_000, "error" => 0, "meta" => %{ - "bar" => "321", "baz" => "{1, 2}", "buz" => "blitz", "env" => "local", @@ -178,9 +185,8 @@ defmodule SpandexDatadog.ApiServerTest do "zyx" => "[xyz: {1, 2}]" }, "metrics" => %{ - "_sampling_priority_v1" => 1, - "_dd.rule_psr" => 1.0, - "_dd.limit_psr" => 1.0 + "bar" => 321, + "_sampling_priority_v1" => 2 }, "name" => "foo", "resource" => "foo", @@ -196,9 +202,8 @@ defmodule SpandexDatadog.ApiServerTest do "env" => "local" }, "metrics" => %{ - "_sampling_priority_v1" => 1, - "_dd.rule_psr" => 1.0, - "_dd.limit_psr" => 1.0 + "_dd1.sr.eausr" => 1, + "_sampling_priority_v1" => 2 }, "name" => "bar", "resource" => "bar", @@ -211,13 +216,16 @@ defmodule SpandexDatadog.ApiServerTest do "duration" => 100_000_000, "error" => 0, "meta" => %{ - "env" => "local" + "env" => "local", + "_dd.hostname" => "moon", + "_dd.origin" => "web" }, "metrics" => %{ - "_dd1.sr.eausr" => 1, - "_sampling_priority_v1" => 1, + "_dd.agent_psr" => 1.0, + "_dd1.sr.eausr" => 1.0, "_dd.rule_psr" => 1.0, - "_dd.limit_psr" => 1.0 + "_dd.limit_psr" => 1.0, + "_sampling_priority_v1" => 2 }, "name" => "bar", "resource" => "bar", @@ -260,7 +268,6 @@ defmodule SpandexDatadog.ApiServerTest do "duration" => 100_000, "error" => 0, "meta" => %{ - "bar" => "321", "baz" => "{1, 2}", "buz" => "blitz", "env" => "local", @@ -268,9 +275,8 @@ defmodule SpandexDatadog.ApiServerTest do "zyx" => "[xyz: {1, 2}]" }, "metrics" => %{ - "_sampling_priority_v1" => 1, - "_dd.rule_psr" => 1.0, - "_dd.limit_psr" => 1.0 + "bar" => 321, + "_sampling_priority_v1" => 2 }, "name" => "foo", "resource" => "foo", @@ -286,9 +292,8 @@ defmodule SpandexDatadog.ApiServerTest do "env" => "local" }, "metrics" => %{ - "_sampling_priority_v1" => 1, - "_dd.rule_psr" => 1.0, - "_dd.limit_psr" => 1.0 + "_dd1.sr.eausr" => 1, + "_sampling_priority_v1" => 2 }, "name" => "bar", "resource" => "bar", @@ -301,13 +306,16 @@ defmodule SpandexDatadog.ApiServerTest do "duration" => 100_000_000, "error" => 0, "meta" => %{ - "env" => "local" + "env" => "local", + "_dd.hostname" => "moon", + "_dd.origin" => "web" }, "metrics" => %{ + "_dd.agent_psr" => 1.0, "_dd.rule_psr" => 1.0, "_dd.limit_psr" => 1.0, - "_dd1.sr.eausr" => 1, - "_sampling_priority_v1" => 1 + "_dd1.sr.eausr" => 1.0, + "_sampling_priority_v1" => 2 }, "name" => "bar", "resource" => "bar", From 7294fa0558019587a76059976974b6f1b85c0ddc Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 14:45:54 -0500 Subject: [PATCH 06/44] Use Config instead of Mix.Config --- config/config.exs | 13 ++++++++++--- config/dev.exs | 2 +- config/test.exs | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 0218721..40bbb8f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,5 +1,12 @@ # This file is responsible for configuring your application -# and its dependencies with the aid of the Mix.Config module. -use Mix.Config +# and its dependencies with the aid of the Config module. +# +# This configuration file is loaded before any dependency and +# is restricted to this project. -import_config "./#{Mix.env()}.exs" +# General application configuration +import Config + +# Import environment specific config. This must remain at the bottom +# of this file so it overrides the configuration defined above. +import_config "#{config_env()}.exs" diff --git a/config/dev.exs b/config/dev.exs index 45c2b77..3ea6615 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :git_ops, mix_project: SpandexDatadog.MixProject, diff --git a/config/test.exs b/config/test.exs index 0a68357..526e7bd 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :logger, :console, level: :debug, From 6c98e16f76f60d2c73ca6c26c4a898768b101f69 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 14:57:30 -0500 Subject: [PATCH 07/44] Remove parens on functions without parameters --- lib/spandex_datadog/adapter.ex | 8 ++++---- lib/spandex_datadog/api_server.ex | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/spandex_datadog/adapter.ex b/lib/spandex_datadog/adapter.ex index 1c6953f..42560e0 100644 --- a/lib/spandex_datadog/adapter.ex +++ b/lib/spandex_datadog/adapter.ex @@ -15,17 +15,17 @@ defmodule SpandexDatadog.Adapter do @max_id 9_223_372_036_854_775_807 @impl Spandex.Adapter - def trace_id(), do: :rand.uniform(@max_id) + def trace_id, do: :rand.uniform(@max_id) @impl Spandex.Adapter - def span_id(), do: trace_id() + def span_id, do: trace_id() @impl Spandex.Adapter - def now(), do: :os.system_time(:nano_seconds) + def now, do: :os.system_time(:nano_seconds) @impl Spandex.Adapter @spec default_sender() :: SpandexDatadog.ApiServer - def default_sender() do + def default_sender do SpandexDatadog.ApiServer end diff --git a/lib/spandex_datadog/api_server.ex b/lib/spandex_datadog/api_server.ex index 2ce8f19..a4fbe0d 100644 --- a/lib/spandex_datadog/api_server.ex +++ b/lib/spandex_datadog/api_server.ex @@ -91,7 +91,7 @@ defmodule SpandexDatadog.ApiServer do @cgroup_task "[0-9a-f]{32}-\\d+" @cgroup_regex Regex.compile!(".*(#{@cgroup_uuid}|#{@cgroup_ctnr}|#{@cgroup_task})(?:\\.scope)?$", "m") - defp get_container_id() do + defp get_container_id do with {:ok, file_binary} <- File.read("/proc/self/cgroup"), [_, container_id] <- Regex.run(@cgroup_regex, file_binary) do container_id @@ -422,7 +422,6 @@ defmodule SpandexDatadog.ApiServer do Map.put(metrics, "_sampling_priority_v1", priority) end - @spec error(nil | Keyword.t()) :: integer defp error(nil), do: 0 From 13f92f46a6234a41075b710d804f3c9ec98d6a91 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 14:57:43 -0500 Subject: [PATCH 08/44] Remove parens on functions without parameters --- test/support/traced_module.ex | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/support/traced_module.ex b/test/support/traced_module.ex index e4f2fec..185907b 100644 --- a/test/support/traced_module.ex +++ b/test/support/traced_module.ex @@ -11,31 +11,31 @@ defmodule SpandexDatadog.Test.TracedModule do # Traces - def trace_one_thing() do + def trace_one_thing do Tracer.trace "trace_one_thing/0" do do_one_thing() end end - def trace_with_special_name() do + def trace_with_special_name do Tracer.trace "special_name", service: :special_service do do_one_special_name_thing() end end - def trace_one_error() do + def trace_one_error do Tracer.trace "trace_one_error/0" do raise TestError, message: "trace_one_error" end end - def error_two_deep() do + def error_two_deep do Tracer.trace "error_two_deep/0" do error_one_deep() end end - def two_fail_one_succeeds() do + def two_fail_one_succeeds do Tracer.trace "two_fail_one_succeeds/0" do try do _ = error_one_deep() @@ -50,25 +50,25 @@ defmodule SpandexDatadog.Test.TracedModule do # Spans - def error_one_deep() do + def error_one_deep do Tracer.span "error_one_deep/0" do raise TestError, message: "error_one_deep" end end - def manually_span_one_thing() do + def manually_span_one_thing do Tracer.span "manually_span_one_thing/0" do :timer.sleep(100) end end - def do_one_thing() do + def do_one_thing do Tracer.span "do_one_thing/0" do :timer.sleep(100) end end - def do_one_special_name_thing() do + def do_one_special_name_thing do Tracer.span "special_name_span", service: :special_span_service do :timer.sleep(100) end From b21745c5fa75aabb104d23aae098ee82d65c870c Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 15:16:30 -0500 Subject: [PATCH 09/44] Add credo and dialyzer. --- mix.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mix.exs b/mix.exs index ad3ce39..2a5e38c 100644 --- a/mix.exs +++ b/mix.exs @@ -55,6 +55,8 @@ defmodule SpandexDatadog.MixProject do {:spandex, "~> 3.0"}, {:telemetry, "~> 0.4.2 or ~> 1.0"}, # Dev- and test-only deps + {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, + {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, {:httpoison, "~> 0.13 or ~> 1.0", only: :test} ] From ea15a735fbc3babec632b0098a326fd75289ad2f Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 15:17:12 -0500 Subject: [PATCH 10/44] Specify spandex from github --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 2a5e38c..34819b1 100644 --- a/mix.exs +++ b/mix.exs @@ -52,7 +52,7 @@ defmodule SpandexDatadog.MixProject do defp deps do [ {:msgpax, "~> 2.2.1 or ~> 2.3"}, - {:spandex, "~> 3.0"}, + {:spandex, github: "TheRealReal/spandex", branch: "distributed-sampling-priority"}, {:telemetry, "~> 0.4.2 or ~> 1.0"}, # Dev- and test-only deps {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, From 97ebb97fe033607c2780634eb4bc21c0a0c737bd Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 15:17:26 -0500 Subject: [PATCH 11/44] Upgrade libraries --- mix.lock | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/mix.lock b/mix.lock index 247ca86..7cdee47 100644 --- a/mix.lock +++ b/mix.lock @@ -1,24 +1,30 @@ %{ - "certifi": {:hex, :certifi, "2.6.1", "dbab8e5e155a0763eea978c913ca280a6b544bfa115633fa20249c3d396d9493", [:rebar3], [], "hexpm", "524c97b4991b3849dd5c17a631223896272c6b0af446778ba4675a1dff53bb7e"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.15", "b29e8e729f4aa4a00436580dcc2c9c5c51890613457c193cc8525c388ccb2f06", [:mix], [], "hexpm", "044523d6438ea19c1b8ec877ec221b008661d3c27e3b848f4c879f500421ca5c"}, - "ex_doc": {:hex, :ex_doc, "0.25.3", "3edf6a0d70a39d2eafde030b8895501b1c93692effcbd21347296c18e47618ce", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "9ebebc2169ec732a38e9e779fd0418c9189b3ca93f4a676c961be6c1527913f5"}, - "hackney": {:hex, :hackney, "1.17.4", "99da4674592504d3fb0cfef0db84c3ba02b4508bae2dff8c0108baa0d6e0977c", [:rebar3], [{:certifi, "~>2.6.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "de16ff4996556c8548d512f4dbe22dd58a587bf3332e7fd362430a7ef3986b16"}, - "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"}, + "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, + "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, + "credo": {:hex, :credo, "1.6.6", "f51f8d45db1af3b2e2f7bee3e6d3c871737bda4a91bff00c5eec276517d1a19c", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "625520ce0984ee0f9f1f198165cd46fa73c1e59a17ebc520038b8fce056a5bdc"}, + "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"}, + "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, + "ex_doc": {:hex, :ex_doc, "0.28.4", "001a0ea6beac2f810f1abc3dbf4b123e9593eaa5f00dd13ded024eae7c523298", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bf85d003dd34911d89c8ddb8bda1a958af3471a274a4c2150a9c01c78ac3f8ed"}, + "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, + "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, + "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, + "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, + "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mime": {:hex, :mime, "2.0.1", "0de4c81303fe07806ebc2494d5321ce8fb4df106e34dd5f9d787b637ebadc256", [:mix], [], "hexpm", "7a86b920d2aedce5fb6280ac8261ac1a739ae6c1a1ad38f5eadf910063008942"}, + "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "msgpax": {:hex, :msgpax, "2.3.0", "14f52ad249a3f77b5e2d59f6143e6c18a6e74f34666989e22bac0a465f9835cc", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "65c36846a62ed5615baf7d7d47babb6541313a6c0b6d2ff19354bd518f52df7e"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm", "1a06ea6a653120226b35b283a1cd10039550f2c566edcdec22b29316d73640fd"}, "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, - "plug": {:hex, :plug, "1.12.1", "645678c800601d8d9f27ad1aebba1fdb9ce5b2623ddb961a074da0b96c35187d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d57e799a777bc20494b784966dc5fbda91eb4a09f571f76545b72a634ce0d30b"}, + "plug": {:hex, :plug, "1.13.6", "187beb6b67c6cec50503e940f0434ea4692b19384d47e5fdfd701e93cadb4cc2", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "02b9c6b9955bce92c829f31d6284bf53c591ca63c4fb9ff81dfd0418667a34ff"}, "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, - "spandex": {:hex, :spandex, "3.0.3", "91aa318f3de696bb4d931adf65f7ebdbe5df25cccce1fe8fd376a44c46bcf69b", [:mix], [{:decorator, "~> 1.2", [hex: :decorator, repo: "hexpm", optional: true]}, {:optimal, "~> 0.3.3", [hex: :optimal, repo: "hexpm", optional: false]}, {:plug, ">= 1.0.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e3e6c319d0ab478ddc9a39102a727a410c962b4d51c0932c72279b86d3b17044"}, + "spandex": {:git, "https://github.com/TheRealReal/spandex.git", "6ae8458e5c558ee5b5972eb89a800c4cf9ec1c17", [branch: "distributed-sampling-priority"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, - "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, + "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, } From a68debd26053477aa5ee377beef292eebaf839df Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 17:14:18 -0500 Subject: [PATCH 12/44] Ignore unrecognized headers --- lib/spandex_datadog/adapter.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spandex_datadog/adapter.ex b/lib/spandex_datadog/adapter.ex index 42560e0..c0e6fc5 100644 --- a/lib/spandex_datadog/adapter.ex +++ b/lib/spandex_datadog/adapter.ex @@ -85,6 +85,10 @@ defmodule SpandexDatadog.Adapter do %{span_context | baggage: span_context.baggage ++ [{:"_dd.origin", value}]} end + defp extract_header(_, span_context) do + span_context + end + # Update context with value if it is an integer @spec put_context_int(Spandex.SpanContext.t(), atom(), binary()) :: Spandex.SpanContext.t() defp put_context_int(span_context, key, value) do From cbd8977c764bfb6ee6e55f26d852541feffd0960 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 18:32:13 -0500 Subject: [PATCH 13/44] Use Keyword module --- lib/spandex_datadog/adapter.ex | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/spandex_datadog/adapter.ex b/lib/spandex_datadog/adapter.ex index c0e6fc5..590f0c4 100644 --- a/lib/spandex_datadog/adapter.ex +++ b/lib/spandex_datadog/adapter.ex @@ -112,12 +112,10 @@ defmodule SpandexDatadog.Adapter do # to have onlly the trace_id and origin headers set, but not parent_id. # This might happen with RUM or synthetic traces. defp validate_context(%{parent_id: nil} = span_context) do - case :proplists.get_value(:"_dd.origin", span_context.baggage) do - :undefined -> - {:error, :no_distributed_trace} - - _ -> - {:ok, span_context} + if Keyword.has_key?(span_context.baggage, :"_dd.origin") do + {:ok, span_context} + else + {:error, :no_distributed_trace} end end @@ -135,6 +133,7 @@ defmodule SpandexDatadog.Adapter do case Keyword.fetch(baggage, :"_dd.origin") do {:ok, value} -> [{"x-datadog-origin", value} | headers] + :error -> headers end From 8f2075aa942ef374dfe2c31f11bbe822e391e3b7 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 18:32:28 -0500 Subject: [PATCH 14/44] mix format --- lib/spandex_datadog/api_server.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spandex_datadog/api_server.ex b/lib/spandex_datadog/api_server.ex index a4fbe0d..4c24c3e 100644 --- a/lib/spandex_datadog/api_server.ex +++ b/lib/spandex_datadog/api_server.ex @@ -259,6 +259,7 @@ defmodule SpandexDatadog.ApiServer do case Keyword.fetch(baggage, :"_dd.origin") do {:ok, value} -> Map.put(meta, "_dd.origin", value) + :error -> meta end From b99404d85314258b4a76e70a546a08d45da6d8c9 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 9 Aug 2022 18:32:42 -0500 Subject: [PATCH 15/44] mix format --- test/api_server_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api_server_test.exs b/test/api_server_test.exs index df54365..b7b30eb 100644 --- a/test/api_server_test.exs +++ b/test/api_server_test.exs @@ -80,7 +80,7 @@ defmodule SpandexDatadog.ApiServerTest do agent_sample_rate: 1.0, sample_rate: 1.0, rule_sample_rate: 1.0, - rate_limiter_rate: 1.0, + rate_limiter_rate: 1.0 ] ) From a73218aac85d20d8c9f0d92119d86f8ea5186d8d Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Wed, 10 Aug 2022 14:53:25 -0500 Subject: [PATCH 16/44] Add docs --- lib/spandex_datadog/api_server.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spandex_datadog/api_server.ex b/lib/spandex_datadog/api_server.ex index 4c24c3e..2f73e7c 100644 --- a/lib/spandex_datadog/api_server.ex +++ b/lib/spandex_datadog/api_server.ex @@ -385,7 +385,8 @@ defmodule SpandexDatadog.ApiServer do Map.put(metrics, "_dd1.sr.eausr", value) end - # Backwards compatibility + # Set a segment to always be sampled. + # `sample_rate` tag should be used instead. defp add_tag_metric({:analytics_event, value}, metrics) when value in [nil, false], do: metrics defp add_tag_metric({:analytics_event, _value}, metrics) do From 508ab18158a30b007c6cfaa24166edf3c47de596 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 11 Aug 2022 14:41:30 -0500 Subject: [PATCH 17/44] Use float value for analytics rate --- lib/spandex_datadog/api_server.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spandex_datadog/api_server.ex b/lib/spandex_datadog/api_server.ex index 2f73e7c..66ab10d 100644 --- a/lib/spandex_datadog/api_server.ex +++ b/lib/spandex_datadog/api_server.ex @@ -390,7 +390,7 @@ defmodule SpandexDatadog.ApiServer do defp add_tag_metric({:analytics_event, value}, metrics) when value in [nil, false], do: metrics defp add_tag_metric({:analytics_event, _value}, metrics) do - Map.put(metrics, "_dd1.sr.eausr", 1) + Map.put(metrics, "_dd1.sr.eausr", 1.0) end # If rule sampling is applied to a span, set metric for the sample rate From 6fd352b1adc910d74a295d6ad7cf166a3a137da4 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 11 Aug 2022 14:42:21 -0500 Subject: [PATCH 18/44] Document max_id --- lib/spandex_datadog/adapter.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spandex_datadog/adapter.ex b/lib/spandex_datadog/adapter.ex index 590f0c4..70499cb 100644 --- a/lib/spandex_datadog/adapter.ex +++ b/lib/spandex_datadog/adapter.ex @@ -12,7 +12,10 @@ defmodule SpandexDatadog.Adapter do Tracer } + # Max value for a trace_id. We only generate 63-bit integers due to + # limitations in other languages, but other systems may use the full 64-bits. @max_id 9_223_372_036_854_775_807 + # @max_id Bitwise.bsl(1, 63) - 1 @impl Spandex.Adapter def trace_id, do: :rand.uniform(@max_id) From 2dcbc6968f296475bfda2a71589de46746b14b22 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 11 Aug 2022 14:44:24 -0500 Subject: [PATCH 19/44] Add functions to randomly sample traces --- lib/spandex_datadog/rate_sampler.ex | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 lib/spandex_datadog/rate_sampler.ex diff --git a/lib/spandex_datadog/rate_sampler.ex b/lib/spandex_datadog/rate_sampler.ex new file mode 100644 index 0000000..05752a5 --- /dev/null +++ b/lib/spandex_datadog/rate_sampler.ex @@ -0,0 +1,39 @@ +defmodule SpandexDatadog.RateSampler do + @moduledoc """ + Randomly sample a percentage of traces based on the trace_id. + """ + + # Max value for a trace_id that we generate. + @max_id Bitwise.bsl(1, 63) - 1 + + # We only generate 63-bit integers due to limitations in other languages, but + # support parsing 64-bit integers for distributed tracing since an upstream + # system may generate one. + @external_max_id Bitwise.bsl(1, 64) + + @knuth_factor 1111111111111111111 + + @doc """ + Determine whether a trace should be sampled. + + This uses the `trace_id` and specified sample rate. + If the `trace_id` is a randomly generated integer, then it will + deterministically select a percentage of traces at random. + + `sample_rate` is a float between 0.0 and 1.0. 0.0 means that no trace will + be sampled, and 1.0 means that all traces will be sampled. + """ + @spec sampled?(non_neg_integer() | Spandex.Trace.t() | Spandex.Span.t(), float()) :: boolean() + def sampled?(trace_id, sample_rate) when is_integer(trace_id) do + threshold = sample_rate * @external_max_id + ((trace_id * @knuth_factor) % @external_max_id) <= threshold + end + + def sampled?(%Spandex.Trace{trace_id: trace_id}, sample_rate) do + sampled?(trace_id, sample_rate) + end + + def sampled?(%Spandex.Span{trace_id: trace_id}, sample_rate) do + sampled?(trace_id, sample_rate) + end +end From 680acc8dc3c80c1fa283a3e4e798c6fc035e2ed1 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 11 Aug 2022 14:51:09 -0500 Subject: [PATCH 20/44] Shortcut for common cases --- lib/spandex_datadog/rate_sampler.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spandex_datadog/rate_sampler.ex b/lib/spandex_datadog/rate_sampler.ex index 05752a5..fe39d48 100644 --- a/lib/spandex_datadog/rate_sampler.ex +++ b/lib/spandex_datadog/rate_sampler.ex @@ -24,6 +24,10 @@ defmodule SpandexDatadog.RateSampler do be sampled, and 1.0 means that all traces will be sampled. """ @spec sampled?(non_neg_integer() | Spandex.Trace.t() | Spandex.Span.t(), float()) :: boolean() + def sampled?(_, 1.0), do: true + + def sampled?(_, 0.0), do: false + def sampled?(trace_id, sample_rate) when is_integer(trace_id) do threshold = sample_rate * @external_max_id ((trace_id * @knuth_factor) % @external_max_id) <= threshold From 31d0a9ea29e7ed52ec3dff6326d5bc7388792c32 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 11 Aug 2022 16:18:54 -0500 Subject: [PATCH 21/44] Use float --- test/api_server_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/api_server_test.exs b/test/api_server_test.exs index b7b30eb..f61f3dd 100644 --- a/test/api_server_test.exs +++ b/test/api_server_test.exs @@ -202,7 +202,7 @@ defmodule SpandexDatadog.ApiServerTest do "env" => "local" }, "metrics" => %{ - "_dd1.sr.eausr" => 1, + "_dd1.sr.eausr" => 1.0, "_sampling_priority_v1" => 2 }, "name" => "bar", @@ -292,7 +292,7 @@ defmodule SpandexDatadog.ApiServerTest do "env" => "local" }, "metrics" => %{ - "_dd1.sr.eausr" => 1, + "_dd1.sr.eausr" => 1.0, "_sampling_priority_v1" => 2 }, "name" => "bar", From 27f9d9e7215f7975a7626e6d1b0a03075e927acb Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 11 Aug 2022 16:45:38 -0500 Subject: [PATCH 22/44] Finish rate sampler --- lib/spandex_datadog/rate_sampler.ex | 17 +++++++++-------- test/rate_sampler_test.exs | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 test/rate_sampler_test.exs diff --git a/lib/spandex_datadog/rate_sampler.ex b/lib/spandex_datadog/rate_sampler.ex index fe39d48..379ce21 100644 --- a/lib/spandex_datadog/rate_sampler.ex +++ b/lib/spandex_datadog/rate_sampler.ex @@ -1,11 +1,8 @@ defmodule SpandexDatadog.RateSampler do @moduledoc """ - Randomly sample a percentage of traces based on the trace_id. + Randomly sample a proportion of traces based on the trace_id. """ - # Max value for a trace_id that we generate. - @max_id Bitwise.bsl(1, 63) - 1 - # We only generate 63-bit integers due to limitations in other languages, but # support parsing 64-bit integers for distributed tracing since an upstream # system may generate one. @@ -23,17 +20,21 @@ defmodule SpandexDatadog.RateSampler do `sample_rate` is a float between 0.0 and 1.0. 0.0 means that no trace will be sampled, and 1.0 means that all traces will be sampled. """ - @spec sampled?(non_neg_integer() | Spandex.Trace.t() | Spandex.Span.t(), float()) :: boolean() + @spec sampled?(non_neg_integer() | nil | Spandex.Trace.t() | Spandex.Span.t(), float()) :: boolean() + # trace_id may be nil when tracing is disabled + def sampled?(nil, 1.0), do: false + + # Shortcut processing for common cases def sampled?(_, 1.0), do: true def sampled?(_, 0.0), do: false def sampled?(trace_id, sample_rate) when is_integer(trace_id) do - threshold = sample_rate * @external_max_id - ((trace_id * @knuth_factor) % @external_max_id) <= threshold + threshold = trunc(sample_rate * @external_max_id) + rem(trace_id * @knuth_factor, @external_max_id) <= threshold end - def sampled?(%Spandex.Trace{trace_id: trace_id}, sample_rate) do + def sampled?(%Spandex.Trace{id: trace_id}, sample_rate) do sampled?(trace_id, sample_rate) end diff --git a/test/rate_sampler_test.exs b/test/rate_sampler_test.exs new file mode 100644 index 0000000..4fe7aeb --- /dev/null +++ b/test/rate_sampler_test.exs @@ -0,0 +1,25 @@ +defmodule SpandexDatadog.Test.RateSamplerTest do + use ExUnit.Case, async: true + + alias SpandexDatadog.RateSampler + + describe "sampled?/2" do + test "sample rate of 0.0 returns false" do + refute RateSampler.sampled?(1, 0.0) + end + + test "sample rate of 1.0 returns true" do + assert RateSampler.sampled?(1, 1.0) + end + + test "nil trace_id returns false" do + refute RateSampler.sampled?(nil, 1.0) + end + + test "trace is sampled" do + trace_id = trunc(Bitwise.bsl(1, 64) / 2) + refute RateSampler.sampled?(trace_id, 0.25) + assert RateSampler.sampled?(trace_id, 0.75) + end + end +end From 132961cee70267f1bbe391f523a291cf3420e5ae Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Fri, 12 Aug 2022 10:24:26 -0500 Subject: [PATCH 23/44] Fix param --- lib/spandex_datadog/rate_sampler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spandex_datadog/rate_sampler.ex b/lib/spandex_datadog/rate_sampler.ex index 379ce21..ed940a5 100644 --- a/lib/spandex_datadog/rate_sampler.ex +++ b/lib/spandex_datadog/rate_sampler.ex @@ -22,7 +22,7 @@ defmodule SpandexDatadog.RateSampler do """ @spec sampled?(non_neg_integer() | nil | Spandex.Trace.t() | Spandex.Span.t(), float()) :: boolean() # trace_id may be nil when tracing is disabled - def sampled?(nil, 1.0), do: false + def sampled?(nil, _), do: false # Shortcut processing for common cases def sampled?(_, 1.0), do: true From 26f3c03905c755822445d7689e3178021d193af3 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Fri, 19 Aug 2022 18:00:29 -0500 Subject: [PATCH 24/44] Update changelog --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45cc293..586e828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ See [Conventional Commits](Https://conventionalcommits.org) for commit guideline +### Breaking Changes: +* Change default `priority` to `nil` (not set) instead of 1 +* Do not send metric for `priority` unless set +* Do not set `x-datadog-sampling-priority` header unless priority is set +* Use `sample_rate` tag to set `_dd1.sr.eausr` value +* Use `rule_sample_rate` tag to set `_dd.rule_psr` without default (previously hard coded to 1.0) +* Use `rate_limiter_rate` tag to set `_dd.limit_psr`, without default (previously hard coded to 1.0) +* Update library deps + +### Features: +* Handle `x-datadog-origin` header +* Use `agent_sample_rate` tag to set `_dd.agent_psr`, without default (was 1.0) +* Add sampler which samples a proportion of traces, setting `priority` + ## [1.2.0](https://github.com/spandex-project/spandex_datadog/compare/1.1.0...1.2.0) (2021-10-23) ### Features: From 2f7c7c362355d45bae44f5330e95395244ce08f3 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Mon, 22 Aug 2022 11:22:24 -0500 Subject: [PATCH 25/44] Update docs --- README.md | 111 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index af2d977..a53a6d7 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,11 @@ [![License](https://img.shields.io/hexpm/l/spandex_datadog.svg)](https://github.com/spandex-project/spandex_datadog/blob/master/LICENSE) [![Last Updated](https://img.shields.io/github/last-commit/spandex-project/spandex_datadog.svg)](https://github.com/spandex-project/spandex_datadog/commits/master) -A datadog adapter for the `:spandex` library. +This is the Datadog adapter for the [Spandex](https://github.com/spandex-project/spandex) tracing library. ## Installation -The package can be installed by adding `:spandex_datadog` to your list of -dependencies in `mix.exs`: +Add `:spandex_datadog` to the list of dependencies in `mix.exs`: ```elixir def deps do @@ -21,14 +20,13 @@ def deps do end ``` -To start the datadog adapter, add a worker to your application's supervisor +To start the Datadog adapter, add a worker to your application's supervisor. ```elixir # Example configuration -# Note: You should put ApiServer before any other children in the list that -# might try to send traces before the ApiServer has started up, for example -# Ecto.Repo and Phoenix.Endpoint +# Note: Put ApiServer before other children that might try to send traces +# before ApiServer has started up, e.g. Ecto.Repo and Phoenix.Endpoint spandex_opts = [ @@ -53,15 +51,73 @@ Supvervisor.start_link(children, opts) ## Distributed Tracing -Distributed tracing is supported via headers `x-datadog-trace-id`, -`x-datadog-parent-id`, and `x-datadog-sampling-priority`. If they are set, the -`Spandex.Plug.StartTrace` plug will act accordingly, continuing that trace and -span instead of starting a new one. *Both* `x-datadog-trace-id` and -`x-datadog-parent-id` must be set for distributed tracing to work. You can -learn more about the behavior of `x-datadog-sampling-priority` in the [Datadog -priority sampling documentation]. +In distributed tracing, multiple processes contribute to the same trace. -[Datadog priority sampling documentation]: https://docs.datadoghq.com/tracing/getting_further/trace_sampling_and_storage/#priority-sampling-for-distributed-tracing +That is handled in this adapter by `SpandexDatadog.Adapter.distributed_context/2` and +`SpandexDatadog.Adapter.inject_context/3`, which read and generate Datadog-specific +HTTP headers based on the state of the trace. If these headers are set, the +trace will be continued instead of starting a new one. +They are called by [spandex_phoenix](https://github.com/spandex-project/spandex_phoenix) +when receiving a request and by your own HTTP requests to downstream services. + +## Sampling and Rate Limiting + +When the load or cost from tracing increases, it is useful to use sampling or +rate limiting to reduce tracing. When many traces are the same, it's enough to +trace only e.g. 10% of them, reducing the bill by 90% while still preserving +the ability to troubleshoot the system. + +With sampling, the tracing still happens, but Datadog may drop it or not +retain detailed information. This keeps metrics such as the number of requests +correct, even without detailed trace data. + +This adapter supports distributed trace sampling according to the new Datadog +[ingestion controls](https://docs.datadoghq.com/tracing/trace_pipeline/ingestion_controls/). + +Spandex stores the `priority` as an integer in the top level `Spandex.Trace`. + +In Datadog, there are four values: +* `USER_KEEP`(2) indicates that the application wants to ensure that a trace is + sampled, e.g. if there is an error +* `AUTO_KEEP` (1) indicates that a trace has been selected for sampling +* `AUTO_REJECT` (0) indicates that the trace has not been selected for sampling +* `USER_REJECT` (-1) indicates that the application wants a trace to be dropped + +Datadog uses the `x-datadog-sampling-priority` header to determine whether a +trace should be sampled. See the [Datadog documentation]: +https://docs.datadoghq.com/tracing/getting_further/trace_sampling_and_storage/#priority-sampling-for-distributed-tracing + +When sampling, the process that starts the trace can make a decision about +whether it should be sampled. It then passes that information to downstream +processes via the HTTP headers. + +In distributed tracing, the priority may be set as follows: + +* Set the priority based on the `x-datadog-sampling-priority` header on the + inbound request, propagating it to downstream http requests. + +* If no priority header is present (i.e. this is the first app in the trace), + make a sampling decision and set the `priority` to 0 or 1 on the current trace. + `SpandexDatadog.RateSampler.sampled?/2` supports sampling a proportion of + traces based on the `trace_id` (which should be assigned randomly). This would + typically be called from in a Phoenix `plug`, which then sets the priority to 1 + on the current trace using Spandex.Tracer.update_priority/2`. + +* Force tracking manually by setting priority to 2 on the trace in application + code. This is usually done for requests with errors, as they are the ones + that need troubleshooting. You can also enable tracing dynamically with a + feature flag to debug a feature in production. This overrides whatever sampling + decision was made upstream, and Datadog will keep the complete trace. + +Older versions of Datadog used rate sampling to tune the priority of spans, but +those are considered obsolete. You can still set them by adding tags to spans: + +* Use `sample_rate` tag to set "_dd1.sr.eausr" +* Use `rule_sample_rate` tag to set "_dd.rule_psr" without default (previously + hard coded to 1.0) +* Use `rate_limiter_rate` tag to set "_dd.limit_psr", without default + (previously hard coded to 1.0) +* Use `agent_sample_rate` tag to set "_dd.agent_psr" without default (was 1.0) ## Telemetry @@ -145,22 +201,23 @@ per trace, then you probably want to keep that number low. If you send only a few spans, then you could set it significantly higher. Sync threshold is how many _simultaneous_ HTTP pushes will be going to Datadog -before it blocks/throttles your application by making the tracing call synchronous instead of async. +before it blocks/throttles your application by making the tracing call +synchronous instead of async. -Ideally, the sync threshold would be set to a point that you wouldn't reasonably reach often, but -that is low enough to not cause systemic performance issues if you don't apply +Ideally, the sync threshold would be set to a point that you wouldn't +reasonably reach often, but that is low enough to not cause systemic +performance issues if you don't apply backpressure. -A simple way to think about it is that if you are seeing 1000 -request per second and `batch_size` is 10, then you'll be making 100 -requests per second to Datadog (probably a bad config). -With `sync_threshold` set to 10, only 10 of those requests can be -processed concurrently before trace calls become synchronous. - -This concept of backpressure is very important, and strategies -for switching to synchronous operation are often surprisingly far more -performant than purely asynchronous strategies (and much more predictable). +A simple way to think about it is that if you are seeing 1000 request per +second and `batch_size` is 10, then you'll be making 100 requests per second to +Datadog (probably a bad config). With `sync_threshold` set to 10, only 10 of +those requests can be processed concurrently before trace calls become +synchronous. +This concept of backpressure is very important, and strategies for switching to +synchronous operation are often surprisingly far more performant than purely +asynchronous strategies (and much more predictable). ## Copyright and License From f1e64a2d89155d057d32bdac352859c401042b22 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Mon, 22 Aug 2022 12:20:23 -0500 Subject: [PATCH 26/44] Fix quotes --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a53a6d7..335798c 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ In distributed tracing, the priority may be set as follows: `SpandexDatadog.RateSampler.sampled?/2` supports sampling a proportion of traces based on the `trace_id` (which should be assigned randomly). This would typically be called from in a Phoenix `plug`, which then sets the priority to 1 - on the current trace using Spandex.Tracer.update_priority/2`. + on the current trace using `Spandex.Tracer.update_priority/2`. * Force tracking manually by setting priority to 2 on the trace in application code. This is usually done for requests with errors, as they are the ones @@ -112,12 +112,12 @@ In distributed tracing, the priority may be set as follows: Older versions of Datadog used rate sampling to tune the priority of spans, but those are considered obsolete. You can still set them by adding tags to spans: -* Use `sample_rate` tag to set "_dd1.sr.eausr" -* Use `rule_sample_rate` tag to set "_dd.rule_psr" without default (previously +* Use `sample_rate` tag to set `_dd1.sr.eausr` +* Use `rule_sample_rate` tag to set `_dd.rule_psr` without default (previously hard coded to 1.0) -* Use `rate_limiter_rate` tag to set "_dd.limit_psr", without default +* Use `rate_limiter_rate` tag to set `_dd.limit_psr`, without default (previously hard coded to 1.0) -* Use `agent_sample_rate` tag to set "_dd.agent_psr" without default (was 1.0) +* Use `agent_sample_rate` tag to set `_dd.agent_psr` without default (was 1.0) ## Telemetry From e0618f2fcbe678deab1345ffc526cc015ac07ae4 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Mon, 22 Aug 2022 12:51:58 -0500 Subject: [PATCH 27/44] Use new names for priority constants --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 335798c..381d0ec 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,11 @@ This adapter supports distributed trace sampling according to the new Datadog Spandex stores the `priority` as an integer in the top level `Spandex.Trace`. In Datadog, there are four values: -* `USER_KEEP`(2) indicates that the application wants to ensure that a trace is +* `MANUAL_KEEP`(2) indicates that the application wants to ensure that a trace is sampled, e.g. if there is an error * `AUTO_KEEP` (1) indicates that a trace has been selected for sampling * `AUTO_REJECT` (0) indicates that the trace has not been selected for sampling -* `USER_REJECT` (-1) indicates that the application wants a trace to be dropped +* `MANUAL_REJECT` (-1) indicates that the application wants a trace to be dropped Datadog uses the `x-datadog-sampling-priority` header to determine whether a trace should be sampled. See the [Datadog documentation]: From dac254a44ceaaa6fd3f7bafacf96b5d56ec9c7de Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Mon, 22 Aug 2022 12:52:13 -0500 Subject: [PATCH 28/44] Add constants for priority --- lib/spandex_datadog/constants.ex | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 lib/spandex_datadog/constants.ex diff --git a/lib/spandex_datadog/constants.ex b/lib/spandex_datadog/constants.ex new file mode 100644 index 0000000..06296dc --- /dev/null +++ b/lib/spandex_datadog/constants.ex @@ -0,0 +1,31 @@ +defmodule SpandexDatadog.Constants do + @moduledoc """ + Defines constants used in Datadog API. + """ + + @doc """ + `priority` value `MANUAL_KEEP` (2) indicates that the application wants to ensure + that a trace is sampled, e.g. if there is an error. + + See https://docs.datadoghq.com/tracing/getting_further/trace_sampling_and_storage/#priority-sampling-for-distributed-tracing + """ + def manual_keep, do: 2 + + @doc """ + `priority` value `AUTO_KEEP` (1) indicates that a trace has been selected for + sampling. + """ + def auto_keep, do: 1 + + @doc """ + `priority` value `AUTO_REJECT` (0) indicates that the trace has not been + selected for sampling. + """ + def auto_reject, do: 1 + + @doc """ + `priority` value `MANUAL_REJECT` (-1) indicates that the application wants a + trace to be dropped. + """ + def manual_reject, do: -1 +end From 5b3a75801dc5fbe05b4415c22ca0260be1d71d7f Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 23 Aug 2022 13:26:56 -0500 Subject: [PATCH 29/44] Doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 381d0ec..2db302c 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ That is handled in this adapter by `SpandexDatadog.Adapter.distributed_context/2 `SpandexDatadog.Adapter.inject_context/3`, which read and generate Datadog-specific HTTP headers based on the state of the trace. If these headers are set, the trace will be continued instead of starting a new one. -They are called by [spandex_phoenix](https://github.com/spandex-project/spandex_phoenix) +These functions are called in [spandex_phoenix](https://github.com/spandex-project/spandex_phoenix) when receiving a request and by your own HTTP requests to downstream services. ## Sampling and Rate Limiting From bd2f7ac58c1ec84bcc9fff483df6335611519f15 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 23 Aug 2022 13:27:22 -0500 Subject: [PATCH 30/44] Add GitHub Actions --- .github/workflows/ci.yml | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b28a96a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,61 @@ +name: CI +on: push + +jobs: + elixir: + name: Elixir Tests + runs-on: ubuntu-20.04 + + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.5.0 + with: + access_token: ${{ github.token }} + + - name: Checkout Code + uses: actions/checkout@v2.3.1 + + - name: Setup Elixir + uses: actions/setup-elixir@v1.5.0 + with: + otp-version: 24.3.4 + elixir-version: 1.13.4 + experimental-otp: true + + - name: Retrieve Cached Dependencies + id: mix-cache + uses: actions/cache@v2 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: | + ${{ runner.os }}-mix- + + - name: Install Dependencies + run: | + mix local.rebar --force + mix local.hex --force + mix deps.get + + - name: Check Formatting + run: mix format --check-formatted + + - name: Compile Code + env: + MIX_ENV: test + run: | + mix compile --warnings-as-errors + + - name: Run Tests + run: | + # mix test --warnings-as-errors + mix test + + # - name: Publish unit test results to GitHub + # uses: EnricoMi/publish-unit-test-result-action@v1 + # if: always() # always run even if tests fail + # with: + # files: _build/test/lib/*/test-junit-report.xml + From 2aa894bf32711af370562ba62036f68a1843ebab Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 23 Aug 2022 13:32:38 -0500 Subject: [PATCH 31/44] mix format --- lib/spandex_datadog/rate_sampler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spandex_datadog/rate_sampler.ex b/lib/spandex_datadog/rate_sampler.ex index ed940a5..90dba25 100644 --- a/lib/spandex_datadog/rate_sampler.ex +++ b/lib/spandex_datadog/rate_sampler.ex @@ -8,7 +8,7 @@ defmodule SpandexDatadog.RateSampler do # system may generate one. @external_max_id Bitwise.bsl(1, 64) - @knuth_factor 1111111111111111111 + @knuth_factor 1_111_111_111_111_111_111 @doc """ Determine whether a trace should be sampled. From a4c87d146aa719a34fdc287c8103984266c7366a Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 23 Aug 2022 14:02:05 -0500 Subject: [PATCH 32/44] Enable --warnings-as-errors --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b28a96a..f80f6d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,8 +50,7 @@ jobs: - name: Run Tests run: | - # mix test --warnings-as-errors - mix test + mix test --warnings-as-errors # - name: Publish unit test results to GitHub # uses: EnricoMi/publish-unit-test-result-action@v1 From 418834d6828de08a77aa08e4cff37b321e0139b3 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 23 Aug 2022 14:44:55 -0500 Subject: [PATCH 33/44] Add :file and :line to logger metadata for tests --- config/test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 526e7bd..f00696b 100644 --- a/config/test.exs +++ b/config/test.exs @@ -4,7 +4,7 @@ config :logger, :console, level: :debug, colors: [enabled: false], format: "$time $metadata[$level] $message\n", - metadata: [:trace_id, :span_id] + metadata: [:trace_id, :span_id, :file, :line] config :spandex_datadog, SpandexDatadog.Test.Support.Tracer, service: :spandex_test, From 257451ccb301bb1fe4b25278e4ca443558dade15 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 23 Aug 2022 15:12:08 -0500 Subject: [PATCH 34/44] Run mix format check after deps.get --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 343e4dd..3db92d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: - checkout - run: mix local.hex --force - run: mix local.rebar --force - - run: mix format --check-formatted - run: mix deps.get + - run: mix format --check-formatted - run: mix compile --warnings-as-errors - run: mix test From 78c01a30f84d3e36151dc383ccd1fbc0fda6672e Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Tue, 23 Aug 2022 15:16:38 -0500 Subject: [PATCH 35/44] Require Elixir 1.11 --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 34819b1..1e2f586 100644 --- a/mix.exs +++ b/mix.exs @@ -10,7 +10,7 @@ defmodule SpandexDatadog.MixProject do deps: deps(), description: "A datadog API adapter for spandex.", docs: docs(), - elixir: "~> 1.6", + elixir: "~> 1.11", elixirc_paths: elixirc_paths(Mix.env()), package: package(), start_permanent: Mix.env() == :prod, From 968aa6b4e229d4b5d7404ca12a40f66d50786d43 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Wed, 24 Aug 2022 13:38:17 -0500 Subject: [PATCH 36/44] Test older versions of Elixir --- .github/workflows/ci.yml | 53 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f80f6d9..8822909 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,4 @@ +--- name: CI on: push @@ -5,56 +6,56 @@ jobs: elixir: name: Elixir Tests runs-on: ubuntu-20.04 - + env: + MIX_ENV: test + strategy: + matrix: + elixir: ['1.11', '1.13.4'] + otp: ['24', '25'] steps: - - name: Cancel Previous Runs + - name: Cancel previous runs uses: styfle/cancel-workflow-action@0.5.0 with: access_token: ${{ github.token }} - - name: Checkout Code - uses: actions/checkout@v2.3.1 + - name: Checkout code + uses: actions/checkout@v2 - name: Setup Elixir - uses: actions/setup-elixir@v1.5.0 + uses: actions/setup-beam@v1 + with: + otp-version: ${{ matrix.otp }} + elixir-version: ${{ matrix.otp }} + + - name: Get deps cache + uses: actions/cache@v2 with: - otp-version: 24.3.4 - elixir-version: 1.13.4 - experimental-otp: true + path: deps/ + key: deps-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('**/mix.lock') }} - - name: Retrieve Cached Dependencies - id: mix-cache + - name: Get build cache uses: actions/cache@v2 with: - path: | - deps - _build - key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} - restore-keys: | - ${{ runner.os }}-mix- - - - name: Install Dependencies + path: _build/test/ + key: build-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('**/mix.lock') }} + + - name: Install deps run: | mix local.rebar --force mix local.hex --force mix deps.get - - name: Check Formatting - run: mix format --check-formatted - - - name: Compile Code - env: - MIX_ENV: test + - name: Compile code run: | mix compile --warnings-as-errors - - name: Run Tests + - name: Run tests run: | mix test --warnings-as-errors + mix format --check-formatted # - name: Publish unit test results to GitHub # uses: EnricoMi/publish-unit-test-result-action@v1 # if: always() # always run even if tests fail # with: # files: _build/test/lib/*/test-junit-report.xml - From ba93460bd2ec7166ea8cb2f4c4ce1fde267a749f Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Wed, 24 Aug 2022 15:30:48 -0500 Subject: [PATCH 37/44] Use setup-elixir instead of setup-beam --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8822909..05739fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@v2 - name: Setup Elixir - uses: actions/setup-beam@v1 + uses: actions/setup-elixir@v1 with: otp-version: ${{ matrix.otp }} elixir-version: ${{ matrix.otp }} From 4ace750173ad586826e5999803c1a28264f46f20 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Wed, 24 Aug 2022 15:37:57 -0500 Subject: [PATCH 38/44] Use newer version of setup-elixir --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05739fb..6fcc67f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,10 +22,11 @@ jobs: uses: actions/checkout@v2 - name: Setup Elixir - uses: actions/setup-elixir@v1 + uses: actions/setup-elixir@v1.5.0 with: otp-version: ${{ matrix.otp }} elixir-version: ${{ matrix.otp }} + experimental-otp: true - name: Get deps cache uses: actions/cache@v2 @@ -55,7 +56,7 @@ jobs: mix format --check-formatted # - name: Publish unit test results to GitHub - # uses: EnricoMi/publish-unit-test-result-action@v1 + # uses: EnricoMi/publish-unit-test-result-action@v2 # if: always() # always run even if tests fail # with: - # files: _build/test/lib/*/test-junit-report.xml + # junit_files: _build/test/lib/*/test-junit-report.xml From 2c22a7ff1b4f16714bdca22cf1a84553877d776f Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Wed, 24 Aug 2022 15:55:56 -0500 Subject: [PATCH 39/44] Specify detailed versions of Elixir and OTP --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fcc67f..3d1ab21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,8 @@ jobs: MIX_ENV: test strategy: matrix: - elixir: ['1.11', '1.13.4'] - otp: ['24', '25'] + elixir: ['1.11.4', '1.13.4'] + otp: ['24.3.4', '25.0.4'] steps: - name: Cancel previous runs uses: styfle/cancel-workflow-action@0.5.0 From 7d903be4820d2b8358b9425817a3ee90b6b405f3 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Wed, 24 Aug 2022 16:02:50 -0500 Subject: [PATCH 40/44] Fix variable --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d1ab21..03dd043 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: elixir: ['1.11.4', '1.13.4'] - otp: ['24.3.4', '25.0.4'] + otp: ['24.3', '25.0'] steps: - name: Cancel previous runs uses: styfle/cancel-workflow-action@0.5.0 @@ -25,7 +25,7 @@ jobs: uses: actions/setup-elixir@v1.5.0 with: otp-version: ${{ matrix.otp }} - elixir-version: ${{ matrix.otp }} + elixir-version: ${{ matrix.elixir }} experimental-otp: true - name: Get deps cache From dd69b3003c66a46dd8bba93a5a7785d8b056a94a Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 25 Aug 2022 11:30:03 -0500 Subject: [PATCH 41/44] Use erlef/setup-beam; only build on OTP 24 --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03dd043..cb231f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: elixir: ['1.11.4', '1.13.4'] - otp: ['24.3', '25.0'] + otp: ['24.3'] steps: - name: Cancel previous runs uses: styfle/cancel-workflow-action@0.5.0 @@ -22,11 +22,10 @@ jobs: uses: actions/checkout@v2 - name: Setup Elixir - uses: actions/setup-elixir@v1.5.0 + uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.otp }} elixir-version: ${{ matrix.elixir }} - experimental-otp: true - name: Get deps cache uses: actions/cache@v2 From 4d018cbbbf4f7beebe290ec608e383f08ff9039c Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 25 Aug 2022 11:42:02 -0500 Subject: [PATCH 42/44] Remove invalid --warnings-as-errors option --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb231f2..79a6381 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: - name: Run tests run: | - mix test --warnings-as-errors + mix test mix format --check-formatted # - name: Publish unit test results to GitHub From 6fd650dede4da13994974abd8466473fa60ef652 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Thu, 25 Aug 2022 11:44:28 -0500 Subject: [PATCH 43/44] Use latest syntax --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3db92d5..34a931c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ -# Elixir CircleCI 2.0 configuration file -# +--- +# CircleCI 2.0 configuration file # Check https://circleci.com/docs/2.0/language-elixir/ for more details -version: 2 +version: 2.1 jobs: build: docker: From 71cda32bb2f9bf9a48f078b8a4b9e8b3a997d7e9 Mon Sep 17 00:00:00 2001 From: Jake Morrison Date: Fri, 9 Sep 2022 03:33:10 +0800 Subject: [PATCH 44/44] Update lib/spandex_datadog/adapter.ex Co-authored-by: Greg Mefford --- lib/spandex_datadog/adapter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spandex_datadog/adapter.ex b/lib/spandex_datadog/adapter.ex index 70499cb..30a30e0 100644 --- a/lib/spandex_datadog/adapter.ex +++ b/lib/spandex_datadog/adapter.ex @@ -112,7 +112,7 @@ defmodule SpandexDatadog.Adapter do end # Based on the code of the Ruby libary, it seems valid - # to have onlly the trace_id and origin headers set, but not parent_id. + # to have only the trace_id and origin headers set, but not parent_id. # This might happen with RUM or synthetic traces. defp validate_context(%{parent_id: nil} = span_context) do if Keyword.has_key?(span_context.baggage, :"_dd.origin") do