diff --git a/packages/sync-service/config/runtime.exs b/packages/sync-service/config/runtime.exs index 2203fbe3d8..b1b6b79f2b 100644 --- a/packages/sync-service/config/runtime.exs +++ b/packages/sync-service/config/runtime.exs @@ -82,8 +82,22 @@ otel_simple_processor = {:otel_simple_processor, %{exporter: {:otel_exporter_stdout, []}}} end +otel_sampling_ratio = env!("ELECTRIC_OTEL_SAMPLING_RATIO", :float, 0.01) + config :opentelemetry, - processors: [otel_batch_processor, otel_simple_processor] |> Enum.reject(&is_nil/1) + processors: [otel_batch_processor, otel_simple_processor] |> Enum.reject(&is_nil/1), + # sampler: {Electric.Telemetry.Sampler, %{ratio: otel_sampling_ratio}} + # Sample root spans based on our custom sampler + # and inherit sampling decision from remote parents + sampler: + {:parent_based, + %{ + root: {Electric.Telemetry.Sampler, %{ratio: otel_sampling_ratio}}, + remote_parent_sampled: :always_on, + remote_parent_not_sampled: :always_off, + local_parent_sampled: :always_on, + local_parent_not_sampled: :always_off + }} database_url = env!("DATABASE_URL", :string!) diff --git a/packages/sync-service/lib/electric/telemetry/sampler.ex b/packages/sync-service/lib/electric/telemetry/sampler.ex new file mode 100644 index 0000000000..eb27e1b79c --- /dev/null +++ b/packages/sync-service/lib/electric/telemetry/sampler.ex @@ -0,0 +1,42 @@ +defmodule Electric.Telemetry.Sampler do + @moduledoc """ + Custom sampler that samples all spans except for specifically configured spans for which a given ratio is sampled. + """ + + require OpenTelemetry.Tracer, as: Tracer + + @behaviour :otel_sampler + + # Span names that are sampled probabilistically + @probabilistic_span_names [ + "pg_txn.replication_client.process_x_log_data" + ] + + @impl :otel_sampler + def setup(%{ratio: ratio}) do + %{sampling_probability: ratio} + end + + @impl :otel_sampler + def description(%{sampling_probability: sampling_probability}) do + "Custom sampler that samples all spans except for specifically configured spans for which #{sampling_probability * 100}% are sampled." + end + + @impl true + def should_sample(ctx, _trace_id, _links, span_name, _span_kind, _attributes, %{ + sampling_probability: sampling_probability + }) do + tracestate = Tracer.current_span_ctx(ctx) |> OpenTelemetry.Span.tracestate() + + if span_name in @probabilistic_span_names do + if :rand.uniform() <= sampling_probability do + {:record_and_sample, [], tracestate} + else + {:drop, [], tracestate} + end + else + # Always sample other spans + {:record_and_sample, [], tracestate} + end + end +end