diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 2f293a93b3..7f9771ece9 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -118,6 +118,11 @@ var ( type config struct { *knobs.Scope + dynamic struct { + // enabled reports whether tracing is enabled. + enabled dynamicConfig[bool] + } + // debug, when true, writes details to logs. debug bool @@ -233,9 +238,6 @@ type config struct { // profilerEndpoints specifies whether profiler endpoint filtering is enabled. profilerEndpoints bool - // enabled reports whether tracing is enabled. - enabled dynamicConfig[bool] - // enableHostnameDetection specifies whether the tracer should enable hostname detection. enableHostnameDetection bool @@ -289,6 +291,36 @@ type config struct { } var ( + enabled = knobs.Register(&knobs.Definition[bool]{ + Default: true, + EnvVars: []knobs.EnvVar{ + { + Key: "DD_TRACE_ENABLED", + }, + { + Key: "OTEL_TRACES_EXPORTER", + Transform: func(s string) string { + v, err := mapEnabled(s) + if err != nil { + log.Warn("ignoring OTEL_TRACES_EXPORTER=%q, error: %v", s, err) + return "" + } + return v + }, + }, + }, + Parse: func(s string) (bool, error) { + if s == "" { + return true, nil + } + v, err := strconv.ParseBool(s) + if err != nil { + return true, err + } + return v, nil + }, + }) + // globalSampleRate holds sample rate read from environment variables. globalSampleRate = knobs.Register(&knobs.Definition[float64]{ Default: math.NaN(), @@ -443,9 +475,10 @@ func newConfig(opts ...StartOption) *config { c.runtimeMetricsV2 = internal.BoolEnv("DD_RUNTIME_METRICS_V2_ENABLED", false) c.debug = internal.BoolVal(getDDorOtelConfig("debugMode"), false) c.logDirectory = os.Getenv("DD_TRACE_LOG_DIRECTORY") - c.enabled = newDynamicConfig("tracing_enabled", internal.BoolVal(getDDorOtelConfig("enabled"), true), func(b bool) bool { return true }, equal[bool]) + c.dynamic.enabled = newDynamicConfig("tracing_enabled", knobs.GetScope(c.Scope, enabled), func(b bool) bool { knobs.SetScope(c.Scope, enabled, knobs.Code, b); return true }, equal[bool]) if _, ok := os.LookupEnv("DD_TRACE_ENABLED"); ok { - c.enabled.cfgOrigin = telemetry.OriginEnvVar + // TODO: shouldn't we track this for the OTEL_TRACES_EXPORTER case? + c.dynamic.enabled.cfgOrigin = telemetry.OriginEnvVar } c.profilerEndpoints = internal.BoolEnv(traceprof.EndpointEnvVar, true) c.profilerHotspots = internal.BoolEnv(traceprof.CodeHotspotsEnvVar, true) @@ -560,7 +593,7 @@ func newConfig(opts ...StartOption) *config { log.SetLevel(log.LevelDebug) } // if using stdout or traces are disabled, agent is disabled - agentDisabled := c.logToStdout || !c.enabled.current + agentDisabled := c.logToStdout || !knobs.GetScope(c.Scope, enabled) c.agent = loadAgentFeatures(agentDisabled, c.agentURL, c.httpClient) info, ok := debug.ReadBuildInfo() if !ok { @@ -1162,7 +1195,7 @@ func WithHostname(name string) StartOption { // WithTraceEnabled allows specifying whether tracing will be enabled func WithTraceEnabled(enabled bool) StartOption { return func(c *config) { - c.enabled = newDynamicConfig("tracing_enabled", enabled, func(b bool) bool { return true }, equal[bool]) + c.dynamic.enabled.update(enabled, telemetry.OriginCode) } } diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index 35a150d058..acc3f0d728 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -637,8 +637,9 @@ func TestTracerOptionsDefaults(t *testing.T) { tracer := newTracer(WithAgentTimeout(2)) defer tracer.Stop() c := tracer.config - assert.True(t, c.enabled.current) - assert.Equal(t, c.enabled.cfgOrigin, telemetry.OriginDefault) + assert.True(t, c.dynamic.enabled.current) + assert.True(t, knobs.GetScope(c.Scope, enabled)) + assert.Equal(t, c.dynamic.enabled.cfgOrigin, telemetry.OriginDefault) }) t.Run("override", func(t *testing.T) { @@ -646,8 +647,9 @@ func TestTracerOptionsDefaults(t *testing.T) { tracer := newTracer(WithAgentTimeout(2)) defer tracer.Stop() c := tracer.config - assert.False(t, c.enabled.current) - assert.Equal(t, c.enabled.cfgOrigin, telemetry.OriginEnvVar) + assert.False(t, c.dynamic.enabled.current) + assert.False(t, knobs.GetScope(c.Scope, enabled)) + assert.Equal(t, c.dynamic.enabled.cfgOrigin, telemetry.OriginEnvVar) }) }) @@ -1365,21 +1367,24 @@ func TestWithTraceEnabled(t *testing.T) { t.Run("WithTraceEnabled", func(t *testing.T) { assert := assert.New(t) c := newConfig(WithTraceEnabled(false)) - assert.False(c.enabled.current) + assert.False(c.dynamic.enabled.current) + assert.False(knobs.GetScope(c.Scope, enabled)) }) t.Run("otel-env", func(t *testing.T) { assert := assert.New(t) t.Setenv("OTEL_TRACES_EXPORTER", "none") c := newConfig() - assert.False(c.enabled.current) + assert.False(c.dynamic.enabled.current) + assert.False(knobs.GetScope(c.Scope, enabled)) }) t.Run("dd-env", func(t *testing.T) { assert := assert.New(t) t.Setenv("DD_TRACE_ENABLED", "false") c := newConfig() - assert.False(c.enabled.current) + assert.False(c.dynamic.enabled.current) + assert.False(knobs.GetScope(c.Scope, enabled)) }) t.Run("override-chain", func(t *testing.T) { @@ -1388,10 +1393,12 @@ func TestWithTraceEnabled(t *testing.T) { t.Setenv("OTEL_TRACES_EXPORTER", "none") t.Setenv("DD_TRACE_ENABLED", "true") c := newConfig() - assert.True(c.enabled.current) + assert.True(c.dynamic.enabled.current) + assert.True(knobs.GetScope(c.Scope, enabled)) // tracer option overrides dd env c = newConfig(WithTraceEnabled(false)) - assert.False(c.enabled.current) + assert.False(c.dynamic.enabled.current) + assert.False(knobs.GetScope(c.Scope, enabled)) }) } diff --git a/ddtrace/tracer/remote_config.go b/ddtrace/tracer/remote_config.go index eaa8641730..1809bd9395 100644 --- a/ddtrace/tracer/remote_config.go +++ b/ddtrace/tracer/remote_config.go @@ -181,7 +181,7 @@ func (t *tracer) onRemoteConfigUpdate(u remoteconfig.ProductUpdate) map[string]s if updated { telemConfigs = append(telemConfigs, t.config.globalTags.toTelemetry()) } - if !t.config.enabled.current { + if !t.config.dynamic.enabled.current { log.Debug("APM Tracing is disabled. Restart the service to enable it.") } if len(telemConfigs) > 0 { @@ -233,11 +233,11 @@ func (t *tracer) onRemoteConfigUpdate(u remoteconfig.ProductUpdate) map[string]s telemConfigs = append(telemConfigs, t.config.globalTags.toTelemetry()) } if c.LibConfig.Enabled != nil { - if t.config.enabled.current == true && *c.LibConfig.Enabled == false { + if t.config.dynamic.enabled.current == true && *c.LibConfig.Enabled == false { log.Debug("Disabled APM Tracing through RC. Restart the service to enable it.") - t.config.enabled.handleRC(c.LibConfig.Enabled) - telemConfigs = append(telemConfigs, t.config.enabled.toTelemetry()) - } else if t.config.enabled.current == false && *c.LibConfig.Enabled == true { + t.config.dynamic.enabled.handleRC(c.LibConfig.Enabled) + telemConfigs = append(telemConfigs, t.config.dynamic.enabled.toTelemetry()) + } else if t.config.dynamic.enabled.current == false && *c.LibConfig.Enabled == true { log.Debug("APM Tracing is disabled. Restart the service to enable it.") } } diff --git a/ddtrace/tracer/remote_config_test.go b/ddtrace/tracer/remote_config_test.go index d64c2666fe..a6d703d272 100644 --- a/ddtrace/tracer/remote_config_test.go +++ b/ddtrace/tracer/remote_config_test.go @@ -18,6 +18,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry/telemetrytest" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" + "github.com/darccio/knobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -395,7 +396,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { applyStatus := tr.onRemoteConfigUpdate(input) require.Equal(t, state.ApplyStateAcknowledged, applyStatus["path"].State) - require.Equal(t, false, tr.config.enabled.current) + require.Equal(t, false, tr.config.dynamic.enabled.current) headers := TextMapCarrier{ traceparentHeader: "00-12345678901234567890123456789012-1234567890123456-01", tracestateHeader: "dd=s:2;o:rum;t.usr.id:baz64~~", @@ -417,7 +418,7 @@ func TestOnRemoteConfigUpdate(t *testing.T) { input = remoteconfig.ProductUpdate{"path": nil} applyStatus = tr.onRemoteConfigUpdate(input) require.Equal(t, state.ApplyStateAcknowledged, applyStatus["path"].State) - require.Equal(t, false, tr.config.enabled.current) + require.Equal(t, false, knobs.GetScope(tr.config.Scope, enabled)) // turning tracing back explicitly is not allowed input = remoteconfig.ProductUpdate{ @@ -427,7 +428,8 @@ func TestOnRemoteConfigUpdate(t *testing.T) { } applyStatus = tr.onRemoteConfigUpdate(input) require.Equal(t, state.ApplyStateAcknowledged, applyStatus["path"].State) - require.Equal(t, false, tr.config.enabled.current) + require.Equal(t, false, tr.config.dynamic.enabled.current) + require.Equal(t, false, knobs.GetScope(tr.config.Scope, enabled)) telemetryClient.AssertNumberOfCalls(t, "ConfigChange", 1) }) diff --git a/ddtrace/tracer/span.go b/ddtrace/tracer/span.go index ebfa681fb3..1fe8f9385c 100644 --- a/ddtrace/tracer/span.go +++ b/ddtrace/tracer/span.go @@ -32,6 +32,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames" "gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof" + "github.com/darccio/knobs" "github.com/tinylib/msgp/msgp" "golang.org/x/xerrors" @@ -548,7 +549,7 @@ func (s *span) finish(finishTime int64) { keep := true if t, ok := internal.GetGlobalTracer().(*tracer); ok { - if !t.config.enabled.current { + if !knobs.GetScope(t.config.Scope, enabled) { return } // we have an active tracer diff --git a/ddtrace/tracer/telemetry.go b/ddtrace/tracer/telemetry.go index bc9185858f..a132a3d521 100644 --- a/ddtrace/tracer/telemetry.go +++ b/ddtrace/tracer/telemetry.go @@ -9,6 +9,7 @@ import ( "fmt" "strings" + "github.com/darccio/knobs" "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" ) @@ -63,7 +64,7 @@ func startTelemetry(c *config) { {Name: "trace_span_attribute_schema", Value: c.spanAttributeSchemaVersion}, {Name: "trace_peer_service_defaults_enabled", Value: c.peerServiceDefaultsEnabled}, {Name: "orchestrion_enabled", Value: c.orchestrionCfg.Enabled}, - {Name: "trace_enabled", Value: c.enabled.current, Origin: c.enabled.cfgOrigin}, + {Name: "trace_enabled", Value: knobs.GetScope(c.Scope, enabled), Origin: c.dynamic.enabled.cfgOrigin}, {Name: "trace_log_directory", Value: c.logDirectory}, c.traceSampleRate.toTelemetry(), c.headerAsTags.toTelemetry(), diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 357ccd4c9c..d931b3e0c9 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -152,7 +152,7 @@ func Start(opts ...StartOption) { } defer telemetry.Time(telemetry.NamespaceGeneral, "init_time", nil, true)() t := newTracer(opts...) - if !t.config.enabled.current { + if !knobs.GetScope(t.config.Scope, enabled) { // TODO: instrumentation telemetry client won't get started // if tracing is disabled, but we still want to capture this // telemetry information. Will be fixed when the tracer and profiler @@ -500,7 +500,7 @@ func (t *tracer) pushChunk(trace *chunk) { // StartSpan creates, starts, and returns a new Span with the given `operationName`. func (t *tracer) StartSpan(operationName string, options ...ddtrace.StartSpanOption) ddtrace.Span { - if !t.config.enabled.current { + if !knobs.GetScope(t.config.Scope, enabled) { return internal.NoopSpan{} } var opts ddtrace.StartSpanConfig @@ -716,7 +716,7 @@ func (t *tracer) Stop() { // Inject uses the configured or default TextMap Propagator. func (t *tracer) Inject(ctx ddtrace.SpanContext, carrier interface{}) error { - if !t.config.enabled.current { + if !knobs.GetScope(t.config.Scope, enabled) { return nil } t.updateSampling(ctx) @@ -755,7 +755,7 @@ func (t *tracer) updateSampling(ctx ddtrace.SpanContext) { // Extract uses the configured or default TextMap Propagator. func (t *tracer) Extract(carrier interface{}) (ddtrace.SpanContext, error) { - if !t.config.enabled.current { + if !knobs.GetScope(t.config.Scope, enabled) { return internal.NoopSpanContext{}, nil } return t.config.propagator.Extract(carrier)