diff --git a/.golangci.yml b/.golangci.yml index 5cf1fc6ec8f..0b7eb38d083 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -42,7 +42,6 @@ output: linters: enable: - - deadcode - errcheck - goconst - gofmt @@ -53,12 +52,10 @@ linters: - megacheck - misspell - revive - - structcheck - typecheck - unconvert - unparam - unused - - varcheck linters-settings: errcheck: diff --git a/CHANGELOG.md b/CHANGELOG.md index d116334ee3b..f4e2a13c0ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,8 @@ Internal types are updated to use `scope` instead of `instrumentation_library`. * [BUGFIX] New wal file separator '+' for the NTFS filesystem and backward compatibility with the old separator ':' [#1700](https://github.com/grafana/tempo/pull/1700) (@kilian-kier) * [ENHANCEMENT] metrics-generator: extract `status_message` field from spans [#1786](https://github.com/grafana/tempo/pull/1786), [#1794](https://github.com/grafana/tempo/pull/1794) (@stoewer) * [ENHANCEMENT] metrics-generator: handle collisions between user defined and default dimensions [#1794](https://github.com/grafana/tempo/pull/1794) (@stoewer) + **BREAKING CHANGE** Custom dimensions colliding with intrinsic dimensions will be prefixed with `__`. +* [ENHANCEMENT] metrics-generator: make intrinsic dimensions configurable and disable `status_message` by default [#1960](https://github.com/grafana/tempo/pull/1960) (@stoewer) * [ENHANCEMENT] distributor: Log span names when `distributor.log_received_spans.include_all_attributes` is on [#1790](https://github.com/grafana/tempo/pull/1790) (@suraciii) * [ENHANCEMENT] metrics-generator: truncate label names and values exceeding a configurable length [#1897](https://github.com/grafana/tempo/pull/1897) (@kvrhdn) * [ENHANCEMENT] Add parquet WAL [#1878](https://github.com/grafana/tempo/pull/1878) (@joe-elliott, @mdisibio) diff --git a/docs/tempo/website/configuration/_index.md b/docs/tempo/website/configuration/_index.md index 933ae14213f..19022a144fc 100644 --- a/docs/tempo/website/configuration/_index.md +++ b/docs/tempo/website/configuration/_index.md @@ -284,10 +284,26 @@ metrics_generator: # Buckets for the latency histogram in seconds. [histogram_buckets: | default = 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.02, 2.05, 4.10] - # Additional dimensions to add to the metrics along with the default dimensions - # (service, span_name, span_kind, status_code, and status_message). Dimensions are searched - # for in the resource and span attributes and are added to the metrics if present. + # Configure intrinsic dimensions to add to the metrics. Intrinsic dimensions are taken + # directly from the respective resource and span properties. + intrinsic_dimensions: + # Whether to add the name of the service the span is associated with. + [service: | default = true] + # Whether to add the name of the span. + [span_name: | default = true] + # Whether to add the span kind describing the relationship between spans. + [span_kind: | default = true] + # Whether to add the span status code. + [status_code: | default = true] + # Whether to add a status message. Important note: The span status message may + # contain arbitrary strings and thus have a very high cardinality. + [status_message: | default = false] + + # Additional dimensions to add to the metrics along with the intrinsic dimensions. + # Dimensions are searched for in the resource and span attributes and are added to + # the metrics if present. [dimensions: ] + # Registry configuration registry: @@ -1152,6 +1168,8 @@ overrides: [metrics_generator_processor_service_graphs_histogram_buckets: ] [metrics_generator_processor_service_graphs_dimensions: ] [metrics_generator_processor_span_metrics_histogram_buckets: ] + # Allowed keys for intrinsic dimensions are: service, span_name, span_kind, status_code, and status_message. + [metrics_generator_processor_span_metrics_intrinsic_dimensions: ] [metrics_generator_processor_span_metrics_dimensions: ] # Maximum number of active series in the registry, per instance of the metrics-generator. A diff --git a/docs/tempo/website/configuration/manifest.md b/docs/tempo/website/configuration/manifest.md index 92ca29bb494..ec638eeca2f 100644 --- a/docs/tempo/website/configuration/manifest.md +++ b/docs/tempo/website/configuration/manifest.md @@ -16,7 +16,7 @@ go run ./cmd/tempo --storage.trace.backend=local --storage.trace.local.path=/tmp ## Complete configuration -> **Note**: This manifest was generated on 2022-09-13. +> **Note**: This manifest was generated on 2022-12-19. ```yaml target: all @@ -31,6 +31,8 @@ server: grpc_listen_address: "" grpc_listen_port: 9095 grpc_listen_conn_limit: 0 + tls_cipher_suites: "" + tls_min_version: "" http_tls_config: cert_file: "" key_file: "" @@ -86,6 +88,8 @@ distributor: tls_ca_path: "" tls_server_name: "" tls_insecure_skip_verify: false + tls_cipher_suites: "" + tls_min_version: "" username: "" password: "" multi: @@ -104,6 +108,7 @@ distributor: receivers: {} override_ring_key: distributor log_received_traces: false + forwarders: [] extend_writes: true search_tags_deny_list: [] ingester_client: @@ -129,6 +134,8 @@ ingester_client: tls_ca_path: "" tls_server_name: "" tls_insecure_skip_verify: false + tls_cipher_suites: "" + tls_min_version: "" metrics_generator_client: pool_config: checkinterval: 15s @@ -152,6 +159,8 @@ metrics_generator_client: tls_ca_path: "" tls_server_name: "" tls_insecure_skip_verify: false + tls_cipher_suites: "" + tls_min_version: "" querier: search: query_timeout: 30s @@ -163,7 +172,6 @@ querier: max_concurrent_queries: 5 frontend_worker: frontend_address: 127.0.0.1:9095 - scheduler_address: "" dns_lookup_duration: 10s parallelism: 2 match_max_concurrent: true @@ -185,39 +193,13 @@ querier: tls_ca_path: "" tls_server_name: "" tls_insecure_skip_verify: false + tls_cipher_suites: "" + tls_min_version: "" query_relevant_ingesters: false query_frontend: - log_queries_longer_than: 0s - max_body_size: 0 - query_stats_enabled: false max_outstanding_per_tenant: 100 querier_forget_delay: 0s - scheduler_address: "" - scheduler_dns_lookup_period: 0s - scheduler_worker_concurrency: 0 - grpc_client_config: - max_recv_msg_size: 0 - max_send_msg_size: 0 - grpc_compression: "" - rate_limit: 0 - rate_limit_burst: 0 - backoff_on_ratelimits: false - backoff_config: - min_period: 0s - max_period: 0s - max_retries: 0 - tls_enabled: false - tls_cert_path: "" - tls_key_path: "" - tls_ca_path: "" - tls_server_name: "" - tls_insecure_skip_verify: false - instance_interface_names: [] - address: "" - port: 0 - downstream_url: "" max_retries: 2 - query_shards: 20 search: concurrent_jobs: 50 target_bytes_per_job: 10485760 @@ -227,6 +209,7 @@ query_frontend: query_backend_after: 15m0s query_ingesters_until: 1h0m0s trace_by_id: + query_shards: 20 hedge_requests_at: 2s hedge_requests_up_to: 2 compactor: @@ -252,6 +235,8 @@ compactor: tls_ca_path: "" tls_server_name: "" tls_insecure_skip_verify: false + tls_cipher_suites: "" + tls_min_version: "" username: "" password: "" multi: @@ -307,6 +292,8 @@ ingester: tls_ca_path: "" tls_server_name: "" tls_insecure_skip_verify: false + tls_cipher_suites: "" + tls_min_version: "" username: "" password: "" multi: @@ -342,6 +329,7 @@ ingester: max_block_bytes: 1073741824 complete_block_timeout: 15m0s override_ring_key: ring + use_flatbuffer_search: false metrics_generator: ring: kvstore: @@ -365,6 +353,8 @@ metrics_generator: tls_ca_path: "" tls_server_name: "" tls_insecure_skip_verify: false + tls_cipher_suites: "" + tls_min_version: "" username: "" password: "" multi: @@ -410,10 +400,17 @@ metrics_generator: - 4.096 - 8.192 - 16.384 + intrinsic_dimensions: + service: true + span_name: true + span_kind: true + status_code: true dimensions: [] registry: collection_interval: 15s stale_duration: 15m0s + max_label_name_length: 1024 + max_label_value_length: 2048 storage: path: "" wal: @@ -437,13 +434,14 @@ storage: blocksfilepath: /tmp/tempo/wal/blocks encoding: snappy search_encoding: none + version: v2 ingestion_time_range_slack: 2m0s block: index_downsample_bytes: 1048576 index_page_size_bytes: 256000 bloom_filter_false_positive: 0.01 bloom_filter_shard_size_bytes: 102400 - version: v2 + version: vParquet encoding: zstd search_encoding: snappy search_page_size_bytes: 1048576 @@ -518,6 +516,7 @@ overrides: max_traces_per_user: 10000 max_global_traces_per_user: 0 max_search_bytes_per_trace: 5000 + forwarders: [] metrics_generator_ring_size: 0 metrics_generator_processors: null metrics_generator_max_active_series: 0 @@ -529,6 +528,7 @@ overrides: metrics_generator_processor_service_graphs_dimensions: [] metrics_generator_processor_span_metrics_histogram_buckets: [] metrics_generator_processor_span_metrics_dimensions: [] + metrics_generator_processor_span_metrics_intrinsic_dimensions: {} block_retention: 0s max_bytes_per_tag_values_query: 5000000 max_search_duration: 0s @@ -569,6 +569,8 @@ memberlist: tls_ca_path: "" tls_server_name: "" tls_insecure_skip_verify: false + tls_cipher_suites: "" + tls_min_version: "" usage_report: reporting_enabled: true backoff: diff --git a/modules/generator/config.go b/modules/generator/config.go index 0d275aff91e..95b19c8f8f3 100644 --- a/modules/generator/config.go +++ b/modules/generator/config.go @@ -4,6 +4,8 @@ import ( "flag" "time" + "github.com/pkg/errors" + "github.com/grafana/tempo/modules/generator/processor/servicegraphs" "github.com/grafana/tempo/modules/generator/processor/spanmetrics" "github.com/grafana/tempo/modules/generator/registry" @@ -50,7 +52,7 @@ func (cfg *ProcessorConfig) RegisterFlagsAndApplyDefaults(prefix string, f *flag } // copyWithOverrides creates a copy of the config using values set in the overrides. -func (cfg *ProcessorConfig) copyWithOverrides(o metricsGeneratorOverrides, userID string) ProcessorConfig { +func (cfg *ProcessorConfig) copyWithOverrides(o metricsGeneratorOverrides, userID string) (ProcessorConfig, error) { copyCfg := *cfg if buckets := o.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(userID); buckets != nil { @@ -65,6 +67,12 @@ func (cfg *ProcessorConfig) copyWithOverrides(o metricsGeneratorOverrides, userI if dimensions := o.MetricsGeneratorProcessorSpanMetricsDimensions(userID); dimensions != nil { copyCfg.SpanMetrics.Dimensions = dimensions } + if dimensions := o.MetricsGeneratorProcessorSpanMetricsIntrinsicDimensions(userID); dimensions != nil { + err := copyCfg.SpanMetrics.IntrinsicDimensions.ApplyFromMap(dimensions) + if err != nil { + return ProcessorConfig{}, errors.Wrap(err, "fail to apply overrides") + } + } - return copyCfg + return copyCfg, nil } diff --git a/modules/generator/config_test.go b/modules/generator/config_test.go index 10b98ce8721..6addfc4cc93 100644 --- a/modules/generator/config_test.go +++ b/modules/generator/config_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/grafana/tempo/modules/generator/processor/servicegraphs" "github.com/grafana/tempo/modules/generator/processor/spanmetrics" @@ -16,20 +17,23 @@ func TestProcessorConfig_copyWithOverrides(t *testing.T) { Dimensions: []string{}, }, SpanMetrics: spanmetrics.Config{ - HistogramBuckets: []float64{1, 2}, - Dimensions: []string{"namespace"}, + HistogramBuckets: []float64{1, 2}, + Dimensions: []string{"namespace"}, + IntrinsicDimensions: spanmetrics.IntrinsicDimensions{Service: true}, }, } t.Run("overrides buckets and dimension", func(t *testing.T) { o := &mockOverrides{ - serviceGraphsHistogramBuckets: []float64{1, 2}, - serviceGraphsDimensions: []string{"namespace"}, - spanMetricsHistogramBuckets: []float64{1, 2, 3}, - spanMetricsDimensions: []string{"cluster", "namespace"}, + serviceGraphsHistogramBuckets: []float64{1, 2}, + serviceGraphsDimensions: []string{"namespace"}, + spanMetricsHistogramBuckets: []float64{1, 2, 3}, + spanMetricsDimensions: []string{"cluster", "namespace"}, + spanMetricsIntrinsicDimensions: map[string]bool{"status_code": true}, } - copied := original.copyWithOverrides(o, "tenant") + copied, err := original.copyWithOverrides(o, "tenant") + require.NoError(t, err) assert.NotEqual(t, *original, copied) @@ -38,19 +42,31 @@ func TestProcessorConfig_copyWithOverrides(t *testing.T) { assert.Equal(t, []string{}, original.ServiceGraphs.Dimensions) assert.Equal(t, []float64{1, 2}, original.SpanMetrics.HistogramBuckets) assert.Equal(t, []string{"namespace"}, original.SpanMetrics.Dimensions) + assert.Equal(t, spanmetrics.IntrinsicDimensions{Service: true}, original.SpanMetrics.IntrinsicDimensions) // assert overrides were applied assert.Equal(t, []float64{1, 2}, copied.ServiceGraphs.HistogramBuckets) assert.Equal(t, []string{"namespace"}, copied.ServiceGraphs.Dimensions) assert.Equal(t, []float64{1, 2, 3}, copied.SpanMetrics.HistogramBuckets) assert.Equal(t, []string{"cluster", "namespace"}, copied.SpanMetrics.Dimensions) + assert.Equal(t, spanmetrics.IntrinsicDimensions{Service: true, StatusCode: true}, copied.SpanMetrics.IntrinsicDimensions) }) t.Run("empty overrides", func(t *testing.T) { o := &mockOverrides{} - copied := original.copyWithOverrides(o, "tenant") + copied, err := original.copyWithOverrides(o, "tenant") + require.NoError(t, err) assert.Equal(t, *original, copied) }) + + t.Run("invalid overrides", func(t *testing.T) { + o := &mockOverrides{ + spanMetricsIntrinsicDimensions: map[string]bool{"invalid": true}, + } + + _, err := original.copyWithOverrides(o, "tenant") + require.Error(t, err) + }) } diff --git a/modules/generator/instance.go b/modules/generator/instance.go index 5ed300f2169..4e199c593de 100644 --- a/modules/generator/instance.go +++ b/modules/generator/instance.go @@ -126,7 +126,10 @@ func (i *instance) watchOverrides() { func (i *instance) updateProcessors() error { desiredProcessors := i.overrides.MetricsGeneratorProcessors(i.instanceID) - desiredCfg := i.cfg.Processor.copyWithOverrides(i.overrides, i.instanceID) + desiredCfg, err := i.cfg.Processor.copyWithOverrides(i.overrides, i.instanceID) + if err != nil { + return err + } i.processorsMtx.RLock() toAdd, toRemove, toReplace, err := i.diffProcessors(desiredProcessors, desiredCfg) diff --git a/modules/generator/instance_test.go b/modules/generator/instance_test.go index a187a6c41e0..4fd696efb74 100644 --- a/modules/generator/instance_test.go +++ b/modules/generator/instance_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/grafana/tempo/modules/generator/processor/servicegraphs" + "github.com/grafana/tempo/modules/generator/processor/spanmetrics" "github.com/grafana/tempo/modules/generator/storage" "github.com/grafana/tempo/pkg/tempopb" v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" @@ -83,7 +84,7 @@ func Test_instance_updateProcessors(t *testing.T) { // no processors should be present initially assert.Len(t, instance.processors, 0) - t.Run("add new processor", func(t *testing.T) { + t.Run("add servicegraphs processors", func(t *testing.T) { overrides.processors = map[string]struct{}{ servicegraphs.Name: {}, } @@ -106,20 +107,36 @@ func Test_instance_updateProcessors(t *testing.T) { assert.Equal(t, instance.processors[servicegraphs.Name].Name(), servicegraphs.Name) }) - t.Run("replace processor", func(t *testing.T) { + t.Run("add spanmetrics processor", func(t *testing.T) { overrides.processors = map[string]struct{}{ servicegraphs.Name: {}, + spanmetrics.Name: {}, } - overrides.serviceGraphsDimensions = []string{"namespace"} + err := instance.updateProcessors() + assert.NoError(t, err) + + assert.Len(t, instance.processors, 2) + assert.Equal(t, instance.processors[servicegraphs.Name].Name(), servicegraphs.Name) + assert.Equal(t, instance.processors[spanmetrics.Name].Name(), spanmetrics.Name) + }) + + t.Run("replace spanmetrics processor", func(t *testing.T) { + overrides.processors = map[string]struct{}{ + servicegraphs.Name: {}, + spanmetrics.Name: {}, + } + overrides.spanMetricsDimensions = []string{"namespace"} + overrides.spanMetricsIntrinsicDimensions = map[string]bool{"status_message": true} err := instance.updateProcessors() assert.NoError(t, err) - var expectedConfig servicegraphs.Config + var expectedConfig spanmetrics.Config expectedConfig.RegisterFlagsAndApplyDefaults("", &flag.FlagSet{}) expectedConfig.Dimensions = []string{"namespace"} + expectedConfig.IntrinsicDimensions.StatusMessage = true - assert.Equal(t, expectedConfig, instance.processors[servicegraphs.Name].(*servicegraphs.Processor).Cfg) + assert.Equal(t, expectedConfig, instance.processors[spanmetrics.Name].(*spanmetrics.Processor).Cfg) }) t.Run("remove processor", func(t *testing.T) { diff --git a/modules/generator/overrides.go b/modules/generator/overrides.go index 1da172acb54..f564ec8040b 100644 --- a/modules/generator/overrides.go +++ b/modules/generator/overrides.go @@ -13,6 +13,7 @@ type metricsGeneratorOverrides interface { MetricsGeneratorProcessorServiceGraphsDimensions(userID string) []string MetricsGeneratorProcessorSpanMetricsHistogramBuckets(userID string) []float64 MetricsGeneratorProcessorSpanMetricsDimensions(userID string) []string + MetricsGeneratorProcessorSpanMetricsIntrinsicDimensions(userID string) map[string]bool } var _ metricsGeneratorOverrides = (*overrides.Overrides)(nil) diff --git a/modules/generator/overrides_test.go b/modules/generator/overrides_test.go index 125ac7a6c6b..bf7fe9bf1d4 100644 --- a/modules/generator/overrides_test.go +++ b/modules/generator/overrides_test.go @@ -3,11 +3,12 @@ package generator import "time" type mockOverrides struct { - processors map[string]struct{} - serviceGraphsHistogramBuckets []float64 - serviceGraphsDimensions []string - spanMetricsHistogramBuckets []float64 - spanMetricsDimensions []string + processors map[string]struct{} + serviceGraphsHistogramBuckets []float64 + serviceGraphsDimensions []string + spanMetricsHistogramBuckets []float64 + spanMetricsDimensions []string + spanMetricsIntrinsicDimensions map[string]bool } var _ metricsGeneratorOverrides = (*mockOverrides)(nil) @@ -43,3 +44,7 @@ func (m *mockOverrides) MetricsGeneratorProcessorSpanMetricsHistogramBuckets(use func (m *mockOverrides) MetricsGeneratorProcessorSpanMetricsDimensions(userID string) []string { return m.spanMetricsDimensions } + +func (m *mockOverrides) MetricsGeneratorProcessorSpanMetricsIntrinsicDimensions(userID string) map[string]bool { + return m.spanMetricsIntrinsicDimensions +} diff --git a/modules/generator/processor/spanmetrics/config.go b/modules/generator/processor/spanmetrics/config.go index afbe8a68bf4..aeeaeeef1ed 100644 --- a/modules/generator/processor/spanmetrics/config.go +++ b/modules/generator/processor/spanmetrics/config.go @@ -3,21 +3,64 @@ package spanmetrics import ( "flag" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" ) const ( Name = "span-metrics" + + dimService = "service" + dimSpanName = "span_name" + dimSpanKind = "span_kind" + dimStatusCode = "status_code" + dimStatusMessage = "status_message" ) type Config struct { // Buckets for latency histogram in seconds. HistogramBuckets []float64 `yaml:"histogram_buckets"` - // Additional dimensions (labels) to be added to the metric, - // along with the default ones (service, span_name, span_kind and span_status). + // Intrinsic dimensions (labels) added to the metric, that are generated from fixed span + // data. The dimensions service, span_name, span_kind, and status_code are enabled by + // default, whereas the dimension status_message must be enabled explicitly. + IntrinsicDimensions IntrinsicDimensions `yaml:"intrinsic_dimensions"` + // Additional dimensions (labels) to be added to the metric. The dimensions are generated + // from span attributes and are created along with the intrinsic dimensions. Dimensions []string `yaml:"dimensions"` } func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) { cfg.HistogramBuckets = prometheus.ExponentialBuckets(0.002, 2, 14) + cfg.IntrinsicDimensions.Service = true + cfg.IntrinsicDimensions.SpanName = true + cfg.IntrinsicDimensions.SpanKind = true + cfg.IntrinsicDimensions.StatusCode = true +} + +type IntrinsicDimensions struct { + Service bool `yaml:"service"` + SpanName bool `yaml:"span_name"` + SpanKind bool `yaml:"span_kind"` + StatusCode bool `yaml:"status_code"` + StatusMessage bool `yaml:"status_message,omitempty"` +} + +func (ic *IntrinsicDimensions) ApplyFromMap(dimensions map[string]bool) error { + for label, active := range dimensions { + switch label { + case dimService: + ic.Service = active + case dimSpanName: + ic.SpanName = active + case dimSpanKind: + ic.SpanKind = active + case dimStatusCode: + ic.StatusCode = active + case dimStatusMessage: + ic.StatusMessage = active + default: + return errors.Errorf("%s is not a valid intrinsic dimension", label) + } + } + return nil } diff --git a/modules/generator/processor/spanmetrics/spanmetrics.go b/modules/generator/processor/spanmetrics/spanmetrics.go index 5d049933e49..f2c8dd825f2 100644 --- a/modules/generator/processor/spanmetrics/spanmetrics.go +++ b/modules/generator/processor/spanmetrics/spanmetrics.go @@ -22,10 +22,6 @@ const ( metricSizeTotal = "traces_spanmetrics_size_total" ) -var ( - intrinsicDimensions = []string{"service", "span_name", "span_kind", "status_code", "status_message"} -) - type Processor struct { Cfg Config @@ -40,8 +36,24 @@ type Processor struct { } func New(cfg Config, registry registry.Registry) gen.Processor { - labels := make([]string, 0, len(intrinsicDimensions)+len(cfg.Dimensions)) - labels = append(labels, intrinsicDimensions...) + labels := make([]string, 0, 4+len(cfg.Dimensions)) + + if cfg.IntrinsicDimensions.Service { + labels = append(labels, dimService) + } + if cfg.IntrinsicDimensions.SpanName { + labels = append(labels, dimSpanName) + } + if cfg.IntrinsicDimensions.SpanKind { + labels = append(labels, dimSpanKind) + } + if cfg.IntrinsicDimensions.StatusCode { + labels = append(labels, dimStatusCode) + } + if cfg.IntrinsicDimensions.StatusMessage { + labels = append(labels, dimStatusMessage) + } + for _, d := range cfg.Dimensions { labels = append(labels, sanitizeLabelNameWithCollisions(d)) } @@ -87,15 +99,22 @@ func (p *Processor) aggregateMetricsForSpan(svcName string, rs *v1.Resource, spa latencySeconds := float64(span.GetEndTimeUnixNano()-span.GetStartTimeUnixNano()) / float64(time.Second.Nanoseconds()) labelValues := make([]string, 0, 4+len(p.Cfg.Dimensions)) - // important: the order of labelValues must correspond to the order of labels / intrinsicDimensions - labelValues = append( - labelValues, - svcName, - span.GetName(), - span.GetKind().String(), - span.GetStatus().GetCode().String(), - span.GetStatus().GetMessage()) - + // important: the order of labelValues must correspond to the order of labels / intrinsic dimensions + if p.Cfg.IntrinsicDimensions.Service { + labelValues = append(labelValues, svcName) + } + if p.Cfg.IntrinsicDimensions.SpanName { + labelValues = append(labelValues, span.GetName()) + } + if p.Cfg.IntrinsicDimensions.SpanKind { + labelValues = append(labelValues, span.GetKind().String()) + } + if p.Cfg.IntrinsicDimensions.StatusCode { + labelValues = append(labelValues, span.GetStatus().GetCode().String()) + } + if p.Cfg.IntrinsicDimensions.StatusMessage { + labelValues = append(labelValues, span.GetStatus().GetMessage()) + } for _, d := range p.Cfg.Dimensions { value, _ := processor_util.FindAttributeValue(d, rs.Attributes, span.Attributes) labelValues = append(labelValues, value) @@ -111,11 +130,17 @@ func (p *Processor) aggregateMetricsForSpan(svcName string, rs *v1.Resource, spa func sanitizeLabelNameWithCollisions(name string) string { sanitized := strutil.SanitizeLabelName(name) - for _, dim := range intrinsicDimensions { - if sanitized == dim { - return "__" + sanitized - } + if isIntrinsicDimension(sanitized) { + return "__" + sanitized } return sanitized } + +func isIntrinsicDimension(name string) bool { + return name == dimService || + name == dimSpanName || + name == dimSpanKind || + name == dimStatusCode || + name == dimStatusMessage +} diff --git a/modules/generator/processor/spanmetrics/spanmetrics_test.go b/modules/generator/processor/spanmetrics/spanmetrics_test.go index ed4afd945b7..39001a0bb0e 100644 --- a/modules/generator/processor/spanmetrics/spanmetrics_test.go +++ b/modules/generator/processor/spanmetrics/spanmetrics_test.go @@ -35,11 +35,10 @@ func TestSpanMetrics(t *testing.T) { fmt.Println(testRegistry) lbls := labels.FromMap(map[string]string{ - "service": "test-service", - "span_name": "test", - "span_kind": "SPAN_KIND_CLIENT", - "status_code": "STATUS_CODE_OK", - "status_message": "OK", + "service": "test-service", + "span_name": "test", + "span_kind": "SPAN_KIND_CLIENT", + "status_code": "STATUS_CODE_OK", }) assert.Equal(t, 10.0, testRegistry.Query("traces_spanmetrics_calls_total", lbls)) @@ -57,6 +56,8 @@ func TestSpanMetrics_dimensions(t *testing.T) { cfg := Config{} cfg.RegisterFlagsAndApplyDefaults("", nil) cfg.HistogramBuckets = []float64{0.5, 1} + cfg.IntrinsicDimensions.SpanKind = false + cfg.IntrinsicDimensions.StatusMessage = true cfg.Dimensions = []string{"foo", "bar", "does-not-exist"} p := New(cfg, testRegistry) @@ -86,7 +87,6 @@ func TestSpanMetrics_dimensions(t *testing.T) { lbls := labels.FromMap(map[string]string{ "service": "test-service", "span_name": "test", - "span_kind": "SPAN_KIND_CLIENT", "status_code": "STATUS_CODE_OK", "status_message": "OK", "foo": "foo-value", @@ -109,7 +109,8 @@ func TestSpanMetrics_collisions(t *testing.T) { cfg := Config{} cfg.RegisterFlagsAndApplyDefaults("", nil) cfg.HistogramBuckets = []float64{0.5, 1} - cfg.Dimensions = []string{"span.kind", "status_message"} + cfg.Dimensions = []string{"span.kind", "span_name"} + cfg.IntrinsicDimensions.SpanKind = false p := New(cfg, testRegistry) defer p.Shutdown(context.Background()) @@ -122,8 +123,8 @@ func TestSpanMetrics_collisions(t *testing.T) { Value: &common_v1.AnyValue{Value: &common_v1.AnyValue_StringValue{StringValue: "colliding_kind"}}, }) s.Attributes = append(s.Attributes, &common_v1.KeyValue{ - Key: "status_message", - Value: &common_v1.AnyValue{Value: &common_v1.AnyValue_StringValue{StringValue: "colliding_message"}}, + Key: "span_name", + Value: &common_v1.AnyValue{Value: &common_v1.AnyValue_StringValue{StringValue: "colliding_name"}}, }) } } @@ -133,13 +134,11 @@ func TestSpanMetrics_collisions(t *testing.T) { fmt.Println(testRegistry) lbls := labels.FromMap(map[string]string{ - "service": "test-service", - "span_name": "test", - "span_kind": "SPAN_KIND_CLIENT", - "status_code": "STATUS_CODE_OK", - "status_message": "OK", - "__span_kind": "colliding_kind", - "__status_message": "colliding_message", + "service": "test-service", + "span_name": "test", + "status_code": "STATUS_CODE_OK", + "__span_kind": "colliding_kind", + "__span_name": "colliding_name", }) assert.Equal(t, 10.0, testRegistry.Query("traces_spanmetrics_calls_total", lbls)) diff --git a/modules/overrides/limits.go b/modules/overrides/limits.go index dd6a94bd54f..5d3c38e10b9 100644 --- a/modules/overrides/limits.go +++ b/modules/overrides/limits.go @@ -60,17 +60,18 @@ type Limits struct { Forwarders []string `yaml:"forwarders" json:"forwarders"` // Metrics-generator config - MetricsGeneratorRingSize int `yaml:"metrics_generator_ring_size" json:"metrics_generator_ring_size"` - MetricsGeneratorProcessors ListToMap `yaml:"metrics_generator_processors" json:"metrics_generator_processors"` - MetricsGeneratorMaxActiveSeries uint32 `yaml:"metrics_generator_max_active_series" json:"metrics_generator_max_active_series"` - MetricsGeneratorCollectionInterval time.Duration `yaml:"metrics_generator_collection_interval" json:"metrics_generator_collection_interval"` - MetricsGeneratorDisableCollection bool `yaml:"metrics_generator_disable_collection" json:"metrics_generator_disable_collection"` - MetricsGeneratorForwarderQueueSize int `yaml:"metrics_generator_forwarder_queue_size" json:"metrics_generator_forwarder_queue_size"` - MetricsGeneratorForwarderWorkers int `yaml:"metrics_generator_forwarder_workers" json:"metrics_generator_forwarder_workers"` - MetricsGeneratorProcessorServiceGraphsHistogramBuckets []float64 `yaml:"metrics_generator_processor_service_graphs_histogram_buckets" json:"metrics_generator_processor_service_graphs_histogram_buckets"` - MetricsGeneratorProcessorServiceGraphsDimensions []string `yaml:"metrics_generator_processor_service_graphs_dimensions" json:"metrics_generator_processor_service_graphs_dimensions"` - MetricsGeneratorProcessorSpanMetricsHistogramBuckets []float64 `yaml:"metrics_generator_processor_span_metrics_histogram_buckets" json:"metrics_generator_processor_span_metrics_histogram_buckets"` - MetricsGeneratorProcessorSpanMetricsDimensions []string `yaml:"metrics_generator_processor_span_metrics_dimensions" json:"metrics_generator_processor_span_metrics_dimensions"` + MetricsGeneratorRingSize int `yaml:"metrics_generator_ring_size" json:"metrics_generator_ring_size"` + MetricsGeneratorProcessors ListToMap `yaml:"metrics_generator_processors" json:"metrics_generator_processors"` + MetricsGeneratorMaxActiveSeries uint32 `yaml:"metrics_generator_max_active_series" json:"metrics_generator_max_active_series"` + MetricsGeneratorCollectionInterval time.Duration `yaml:"metrics_generator_collection_interval" json:"metrics_generator_collection_interval"` + MetricsGeneratorDisableCollection bool `yaml:"metrics_generator_disable_collection" json:"metrics_generator_disable_collection"` + MetricsGeneratorForwarderQueueSize int `yaml:"metrics_generator_forwarder_queue_size" json:"metrics_generator_forwarder_queue_size"` + MetricsGeneratorForwarderWorkers int `yaml:"metrics_generator_forwarder_workers" json:"metrics_generator_forwarder_workers"` + MetricsGeneratorProcessorServiceGraphsHistogramBuckets []float64 `yaml:"metrics_generator_processor_service_graphs_histogram_buckets" json:"metrics_generator_processor_service_graphs_histogram_buckets"` + MetricsGeneratorProcessorServiceGraphsDimensions []string `yaml:"metrics_generator_processor_service_graphs_dimensions" json:"metrics_generator_processor_service_graphs_dimensions"` + MetricsGeneratorProcessorSpanMetricsHistogramBuckets []float64 `yaml:"metrics_generator_processor_span_metrics_histogram_buckets" json:"metrics_generator_processor_span_metrics_histogram_buckets"` + MetricsGeneratorProcessorSpanMetricsDimensions []string `yaml:"metrics_generator_processor_span_metrics_dimensions" json:"metrics_generator_processor_span_metrics_dimensions"` + MetricsGeneratorProcessorSpanMetricsIntrinsicDimensions map[string]bool `yaml:"metrics_generator_processor_span_metrics_intrinsic_dimensions" json:"metrics_generator_processor_span_metrics_intrinsic_dimensions"` // Compactor enforced limits. BlockRetention model.Duration `yaml:"block_retention" json:"block_retention"` diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index 9e74e25c552..9900dc31a87 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -345,6 +345,12 @@ func (o *Overrides) MetricsGeneratorProcessorSpanMetricsDimensions(userID string return o.getOverridesForUser(userID).MetricsGeneratorProcessorSpanMetricsDimensions } +// MetricsGeneratorProcessorSpanMetricsIntrinsicDimensions controls the intrinsic dimensions such as service, span_kind, or +// span_name that are activated or deactivated on the span metrics processor. +func (o *Overrides) MetricsGeneratorProcessorSpanMetricsIntrinsicDimensions(userID string) map[string]bool { + return o.getOverridesForUser(userID).MetricsGeneratorProcessorSpanMetricsIntrinsicDimensions +} + // BlockRetention is the duration of the block retention for this tenant. func (o *Overrides) BlockRetention(userID string) time.Duration { return time.Duration(o.getOverridesForUser(userID).BlockRetention)