Skip to content

Commit 8c3af7f

Browse files
SungJin1212SungJin1212
committed
Override exporter: expose all fields that can be converted to float64
Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>
1 parent 59491e9 commit 8c3af7f

File tree

3 files changed

+211
-6
lines changed

3 files changed

+211
-6
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
* [FEATURE] Querier: Allow choosing PromQL engine via header. #6777
2424
* [FEATURE] Querier: Support for configuring query optimizers and enabling XFunctions in the Thanos engine. #6873
2525
* [FEATURE] Query Frontend: Add support /api/v1/format_query API for formatting queries. #6893
26+
* [ENHANCEMENT] Overrides Exporter: Expose all fields that can be converted to float64. #6979
2627
* [ENHANCEMENT] Ingester: Add `cortex_ingester_tsdb_wal_replay_unknown_refs_total` and `cortex_ingester_tsdb_wbl_replay_unknown_refs_total` metrics to track unknown series references during wal/wbl replaying. #6945
2728
* [ENHANCEMENT] Ruler: Emit an error message when the rule synchronization fails. #6902
2829
* [ENHANCEMENT] Querier: Support snappy and zstd response compression for `-querier.response-compression` flag. #6848

‎pkg/util/validation/exporter.go‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package validation
22

33
import (
4+
"reflect"
5+
"strings"
6+
"time"
7+
48
"github.com/prometheus/client_golang/prometheus"
59
)
610

@@ -31,12 +35,56 @@ func (oe *OverridesExporter) Describe(ch chan<- *prometheus.Desc) {
3135
func (oe *OverridesExporter) Collect(ch chan<- prometheus.Metric) {
3236
allLimits := oe.tenantLimits.AllByUserID()
3337
for tenant, limits := range allLimits {
34-
ch <- prometheus.MustNewConstMetric(oe.description, prometheus.GaugeValue, limits.IngestionRate, "ingestion_rate", tenant)
35-
ch <- prometheus.MustNewConstMetric(oe.description, prometheus.GaugeValue, float64(limits.IngestionBurstSize), "ingestion_burst_size", tenant)
38+
for metricName, value := range ExtractNumericalValues(limits) {
39+
ch <- prometheus.MustNewConstMetric(oe.description, prometheus.GaugeValue, value, metricName, tenant)
40+
}
41+
}
42+
}
43+
44+
func ExtractNumericalValues(l *Limits) map[string]float64 {
45+
metrics := make(map[string]float64)
46+
47+
v := reflect.ValueOf(l).Elem()
48+
t := v.Type()
49+
50+
for i := 0; i < v.NumField(); i++ {
51+
field := v.Field(i)
52+
fieldType := t.Field(i)
53+
54+
tag := fieldType.Tag.Get("yaml")
55+
if tag == "" || tag == "-" {
56+
// not exist tag or tag is "-"
57+
continue
58+
}
59+
60+
// remove options like omitempty
61+
if idx := strings.Index(tag, ","); idx != -1 {
62+
tag = tag[:idx]
63+
}
3664

37-
ch <- prometheus.MustNewConstMetric(oe.description, prometheus.GaugeValue, float64(limits.MaxLocalSeriesPerUser), "max_local_series_per_user", tenant)
38-
ch <- prometheus.MustNewConstMetric(oe.description, prometheus.GaugeValue, float64(limits.MaxLocalSeriesPerMetric), "max_local_series_per_metric", tenant)
39-
ch <- prometheus.MustNewConstMetric(oe.description, prometheus.GaugeValue, float64(limits.MaxGlobalSeriesPerUser), "max_global_series_per_user", tenant)
40-
ch <- prometheus.MustNewConstMetric(oe.description, prometheus.GaugeValue, float64(limits.MaxGlobalSeriesPerMetric), "max_global_series_per_metric", tenant)
65+
switch field.Kind() {
66+
case reflect.Int, reflect.Int64:
67+
if field.Type().String() == "model.Duration" {
68+
// we export the model.Duration in seconds
69+
metrics[tag] = time.Duration(field.Int()).Seconds()
70+
} else {
71+
metrics[tag] = float64(field.Int())
72+
}
73+
case reflect.Uint, reflect.Uint64:
74+
metrics[tag] = float64(field.Uint())
75+
case reflect.Float64:
76+
metrics[tag] = field.Float()
77+
case reflect.Bool:
78+
if field.Bool() {
79+
// true as 1.0
80+
metrics[tag] = 1.0
81+
} else {
82+
// false as 0.0
83+
metrics[tag] = 0.0
84+
}
85+
case reflect.String, reflect.Slice, reflect.Map, reflect.Struct:
86+
continue
87+
}
4188
}
89+
return metrics
4290
}

‎pkg/util/validation/exporter_test.go‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
package validation
22

33
import (
4+
"flag"
5+
"strings"
46
"testing"
7+
"time"
58

69
"github.com/prometheus/client_golang/prometheus/testutil"
710
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
812
)
913

1014
func TestOverridesExporter_noConfig(t *testing.T) {
@@ -21,10 +25,162 @@ func TestOverridesExporter_withConfig(t *testing.T) {
2125
MaxQueriersPerTenant: 5,
2226
},
2327
}
28+
tenantLimits["tenant-a"].RegisterFlags(flag.CommandLine)
2429

2530
exporter := NewOverridesExporter(newMockTenantLimits(tenantLimits))
2631

2732
// There should be at least a few metrics generated by receiving an override configuration map
2833
count := testutil.CollectAndCount(exporter, "cortex_overrides")
2934
assert.Greater(t, count, 0)
35+
require.NoError(t, testutil.CollectAndCompare(exporter, strings.NewReader(`
36+
# HELP cortex_overrides Resource limit overrides applied to tenants
37+
# TYPE cortex_overrides gauge
38+
cortex_overrides{limit_name="accept_ha_samples",user="tenant-a"} 0
39+
cortex_overrides{limit_name="accept_mixed_ha_samples",user="tenant-a"} 0
40+
cortex_overrides{limit_name="alertmanager_max_alerts_count",user="tenant-a"} 0
41+
cortex_overrides{limit_name="alertmanager_max_alerts_size_bytes",user="tenant-a"} 0
42+
cortex_overrides{limit_name="alertmanager_max_config_size_bytes",user="tenant-a"} 0
43+
cortex_overrides{limit_name="alertmanager_max_dispatcher_aggregation_groups",user="tenant-a"} 0
44+
cortex_overrides{limit_name="alertmanager_max_silences_count",user="tenant-a"} 0
45+
cortex_overrides{limit_name="alertmanager_max_silences_size_bytes",user="tenant-a"} 0
46+
cortex_overrides{limit_name="alertmanager_max_template_size_bytes",user="tenant-a"} 0
47+
cortex_overrides{limit_name="alertmanager_max_templates_count",user="tenant-a"} 0
48+
cortex_overrides{limit_name="alertmanager_notification_rate_limit",user="tenant-a"} 0
49+
cortex_overrides{limit_name="alertmanager_receivers_firewall_block_private_addresses",user="tenant-a"} 0
50+
cortex_overrides{limit_name="compactor_blocks_retention_period",user="tenant-a"} 0
51+
cortex_overrides{limit_name="compactor_partition_index_size_bytes",user="tenant-a"} 6.8719476736e+10
52+
cortex_overrides{limit_name="compactor_partition_series_count",user="tenant-a"} 0
53+
cortex_overrides{limit_name="compactor_tenant_shard_size",user="tenant-a"} 0
54+
cortex_overrides{limit_name="creation_grace_period",user="tenant-a"} 600
55+
cortex_overrides{limit_name="enable_native_histograms",user="tenant-a"} 0
56+
cortex_overrides{limit_name="enforce_metadata_metric_name",user="tenant-a"} 1
57+
cortex_overrides{limit_name="enforce_metric_name",user="tenant-a"} 1
58+
cortex_overrides{limit_name="ha_max_clusters",user="tenant-a"} 0
59+
cortex_overrides{limit_name="ingestion_burst_size",user="tenant-a"} 50000
60+
cortex_overrides{limit_name="ingestion_rate",user="tenant-a"} 25000
61+
cortex_overrides{limit_name="ingestion_tenant_shard_size",user="tenant-a"} 0
62+
cortex_overrides{limit_name="max_cache_freshness",user="tenant-a"} 60
63+
cortex_overrides{limit_name="max_downloaded_bytes_per_request",user="tenant-a"} 0
64+
cortex_overrides{limit_name="max_exemplars",user="tenant-a"} 0
65+
cortex_overrides{limit_name="max_fetched_chunk_bytes_per_query",user="tenant-a"} 0
66+
cortex_overrides{limit_name="max_fetched_chunks_per_query",user="tenant-a"} 2e+06
67+
cortex_overrides{limit_name="max_fetched_data_bytes_per_query",user="tenant-a"} 0
68+
cortex_overrides{limit_name="max_fetched_series_per_query",user="tenant-a"} 0
69+
cortex_overrides{limit_name="max_global_metadata_per_metric",user="tenant-a"} 0
70+
cortex_overrides{limit_name="max_global_metadata_per_user",user="tenant-a"} 0
71+
cortex_overrides{limit_name="max_global_native_histogram_series_per_user",user="tenant-a"} 0
72+
cortex_overrides{limit_name="max_global_series_per_metric",user="tenant-a"} 0
73+
cortex_overrides{limit_name="max_global_series_per_user",user="tenant-a"} 0
74+
cortex_overrides{limit_name="max_label_name_length",user="tenant-a"} 1024
75+
cortex_overrides{limit_name="max_label_names_per_series",user="tenant-a"} 30
76+
cortex_overrides{limit_name="max_label_value_length",user="tenant-a"} 2048
77+
cortex_overrides{limit_name="max_labels_size_bytes",user="tenant-a"} 0
78+
cortex_overrides{limit_name="max_metadata_length",user="tenant-a"} 1024
79+
cortex_overrides{limit_name="max_metadata_per_metric",user="tenant-a"} 10
80+
cortex_overrides{limit_name="max_metadata_per_user",user="tenant-a"} 8000
81+
cortex_overrides{limit_name="max_native_histogram_buckets",user="tenant-a"} 0
82+
cortex_overrides{limit_name="max_native_histogram_sample_size_bytes",user="tenant-a"} 0
83+
cortex_overrides{limit_name="max_native_histogram_series_per_user",user="tenant-a"} 0
84+
cortex_overrides{limit_name="max_outstanding_requests_per_tenant",user="tenant-a"} 100
85+
cortex_overrides{limit_name="max_queriers_per_tenant",user="tenant-a"} 0
86+
cortex_overrides{limit_name="max_query_length",user="tenant-a"} 0
87+
cortex_overrides{limit_name="max_query_lookback",user="tenant-a"} 0
88+
cortex_overrides{limit_name="max_query_parallelism",user="tenant-a"} 14
89+
cortex_overrides{limit_name="max_query_response_size",user="tenant-a"} 0
90+
cortex_overrides{limit_name="max_series_per_metric",user="tenant-a"} 50000
91+
cortex_overrides{limit_name="max_series_per_user",user="tenant-a"} 5e+06
92+
cortex_overrides{limit_name="native_histogram_ingestion_burst_size",user="tenant-a"} 0
93+
cortex_overrides{limit_name="native_histogram_ingestion_rate",user="tenant-a"} 1.7976931348623157e+308
94+
cortex_overrides{limit_name="out_of_order_time_window",user="tenant-a"} 0
95+
cortex_overrides{limit_name="parquet_converter_enabled",user="tenant-a"} 0
96+
cortex_overrides{limit_name="parquet_converter_tenant_shard_size",user="tenant-a"} 0
97+
cortex_overrides{limit_name="parquet_max_fetched_chunk_bytes",user="tenant-a"} 0
98+
cortex_overrides{limit_name="parquet_max_fetched_data_bytes",user="tenant-a"} 0
99+
cortex_overrides{limit_name="parquet_max_fetched_row_count",user="tenant-a"} 0
100+
cortex_overrides{limit_name="query_partial_data",user="tenant-a"} 0
101+
cortex_overrides{limit_name="query_vertical_shard_size",user="tenant-a"} 0
102+
cortex_overrides{limit_name="reject_old_samples",user="tenant-a"} 0
103+
cortex_overrides{limit_name="reject_old_samples_max_age",user="tenant-a"} 1.2096e+06
104+
cortex_overrides{limit_name="ruler_evaluation_delay_duration",user="tenant-a"} 0
105+
cortex_overrides{limit_name="ruler_max_rule_groups_per_tenant",user="tenant-a"} 0
106+
cortex_overrides{limit_name="ruler_max_rules_per_rule_group",user="tenant-a"} 0
107+
cortex_overrides{limit_name="ruler_query_offset",user="tenant-a"} 0
108+
cortex_overrides{limit_name="ruler_tenant_shard_size",user="tenant-a"} 0
109+
cortex_overrides{limit_name="rules_partial_data",user="tenant-a"} 0
110+
cortex_overrides{limit_name="store_gateway_tenant_shard_size",user="tenant-a"} 0
111+
`), "cortex_overrides"))
112+
}
113+
114+
func TestExtractNumericalValues(t *testing.T) {
115+
limits := Limits{}
116+
limits.RegisterFlags(flag.CommandLine)
117+
extracted := ExtractNumericalValues(&limits)
118+
t.Run("float64 should be converted", func(t *testing.T) {
119+
require.Equal(t, limits.IngestionRate, extracted["ingestion_rate"])
120+
})
121+
t.Run("int should be converted", func(t *testing.T) {
122+
require.Equal(t, float64(limits.IngestionBurstSize), extracted["ingestion_burst_size"])
123+
})
124+
t.Run("int64 should be converted", func(t *testing.T) {
125+
require.Equal(t, float64(limits.MaxQueryResponseSize), extracted["max_query_response_size"])
126+
})
127+
t.Run("string shouldn't be converted", func(t *testing.T) {
128+
_, ok := extracted["ingestion_rate_strategy"]
129+
require.False(t, ok, "string should be not converted")
130+
})
131+
t.Run("bool should be converted, default value false converted to 0", func(t *testing.T) {
132+
val, ok := extracted["accept_ha_samples"]
133+
require.True(t, ok)
134+
require.Equal(t, 0.0, val)
135+
})
136+
t.Run("bool should be converted, default value true converted to 1", func(t *testing.T) {
137+
val, ok := extracted["enforce_metric_name"]
138+
require.True(t, ok)
139+
require.Equal(t, 1.0, val)
140+
})
141+
t.Run("flagext.StringSlice shouldn't be converted", func(t *testing.T) {
142+
_, ok := extracted["drop_labels"]
143+
require.False(t, ok)
144+
})
145+
t.Run("model.Duration should be converted", func(t *testing.T) {
146+
val, ok := extracted["reject_old_samples_max_age"]
147+
require.True(t, ok)
148+
require.Equal(t, time.Duration(limits.RejectOldSamplesMaxAge).Seconds(), val)
149+
})
150+
t.Run("[]*relabel.Config shouldn't be converted", func(t *testing.T) {
151+
_, ok := extracted["metric_relabel_configs"]
152+
require.False(t, ok)
153+
})
154+
t.Run("[]string shouldn't be converted", func(t *testing.T) {
155+
_, ok := extracted["promote_resource_attributes"]
156+
require.False(t, ok)
157+
})
158+
t.Run("[]LimitsPerLabelSet shouldn't be converted", func(t *testing.T) {
159+
_, ok := extracted["limits_per_label_set"]
160+
require.False(t, ok)
161+
})
162+
t.Run("QueryPriority shouldn't be converted", func(t *testing.T) {
163+
_, ok := extracted["query_priority"]
164+
require.False(t, ok)
165+
})
166+
t.Run("QueryRejection shouldn't be converted", func(t *testing.T) {
167+
_, ok := extracted["query_rejection"]
168+
require.False(t, ok)
169+
})
170+
t.Run("labels.Labels shouldn't be converted", func(t *testing.T) {
171+
_, ok := extracted["ruler_external_labels"]
172+
require.False(t, ok)
173+
})
174+
t.Run("flagext.CIDRSliceCSV shouldn't be converted", func(t *testing.T) {
175+
_, ok := extracted["alertmanager_receivers_firewall_block_cidr_networks"]
176+
require.False(t, ok)
177+
})
178+
t.Run("NotificationRateLimitMap shouldn't be converted", func(t *testing.T) {
179+
_, ok := extracted["alertmanager_notification_rate_limit_per_integration"]
180+
require.False(t, ok)
181+
})
182+
t.Run("DisabledRuleGroups shouldn't be converted", func(t *testing.T) {
183+
_, ok := extracted["disabled_rule_groups"]
184+
require.False(t, ok)
185+
})
30186
}

0 commit comments

Comments
 (0)