From 3cc8b4ac809d1ebd96bcb8de7dea9010a964d412 Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Thu, 11 Apr 2019 09:21:54 +0200 Subject: [PATCH 01/11] add counter and gauge + keylabel metrics tests --- .../helper/prometheus/prometheus_test.go | 185 ++++++++++++++++-- 1 file changed, 172 insertions(+), 13 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index 0bcdc3ba3ff..1fba363fe60 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -21,7 +21,6 @@ import ( "bytes" "io/ioutil" "net/http" - "net/http/httptest" "sort" "testing" @@ -31,7 +30,8 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -const promMetrics = ` +const ( + promMetrics = ` # TYPE first_metric gauge first_metric{label1="value1",label2="value2",label3="Value3",label4="FOO"} 1 # TYPE second_metric gauge @@ -63,26 +63,41 @@ histogram_decimal_metric_count 5 ` -type mockFetcher struct{} + promGaugeKeyLabel = ` +# TYPE metrics_one_count_total gauge +metrics_one_count_total{name="jane",surname="foster"} 1 +metrics_one_count_total{name="john",surname="williams"} 2 +metrics_one_count_total{name="jahn",surname="baldwin",age="30"} 3 +` + + promCounterKeyLabel = ` +# TYPE metrics_one_count_total counter +metrics_one_count_total{name="jane",surname="foster"} 1 +metrics_one_count_total{name="john",surname="williams"} 2 +metrics_one_count_total{name="jahn",surname="baldwin",age=30} 3 + +` +) + +type mockFetcher struct { + response string +} + +var _ = httpfetcher(&mockFetcher{}) + +// FetchResponse returns an HTTP response but for the Body, which +// returns the mockFetcher.Response contents func (m mockFetcher) FetchResponse() (*http.Response, error) { return &http.Response{ Header: make(http.Header), - Body: ioutil.NopCloser(bytes.NewReader([]byte(promMetrics))), + Body: ioutil.NopCloser(bytes.NewReader([]byte(m.response))), }, nil } func TestPrometheus(t *testing.T) { - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "text/plain; charset=ISO-8859-1") - w.Write([]byte(promMetrics)) - })) - server.Start() - defer server.Close() - - p := &prometheus{mockFetcher{}} + p := &prometheus{mockFetcher{response: promMetrics}} tests := []struct { mapping *MetricsMapping @@ -381,3 +396,147 @@ func TestPrometheus(t *testing.T) { }) } } + +func TestPrometheusExtra(t *testing.T) { + + testCases := []struct { + testName string + prometheusResponse string + mapping *MetricsMapping + expectedEvents []common.MapStr + }{ + { + testName: "Test gauge with KeyLabel", + prometheusResponse: promGaugeKeyLabel, + mapping: &MetricsMapping{ + Metrics: map[string]MetricMap{ + "metrics_one_count_total": Metric("metrics.one.count"), + }, + Labels: map[string]LabelMap{ + "name": KeyLabel("metrics.one.labels.name"), + "surname": KeyLabel("metrics.one.labels.surname"), + "age": KeyLabel("metrics.one.labels.age"), + }, + }, + expectedEvents: []common.MapStr{ + common.MapStr{ + "metrics": common.MapStr{ + "one": common.MapStr{ + "count": 1.0, + "labels": common.MapStr{ + "name": "jane", + "surname": "foster", + }, + }, + }, + }, + common.MapStr{ + "metrics": common.MapStr{ + "one": common.MapStr{ + "count": 2.0, + "labels": common.MapStr{ + "name": "john", + "surname": "williams", + }, + }, + }, + }, + common.MapStr{ + "metrics": common.MapStr{ + "one": common.MapStr{ + "count": 3.0, + "labels": common.MapStr{ + "name": "jahn", + "surname": "baldwin", + "age": "30", + }, + }, + }, + }, + }, + }, + + { + testName: "Test counter with KeyLabel", + prometheusResponse: promCounterKeyLabel, + mapping: &MetricsMapping{ + Metrics: map[string]MetricMap{ + "metrics_one_count_total": Metric("metrics.one.count"), + }, + Labels: map[string]LabelMap{ + "name": KeyLabel("metrics.one.labels.name"), + "surname": KeyLabel("metrics.one.labels.surname"), + "age": KeyLabel("metrics.one.labels.age"), + }, + }, + expectedEvents: []common.MapStr{ + common.MapStr{ + "metrics": common.MapStr{ + "one": common.MapStr{ + "count": 1.0, + "labels": common.MapStr{ + "name": "jane", + "surname": "foster", + }, + }, + }, + }, + common.MapStr{ + "metrics": common.MapStr{ + "one": common.MapStr{ + "count": 2.0, + "labels": common.MapStr{ + "name": "john", + "surname": "williams", + }, + }, + }, + }, + common.MapStr{ + "metrics": common.MapStr{ + "one": common.MapStr{ + "count": 3.0, + "labels": common.MapStr{ + "name": "jahn", + "surname": "baldwin", + "age": "30", + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + r := &mbtest.CapturingReporterV2{} + p := &prometheus{mockFetcher{response: tc.prometheusResponse}} + p.ReportProcessedMetrics(tc.mapping, r) + if !assert.Nil(t, r.GetErrors(), + "error reporting/processing metrincs, at %q", tc.testName) { + continue + } + + events := r.GetEvents() + if !assert.Equal(t, len(tc.expectedEvents), len(events), + "number of returned events doesn't match expected, at %q", tc.testName) { + continue + } + + // Sort slices of received and expeected to avoid unmatching + sort.Slice(events, func(i, j int) bool { + return events[i].MetricSetFields.String() < events[j].MetricSetFields.String() + }) + sort.Slice(tc.expectedEvents, func(i, j int) bool { + return tc.expectedEvents[i].String() < tc.expectedEvents[j].String() + }) + + for i := range events { + if !assert.Equal(t, tc.expectedEvents[i], events[i].MetricSetFields, + "mismatch at event #%d, at %q", i, tc.testName) { + continue + } + t.Logf("events: %+v", events[i].MetricSetFields) + } + } +} From 31bef806b607c2e4494d778ab1a1fc2ad15af2aa Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Thu, 11 Apr 2019 12:05:18 +0200 Subject: [PATCH 02/11] add histogram + keylabel tests --- .../helper/prometheus/prometheus_test.go | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index 1fba363fe60..20da07ecf4d 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -77,6 +77,27 @@ metrics_one_count_total{name="jane",surname="foster"} 1 metrics_one_count_total{name="john",surname="williams"} 2 metrics_one_count_total{name="jahn",surname="baldwin",age=30} 3 +` + + promHistogramKeyLabel = ` +# TYPE metrics_one_midichlorians histogram +metrics_one_midichlorians_bucket{rank="youngling",alive="yes",le="2000"} 52 +metrics_one_midichlorians_bucket{rank="youngling",alive="yes",le="4000"} 70 +metrics_one_midichlorians_bucket{rank="youngling",alive="yes",le="8000"} 78 +metrics_one_midichlorians_bucket{rank="youngling",alive="yes",le="16000"} 84 +metrics_one_midichlorians_bucket{rank="youngling",alive="yes",le="32000"} 86 +metrics_one_midichlorians_bucket{rank="youngling",alive="yes",le="+Inf"} 86 +metrics_one_midichlorians_sum{rank="youngling",alive="yes"} 1000001 +metrics_one_midichlorians_count{rank="youngling",alive="yes"} 86 +metrics_one_midichlorians_bucket{rank="padawan",alive="yes",le="2000"} 16 +metrics_one_midichlorians_bucket{rank="padawan",alive="yes",le="4000"} 20 +metrics_one_midichlorians_bucket{rank="padawan",alive="yes",le="8000"} 23 +metrics_one_midichlorians_bucket{rank="padawan",alive="yes",le="16000"} 27 +metrics_one_midichlorians_bucket{rank="padawan",alive="yes",le="32000"} 27 +metrics_one_midichlorians_bucket{rank="padawan",alive="yes",le="+Inf"} 28 +metrics_one_midichlorians_sum{rank="padawan",alive="yes"} 800001 +metrics_one_midichlorians_count{rank="padawan",alive="yes"} 28 + ` ) @@ -506,6 +527,66 @@ func TestPrometheusExtra(t *testing.T) { }, }, }, + + { + testName: "Test histogram with KeyLabel", + prometheusResponse: promHistogramKeyLabel, + mapping: &MetricsMapping{ + Metrics: map[string]MetricMap{ + "metrics_one_midichlorians": Metric("metrics.one.midichlorians"), + }, + Labels: map[string]LabelMap{ + "rank": KeyLabel("metrics.one.midichlorians.labels.rank"), + "alive": KeyLabel("metrics.one.midichlorians.labels.alive"), + }, + }, + expectedEvents: []common.MapStr{ + common.MapStr{ + "metrics": common.MapStr{ + "one": common.MapStr{ + "midichlorians": common.MapStr{ + "count": uint64(86), + "sum": 1000001.0, + "bucket": common.MapStr{ + "2000": uint64(52), + "4000": uint64(70), + "8000": uint64(78), + "16000": uint64(84), + "32000": uint64(86), + "+Inf": uint64(86), + }, + "labels": common.MapStr{ + "rank": "youngling", + "alive": "yes", + }, + }, + }, + }, + }, + common.MapStr{ + "metrics": common.MapStr{ + "one": common.MapStr{ + "midichlorians": common.MapStr{ + "count": uint64(28), + "sum": 800001.0, + "bucket": common.MapStr{ + "2000": uint64(16), + "4000": uint64(20), + "8000": uint64(23), + "16000": uint64(27), + "32000": uint64(27), + "+Inf": uint64(28), + }, + "labels": common.MapStr{ + "rank": "padawan", + "alive": "yes", + }, + }, + }, + }, + }, + }, + }, } for _, tc := range testCases { From a923f3783f33f3509423223919757d06e8ec6e9d Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Thu, 11 Apr 2019 12:47:43 +0200 Subject: [PATCH 03/11] add prometheus summary keylabel test --- .../helper/prometheus/prometheus_test.go | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index 20da07ecf4d..1cdea0690d4 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -98,6 +98,25 @@ metrics_one_midichlorians_bucket{rank="padawan",alive="yes",le="+Inf"} 28 metrics_one_midichlorians_sum{rank="padawan",alive="yes"} 800001 metrics_one_midichlorians_count{rank="padawan",alive="yes"} 28 +` + + promSummaryKeyLabel = ` +# TYPE metrics_force_propagation_ms summary +metrics_force_propagation_ms{kind="jedi",quantile="0"} 35 +metrics_force_propagation_ms{kind="jedi",quantile="0.25"} 22 +metrics_force_propagation_ms{kind="jedi",quantile="0.5"} 7 +metrics_force_propagation_ms{kind="jedi",quantile="0.75"} 20 +metrics_force_propagation_ms{kind="jedi",quantile="1"} 30 +metrics_force_propagation_ms_sum{kind="jedi"} 89 +metrics_force_propagation_ms_count{kind="jedi"} 651 +metrics_force_propagation_ms{kind="sith",quantile="0"} 30 +metrics_force_propagation_ms{kind="sith",quantile="0.25"} 20 +metrics_force_propagation_ms{kind="sith",quantile="0.5"} 12 +metrics_force_propagation_ms{kind="sith",quantile="0.75"} 21 +metrics_force_propagation_ms{kind="sith",quantile="1"} 29 +metrics_force_propagation_ms_sum{kind="sith"} 112 +metrics_force_propagation_ms_count{kind="sith"} 711 + ` ) @@ -587,6 +606,87 @@ func TestPrometheusExtra(t *testing.T) { }, }, }, + + { + testName: "Test summary with KeyLabel", + prometheusResponse: promSummaryKeyLabel, + mapping: &MetricsMapping{ + Metrics: map[string]MetricMap{ + "metrics_force_propagation_ms": Metric("metrics.force.propagation.ms"), + }, + Labels: map[string]LabelMap{ + "kind": KeyLabel("metrics.force.propagation.ms.labels.kind"), + }, + }, + expectedEvents: []common.MapStr{ + common.MapStr{ + "metrics": common.MapStr{ + "force": common.MapStr{ + "propagation": common.MapStr{ + "ms": common.MapStr{ + "count": uint64(651), + "sum": 89.0, + "percentile": common.MapStr{ + "0": uint64(35), + "25": uint64(22), + "50": uint64(7), + "75": uint64(20), + "100": uint64(30), + }, + "labels": common.MapStr{ + "kind": "jedi", + }, + }, + }, + }, + }, + }, + common.MapStr{ + "metrics": common.MapStr{ + "force": common.MapStr{ + "propagation": common.MapStr{ + "ms": common.MapStr{ + "count": uint64(711), + "sum": 112.0, + "percentile": common.MapStr{ + "0": uint64(30), + "25": uint64(20), + "50": uint64(12), + "75": uint64(21), + "100": uint64(29), + }, + "labels": common.MapStr{ + "kind": "sith", + }, + }, + }, + }, + }, + }, + }, + }, + + // mapping: &MetricsMapping{ + // Metrics: map[string]MetricMap{ + // "summary_metric": Metric("summary.metric"), + // }, + // }, + // expected: []common.MapStr{ + // common.MapStr{ + // "summary": common.MapStr{ + // "metric": common.MapStr{ + // "sum": 234892394.0, + // "count": uint64(44000), + // "percentile": common.MapStr{ + // "50": 29735.0, + // "90": 47103.0, + // "99": 50681.0, + // }, + // }, + // }, + // }, + // }, + } for _, tc := range testCases { From f7d5d73c7cb01657e0b5cd20c07979202babc988 Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Thu, 11 Apr 2019 13:05:12 +0200 Subject: [PATCH 04/11] [skip ci] remove commented block --- .../helper/prometheus/prometheus_test.go | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index 1cdea0690d4..ee7f88cda9d 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -665,28 +665,6 @@ func TestPrometheusExtra(t *testing.T) { }, }, }, - - // mapping: &MetricsMapping{ - // Metrics: map[string]MetricMap{ - // "summary_metric": Metric("summary.metric"), - // }, - // }, - // expected: []common.MapStr{ - // common.MapStr{ - // "summary": common.MapStr{ - // "metric": common.MapStr{ - // "sum": 234892394.0, - // "count": uint64(44000), - // "percentile": common.MapStr{ - // "50": 29735.0, - // "90": 47103.0, - // "99": 50681.0, - // }, - // }, - // }, - // }, - // }, - } for _, tc := range testCases { From 14cf47bce492e3c9d551fea79d407608759ce7ae Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Thu, 11 Apr 2019 14:46:22 +0200 Subject: [PATCH 05/11] fix prometheus counter keylabel tests --- metricbeat/helper/prometheus/prometheus_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index ee7f88cda9d..3ca08ff0bb9 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -75,7 +75,7 @@ metrics_one_count_total{name="jahn",surname="baldwin",age="30"} 3 # TYPE metrics_one_count_total counter metrics_one_count_total{name="jane",surname="foster"} 1 metrics_one_count_total{name="john",surname="williams"} 2 -metrics_one_count_total{name="jahn",surname="baldwin",age=30} 3 +metrics_one_count_total{name="jahn",surname="baldwin",age="30"} 3 ` @@ -513,7 +513,7 @@ func TestPrometheusExtra(t *testing.T) { common.MapStr{ "metrics": common.MapStr{ "one": common.MapStr{ - "count": 1.0, + "count": int64(1), "labels": common.MapStr{ "name": "jane", "surname": "foster", @@ -524,7 +524,7 @@ func TestPrometheusExtra(t *testing.T) { common.MapStr{ "metrics": common.MapStr{ "one": common.MapStr{ - "count": 2.0, + "count": int64(2), "labels": common.MapStr{ "name": "john", "surname": "williams", @@ -535,7 +535,7 @@ func TestPrometheusExtra(t *testing.T) { common.MapStr{ "metrics": common.MapStr{ "one": common.MapStr{ - "count": 3.0, + "count": int64(3), "labels": common.MapStr{ "name": "jahn", "surname": "baldwin", From d28bff355c319eacc9d771cda792a3c54496c2b2 Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Thu, 11 Apr 2019 23:42:16 +0200 Subject: [PATCH 06/11] add prometheus label restoration when overwritten --- metricbeat/helper/prometheus/prometheus.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index 62404e795d4..46be4b90127 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -161,10 +161,30 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS } if field != "" { - // Put it in the event if it's a common metric event := getEvent(eventsMap, keyLabels) + + // Case: histograms + keyLabels, whose labels are written at the same level as the + // histogram data + // + // if the path where the values are to be written exists, current fields will be overwritten + // if that happens we will keep the existingFields variable for restoring later + existingFields, _ := event.GetValue(field) + event.Put(field, value) event.DeepUpdate(labels) + + // If we overwrited metrics values on top of existing labels, let's restore. + // If there is a conflict (there shoudln't) we will keep values over labels + if existingFields != nil { + overwritten, _ := existingFields.(common.MapStr) + for k, v := range overwritten { + restorePath := field + "." + k + if exists, _ := event.GetValue(restorePath); exists == nil { + event.Put(field+"."+k, v) + } + } + } + } } } From c337f8b867717b68acc7d250b57a9fd833d0c27f Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Thu, 11 Apr 2019 23:43:18 +0200 Subject: [PATCH 07/11] working histogram + keylabel tests --- .../helper/prometheus/prometheus_test.go | 119 +++++++++--------- 1 file changed, 60 insertions(+), 59 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index 3ca08ff0bb9..75a919e6373 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -607,64 +607,64 @@ func TestPrometheusExtra(t *testing.T) { }, }, - { - testName: "Test summary with KeyLabel", - prometheusResponse: promSummaryKeyLabel, - mapping: &MetricsMapping{ - Metrics: map[string]MetricMap{ - "metrics_force_propagation_ms": Metric("metrics.force.propagation.ms"), - }, - Labels: map[string]LabelMap{ - "kind": KeyLabel("metrics.force.propagation.ms.labels.kind"), - }, - }, - expectedEvents: []common.MapStr{ - common.MapStr{ - "metrics": common.MapStr{ - "force": common.MapStr{ - "propagation": common.MapStr{ - "ms": common.MapStr{ - "count": uint64(651), - "sum": 89.0, - "percentile": common.MapStr{ - "0": uint64(35), - "25": uint64(22), - "50": uint64(7), - "75": uint64(20), - "100": uint64(30), - }, - "labels": common.MapStr{ - "kind": "jedi", - }, - }, - }, - }, - }, - }, - common.MapStr{ - "metrics": common.MapStr{ - "force": common.MapStr{ - "propagation": common.MapStr{ - "ms": common.MapStr{ - "count": uint64(711), - "sum": 112.0, - "percentile": common.MapStr{ - "0": uint64(30), - "25": uint64(20), - "50": uint64(12), - "75": uint64(21), - "100": uint64(29), - }, - "labels": common.MapStr{ - "kind": "sith", - }, - }, - }, - }, - }, - }, - }, - }, + // { + // testName: "Test summary with KeyLabel", + // prometheusResponse: promSummaryKeyLabel, + // mapping: &MetricsMapping{ + // Metrics: map[string]MetricMap{ + // "metrics_force_propagation_ms": Metric("metrics.force.propagation.ms"), + // }, + // Labels: map[string]LabelMap{ + // "kind": KeyLabel("metrics.force.propagation.ms.labels.kind"), + // }, + // }, + // expectedEvents: []common.MapStr{ + // common.MapStr{ + // "metrics": common.MapStr{ + // "force": common.MapStr{ + // "propagation": common.MapStr{ + // "ms": common.MapStr{ + // "count": uint64(651), + // "sum": 89.0, + // "percentile": common.MapStr{ + // "0": uint64(35), + // "25": uint64(22), + // "50": uint64(7), + // "75": uint64(20), + // "100": uint64(30), + // }, + // "labels": common.MapStr{ + // "kind": "jedi", + // }, + // }, + // }, + // }, + // }, + // }, + // common.MapStr{ + // "metrics": common.MapStr{ + // "force": common.MapStr{ + // "propagation": common.MapStr{ + // "ms": common.MapStr{ + // "count": uint64(711), + // "sum": 112.0, + // "percentile": common.MapStr{ + // "0": uint64(30), + // "25": uint64(20), + // "50": uint64(12), + // "75": uint64(21), + // "100": uint64(29), + // }, + // "labels": common.MapStr{ + // "kind": "sith", + // }, + // }, + // }, + // }, + // }, + // }, + // }, + // }, } for _, tc := range testCases { @@ -672,7 +672,7 @@ func TestPrometheusExtra(t *testing.T) { p := &prometheus{mockFetcher{response: tc.prometheusResponse}} p.ReportProcessedMetrics(tc.mapping, r) if !assert.Nil(t, r.GetErrors(), - "error reporting/processing metrincs, at %q", tc.testName) { + "error reporting/processing metrics, at %q", tc.testName) { continue } @@ -693,6 +693,7 @@ func TestPrometheusExtra(t *testing.T) { for i := range events { if !assert.Equal(t, tc.expectedEvents[i], events[i].MetricSetFields, "mismatch at event #%d, at %q", i, tc.testName) { + continue } t.Logf("events: %+v", events[i].MetricSetFields) From 2905ee14d16f3997920f130a2f4825adcaf563e7 Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Thu, 11 Apr 2019 23:55:05 +0200 Subject: [PATCH 08/11] add prometheus summary tests --- .../helper/prometheus/prometheus_test.go | 116 +++++++++--------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index 75a919e6373..8aa242a7c0e 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -607,64 +607,64 @@ func TestPrometheusExtra(t *testing.T) { }, }, - // { - // testName: "Test summary with KeyLabel", - // prometheusResponse: promSummaryKeyLabel, - // mapping: &MetricsMapping{ - // Metrics: map[string]MetricMap{ - // "metrics_force_propagation_ms": Metric("metrics.force.propagation.ms"), - // }, - // Labels: map[string]LabelMap{ - // "kind": KeyLabel("metrics.force.propagation.ms.labels.kind"), - // }, - // }, - // expectedEvents: []common.MapStr{ - // common.MapStr{ - // "metrics": common.MapStr{ - // "force": common.MapStr{ - // "propagation": common.MapStr{ - // "ms": common.MapStr{ - // "count": uint64(651), - // "sum": 89.0, - // "percentile": common.MapStr{ - // "0": uint64(35), - // "25": uint64(22), - // "50": uint64(7), - // "75": uint64(20), - // "100": uint64(30), - // }, - // "labels": common.MapStr{ - // "kind": "jedi", - // }, - // }, - // }, - // }, - // }, - // }, - // common.MapStr{ - // "metrics": common.MapStr{ - // "force": common.MapStr{ - // "propagation": common.MapStr{ - // "ms": common.MapStr{ - // "count": uint64(711), - // "sum": 112.0, - // "percentile": common.MapStr{ - // "0": uint64(30), - // "25": uint64(20), - // "50": uint64(12), - // "75": uint64(21), - // "100": uint64(29), - // }, - // "labels": common.MapStr{ - // "kind": "sith", - // }, - // }, - // }, - // }, - // }, - // }, - // }, - // }, + { + testName: "Test summary with KeyLabel", + prometheusResponse: promSummaryKeyLabel, + mapping: &MetricsMapping{ + Metrics: map[string]MetricMap{ + "metrics_force_propagation_ms": Metric("metrics.force.propagation.ms"), + }, + Labels: map[string]LabelMap{ + "kind": KeyLabel("metrics.force.propagation.ms.labels.kind"), + }, + }, + expectedEvents: []common.MapStr{ + common.MapStr{ + "metrics": common.MapStr{ + "force": common.MapStr{ + "propagation": common.MapStr{ + "ms": common.MapStr{ + "count": uint64(651), + "sum": 89.0, + "percentile": common.MapStr{ + "0": 35.0, + "25": 22.0, + "50": 7.0, + "75": 20.0, + "100": 30.0, + }, + "labels": common.MapStr{ + "kind": "jedi", + }, + }, + }, + }, + }, + }, + common.MapStr{ + "metrics": common.MapStr{ + "force": common.MapStr{ + "propagation": common.MapStr{ + "ms": common.MapStr{ + "count": uint64(711), + "sum": 112.0, + "percentile": common.MapStr{ + "0": 30.0, + "25": 20.0, + "50": 12.0, + "75": 21.0, + "100": 29.0, + }, + "labels": common.MapStr{ + "kind": "sith", + }, + }, + }, + }, + }, + }, + }, + }, } for _, tc := range testCases { From c68f8f68bb7c15265cadcf0d1d8c0bb98c8f595b Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Fri, 12 Apr 2019 08:37:08 +0200 Subject: [PATCH 09/11] set test name for KeyLabel tests --- metricbeat/helper/prometheus/prometheus_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index 8aa242a7c0e..bcccdbf3df8 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -437,7 +437,7 @@ func TestPrometheus(t *testing.T) { } } -func TestPrometheusExtra(t *testing.T) { +func TestPrometheusKeyLabels(t *testing.T) { testCases := []struct { testName string From feb27c0b65fe7e8c7db7301124918e453c682362 Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Fri, 12 Apr 2019 12:02:06 +0200 Subject: [PATCH 10/11] replace event building with suggested snippet --- metricbeat/helper/prometheus/prometheus.go | 26 +++++----------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index 46be4b90127..461eb159587 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -163,28 +163,12 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS if field != "" { event := getEvent(eventsMap, keyLabels) - // Case: histograms + keyLabels, whose labels are written at the same level as the - // histogram data - // - // if the path where the values are to be written exists, current fields will be overwritten - // if that happens we will keep the existingFields variable for restoring later - existingFields, _ := event.GetValue(field) - - event.Put(field, value) - event.DeepUpdate(labels) - - // If we overwrited metrics values on top of existing labels, let's restore. - // If there is a conflict (there shoudln't) we will keep values over labels - if existingFields != nil { - overwritten, _ := existingFields.(common.MapStr) - for k, v := range overwritten { - restorePath := field + "." + k - if exists, _ := event.GetValue(restorePath); exists == nil { - event.Put(field+"."+k, v) - } - } - } + // value may be a mapstr (for histograms and summaries), do a deep update to avoid smashing existing fields + update := common.MapStr{} + update.Put(field, value) + event.DeepUpdate(update) + event.DeepUpdate(labels) } } } From f7df84161afd3a86c7f267211618e30f618ece7f Mon Sep 17 00:00:00 2001 From: odacremolbap Date: Fri, 12 Apr 2019 12:12:22 +0200 Subject: [PATCH 11/11] set histogram test labels at the same level as the values --- metricbeat/helper/prometheus/prometheus_test.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index bcccdbf3df8..a782665d320 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -555,8 +555,8 @@ func TestPrometheusKeyLabels(t *testing.T) { "metrics_one_midichlorians": Metric("metrics.one.midichlorians"), }, Labels: map[string]LabelMap{ - "rank": KeyLabel("metrics.one.midichlorians.labels.rank"), - "alive": KeyLabel("metrics.one.midichlorians.labels.alive"), + "rank": KeyLabel("metrics.one.midichlorians.rank"), + "alive": KeyLabel("metrics.one.midichlorians.alive"), }, }, expectedEvents: []common.MapStr{ @@ -574,10 +574,9 @@ func TestPrometheusKeyLabels(t *testing.T) { "32000": uint64(86), "+Inf": uint64(86), }, - "labels": common.MapStr{ - "rank": "youngling", - "alive": "yes", - }, + + "rank": "youngling", + "alive": "yes", }, }, }, @@ -596,10 +595,8 @@ func TestPrometheusKeyLabels(t *testing.T) { "32000": uint64(27), "+Inf": uint64(28), }, - "labels": common.MapStr{ - "rank": "padawan", - "alive": "yes", - }, + "rank": "padawan", + "alive": "yes", }, }, },