From 35abaab7707779761be18879ce370b06dd378cc1 Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 10 Apr 2019 14:41:48 +0200 Subject: [PATCH 1/4] Atomic commit --- .../dropwizard/collector/_meta/data.json | 55 ++++-------- .../collector/_meta/testdata/config.yml | 6 ++ .../collector/_meta/testdata/docs.json | 59 +++++++++++++ .../_meta/testdata/docs.json-expected.json | 87 +++++++++++++++++++ .../module/dropwizard/collector/collector.go | 22 +++-- .../collector/collector_integration_test.go | 25 +++--- 6 files changed, 196 insertions(+), 58 deletions(-) create mode 100644 metricbeat/module/dropwizard/collector/_meta/testdata/config.yml create mode 100644 metricbeat/module/dropwizard/collector/_meta/testdata/docs.json create mode 100644 metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json diff --git a/metricbeat/module/dropwizard/collector/_meta/data.json b/metricbeat/module/dropwizard/collector/_meta/data.json index 2d764371481..d4fc83eb6b8 100644 --- a/metricbeat/module/dropwizard/collector/_meta/data.json +++ b/metricbeat/module/dropwizard/collector/_meta/data.json @@ -1,51 +1,34 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, + "@timestamp": "2019-03-01T08:05:34.853Z", "dropwizard": { "testnamespace": { - "my_gauge": {}, - "my_histogram": { - "count": 0, - "max": 0, - "mean": 0, - "min": 0, - "p50": 0, - "p75": 0, - "p95": 0, - "p98": 0, - "p99": 0, - "p999": 0, - "stddev": 0 + "_namespace": "testnamespace", + "my_counter": { + "count": 1 }, - "my_timer": { + "my_meter": { "count": 0, - "duration_units": "seconds", "m15_rate": 0, "m1_rate": 0, "m5_rate": 0, - "max": 0, - "mean": 0, "mean_rate": 0, - "min": 0, - "p50": 0, - "p75": 0, - "p95": 0, - "p98": 0, - "p99": 0, - "p999": 0, - "rate_units": "calls/second", - "stddev": 0 + "units": "events/second" + }, + "tags": { + "this": "that" } } }, + "event": { + "dataset": "dropwizard.testnamespace", + "duration": 115000, + "module": "dropwizard" + }, "metricset": { - "host": "dropwizard:8080", - "module": "dropwizard", - "name": "collector", - "namespace": "testnamespace", - "rtt": 115 + "name": "collector" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "dropwizard" } } \ No newline at end of file diff --git a/metricbeat/module/dropwizard/collector/_meta/testdata/config.yml b/metricbeat/module/dropwizard/collector/_meta/testdata/config.yml new file mode 100644 index 00000000000..11e382eb5eb --- /dev/null +++ b/metricbeat/module/dropwizard/collector/_meta/testdata/config.yml @@ -0,0 +1,6 @@ +type: http +url: "/metrics/metrics" +omit_documented_fields_check: + - "dropwizard.testnamespace.*" +module: + namespace: testnamespace diff --git a/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json b/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json new file mode 100644 index 00000000000..75a3eda11f3 --- /dev/null +++ b/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json @@ -0,0 +1,59 @@ +{ + "version": "3.0.0", + "gauges": { + "my_gauge": { + "value": null + } + }, + "counters": { + "my_counter{this=that}": { + "count": 1 + } + }, + "histograms": { + "my_histogram": { + "count": 0, + "max": 0, + "mean": 0, + "min": 0, + "p50": 0, + "p75": 0, + "p95": 0, + "p98": 0, + "p99": 0, + "p999": 0, + "stddev": 0 + } + }, + "meters": { + "my_meter{this=that}": { + "count": 0, + "m15_rate": 0, + "m1_rate": 0, + "m5_rate": 0, + "mean_rate": 0, + "units": "events/second" + } + }, + "timers": { + "my_timer": { + "count": 0, + "max": 0, + "mean": 0, + "min": 0, + "p50": 0, + "p75": 0, + "p95": 0, + "p98": 0, + "p99": 0, + "p999": 0, + "stddev": 0, + "m15_rate": 0, + "m1_rate": 0, + "m5_rate": 0, + "mean_rate": 0, + "duration_units": "seconds", + "rate_units": "calls/second" + } + } +} diff --git a/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json b/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json new file mode 100644 index 00000000000..0940b2ee991 --- /dev/null +++ b/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json @@ -0,0 +1,87 @@ +[ + { + "dropwizard": { + "testnamespace": { + "_namespace": "testnamespace", + "my_counter": { + "count": 1 + }, + "my_meter": { + "count": 0, + "m15_rate": 0, + "m1_rate": 0, + "m5_rate": 0, + "mean_rate": 0, + "units": "events/second" + }, + "tags": { + "this": "that" + } + } + }, + "event": { + "dataset": "dropwizard.testnamespace", + "duration": 115000, + "module": "dropwizard" + }, + "metricset": { + "name": "collector" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "dropwizard" + } + }, + { + "dropwizard": { + "testnamespace": { + "_namespace": "testnamespace", + "my_gauge": {}, + "my_histogram": { + "count": 0, + "max": 0, + "mean": 0, + "min": 0, + "p50": 0, + "p75": 0, + "p95": 0, + "p98": 0, + "p99": 0, + "p999": 0, + "stddev": 0 + }, + "my_timer": { + "count": 0, + "duration_units": "seconds", + "m15_rate": 0, + "m1_rate": 0, + "m5_rate": 0, + "max": 0, + "mean": 0, + "mean_rate": 0, + "min": 0, + "p50": 0, + "p75": 0, + "p95": 0, + "p98": 0, + "p99": 0, + "p999": 0, + "rate_units": "calls/second", + "stddev": 0 + } + } + }, + "event": { + "dataset": "dropwizard.testnamespace", + "duration": 115000, + "module": "dropwizard" + }, + "metricset": { + "name": "collector" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "dropwizard" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/dropwizard/collector/collector.go b/metricbeat/module/dropwizard/collector/collector.go index f4f5fb1a80b..e1a768f3e34 100644 --- a/metricbeat/module/dropwizard/collector/collector.go +++ b/metricbeat/module/dropwizard/collector/collector.go @@ -21,7 +21,6 @@ import ( "encoding/json" "strings" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" @@ -56,7 +55,7 @@ func init() { // multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP + http *helper.HTTP namespace string } @@ -79,17 +78,17 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, http: http, - namespace: config.Namespace, + namespace: config.Namespace, }, nil } // Fetch methods implements the data gathering and data conversion to the right format // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { body, err := m.http.FetchContent() if err != nil { - return nil, err + return err } dw := map[string]interface{}{} @@ -98,17 +97,22 @@ func (m *MetricSet) Fetch() ([]common.MapStr, error) { err = d.Decode(&dw) if err != nil { - return nil, err + return err } eventList := eventMapping(dw) // Converts hash list to slice - events := []common.MapStr{} for _, event := range eventList { event[mb.NamespaceKey] = m.namespace - events = append(events, event) + + if reported := reporter.Event(mb.Event{ + MetricSetFields: event, + Namespace: "dropwizard."+m.namespace, + }); !reported { + m.Logger().Debug("event not reported", event) + } } - return events, err + return nil } diff --git a/metricbeat/module/dropwizard/collector/collector_integration_test.go b/metricbeat/module/dropwizard/collector/collector_integration_test.go index c4b3cf490ab..bc62df6ddf0 100644 --- a/metricbeat/module/dropwizard/collector/collector_integration_test.go +++ b/metricbeat/module/dropwizard/collector/collector_integration_test.go @@ -33,25 +33,29 @@ import ( func TestFetch(t *testing.T) { compose.EnsureUp(t, "dropwizard") - f := mbtest.NewEventsFetcher(t, getConfig()) - events, err := f.Fetch() + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) hasTag := false doesntHaveTag := false for _, event := range events { - ok, _ := event.HasKey("my_histogram") + ok, _ := event.MetricSetFields.HasKey("my_histogram") if ok { - _, err := event.GetValue("tags") + _, err := event.MetricSetFields.GetValue("tags") if err == nil { t.Fatal("write", "my_counter not supposed to have tags") } doesntHaveTag = true } - ok, _ = event.HasKey("my_counter") + ok, _ = event.MetricSetFields.HasKey("my_counter") if ok { - tagsRaw, err := event.GetValue("tags") + tagsRaw, err := event.MetricSetFields.GetValue("tags") if err != nil { t.Fatal("write", err) } else { @@ -67,19 +71,14 @@ func TestFetch(t *testing.T) { } assert.Equal(t, hasTag, true) assert.Equal(t, doesntHaveTag, true) - if !assert.NoError(t, err) { - t.FailNow() - } t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events) } func TestData(t *testing.T) { - compose.EnsureUp(t, "dropwizard") + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) - f := mbtest.NewEventsFetcher(t, getConfig()) - err := mbtest.WriteEvents(f, t) - if err != nil { + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { t.Fatal("write", err) } } From 0488ebe98d37b7d13ceaf29c872075b584ba5c23 Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 10 Apr 2019 15:10:35 +0200 Subject: [PATCH 2/4] make fmt --- metricbeat/module/dropwizard/collector/collector.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/dropwizard/collector/collector.go b/metricbeat/module/dropwizard/collector/collector.go index e1a768f3e34..345b1cfda9b 100644 --- a/metricbeat/module/dropwizard/collector/collector.go +++ b/metricbeat/module/dropwizard/collector/collector.go @@ -55,7 +55,7 @@ func init() { // multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP + http *helper.HTTP namespace string } @@ -78,7 +78,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, http: http, - namespace: config.Namespace, + namespace: config.Namespace, }, nil } @@ -108,7 +108,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if reported := reporter.Event(mb.Event{ MetricSetFields: event, - Namespace: "dropwizard."+m.namespace, + Namespace: "dropwizard." + m.namespace, }); !reported { m.Logger().Debug("event not reported", event) } From cdc7a3ad0f9a8ceed3cb0ae37f4ac659807e9371 Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 10 Apr 2019 15:50:27 +0200 Subject: [PATCH 3/4] Drop _namespace field --- .../dropwizard/collector/_meta/data.json | 34 +++++++--- .../_meta/testdata/docs.json-expected.json | 66 +++++++++---------- .../module/dropwizard/collector/collector.go | 2 - 3 files changed, 58 insertions(+), 44 deletions(-) diff --git a/metricbeat/module/dropwizard/collector/_meta/data.json b/metricbeat/module/dropwizard/collector/_meta/data.json index d4fc83eb6b8..ce8608fb51b 100644 --- a/metricbeat/module/dropwizard/collector/_meta/data.json +++ b/metricbeat/module/dropwizard/collector/_meta/data.json @@ -2,20 +2,38 @@ "@timestamp": "2019-03-01T08:05:34.853Z", "dropwizard": { "testnamespace": { - "_namespace": "testnamespace", - "my_counter": { - "count": 1 + "my_gauge": {}, + "my_histogram": { + "count": 0, + "max": 0, + "mean": 0, + "min": 0, + "p50": 0, + "p75": 0, + "p95": 0, + "p98": 0, + "p99": 0, + "p999": 0, + "stddev": 0 }, - "my_meter": { + "my_timer": { "count": 0, + "duration_units": "seconds", "m15_rate": 0, "m1_rate": 0, "m5_rate": 0, + "max": 0, + "mean": 0, "mean_rate": 0, - "units": "events/second" - }, - "tags": { - "this": "that" + "min": 0, + "p50": 0, + "p75": 0, + "p95": 0, + "p98": 0, + "p99": 0, + "p999": 0, + "rate_units": "calls/second", + "stddev": 0 } } }, diff --git a/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json b/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json index 0940b2ee991..16846cb25af 100644 --- a/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json +++ b/metricbeat/module/dropwizard/collector/_meta/testdata/docs.json-expected.json @@ -2,40 +2,6 @@ { "dropwizard": { "testnamespace": { - "_namespace": "testnamespace", - "my_counter": { - "count": 1 - }, - "my_meter": { - "count": 0, - "m15_rate": 0, - "m1_rate": 0, - "m5_rate": 0, - "mean_rate": 0, - "units": "events/second" - }, - "tags": { - "this": "that" - } - } - }, - "event": { - "dataset": "dropwizard.testnamespace", - "duration": 115000, - "module": "dropwizard" - }, - "metricset": { - "name": "collector" - }, - "service": { - "address": "127.0.0.1:55555", - "type": "dropwizard" - } - }, - { - "dropwizard": { - "testnamespace": { - "_namespace": "testnamespace", "my_gauge": {}, "my_histogram": { "count": 0, @@ -83,5 +49,37 @@ "address": "127.0.0.1:55555", "type": "dropwizard" } + }, + { + "dropwizard": { + "testnamespace": { + "my_counter": { + "count": 1 + }, + "my_meter": { + "count": 0, + "m15_rate": 0, + "m1_rate": 0, + "m5_rate": 0, + "mean_rate": 0, + "units": "events/second" + }, + "tags": { + "this": "that" + } + } + }, + "event": { + "dataset": "dropwizard.testnamespace", + "duration": 115000, + "module": "dropwizard" + }, + "metricset": { + "name": "collector" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "dropwizard" + } } ] \ No newline at end of file diff --git a/metricbeat/module/dropwizard/collector/collector.go b/metricbeat/module/dropwizard/collector/collector.go index 345b1cfda9b..dd1f922baf9 100644 --- a/metricbeat/module/dropwizard/collector/collector.go +++ b/metricbeat/module/dropwizard/collector/collector.go @@ -104,8 +104,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { // Converts hash list to slice for _, event := range eventList { - event[mb.NamespaceKey] = m.namespace - if reported := reporter.Event(mb.Event{ MetricSetFields: event, Namespace: "dropwizard." + m.namespace, From 8f18927ece80c82f821d60278833b071c5f9dc6c Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 11 Apr 2019 10:50:25 +0200 Subject: [PATCH 4/4] Remove TestData test --- .../dropwizard/collector/collector_integration_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/metricbeat/module/dropwizard/collector/collector_integration_test.go b/metricbeat/module/dropwizard/collector/collector_integration_test.go index bc62df6ddf0..f97b7c3f8ba 100644 --- a/metricbeat/module/dropwizard/collector/collector_integration_test.go +++ b/metricbeat/module/dropwizard/collector/collector_integration_test.go @@ -75,14 +75,6 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events) } -func TestData(t *testing.T) { - f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) - - if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { - t.Fatal("write", err) - } -} - func getEnvHost() string { host := os.Getenv("DROPWIZARD_HOST")