Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Metricbeat] Migrate Apache status Metricset to use ReporterV2 interface #10986

Merged
merged 13 commits into from
Apr 11, 2019
Merged

[Metricbeat] Migrate Apache status Metricset to use ReporterV2 interface #10986

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
2e1defc
Atomic commit
sayden Feb 28, 2019
822c9ce
Update unit test too
sayden Feb 28, 2019
354ece6
Use new error interface. Try to use new testing framework
sayden Mar 13, 2019
a92d597
make fmt
sayden Mar 13, 2019
5c646f6
Revert timeout change
sayden Apr 4, 2019
3530422
Rename input file name
sayden Apr 4, 2019
9b99213
Rename expected file
sayden Apr 4, 2019
5b265d5
Generate data.json
sayden Apr 4, 2019
2bb1ee3
Update config.yml file to make tests pass
sayden Apr 4, 2019
05a5e75
Rebase and update expected files
sayden Apr 8, 2019
99ca5e6
Add extra config
sayden Apr 9, 2019
1e9262c
Fix unit tests
sayden Apr 10, 2019
932ee4d
Remove TestData test
sayden Apr 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 27 additions & 26 deletions metricbeat/module/apache/status/_meta/data.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,65 +1,66 @@
{
"@timestamp": "2017-10-12T08:05:34.853Z",
"@timestamp": "2019-03-01T08:05:34.853Z",
"apache": {
"status": {
"bytes_per_request": 78.7692,
"bytes_per_sec": 4.94686,
"bytes_per_request": 112.913,
"bytes_per_sec": 103.832,
"connections": {
"async": {
"closing": 0,
"keep_alive": 1,
"keep_alive": 0,
"writing": 0
},
"total": 1
"total": 0
},
"cpu": {
"children_system": 0,
"children_user": 0,
"load": 3.99758,
"system": 16.39,
"load": 0.125874,
"system": 0.2,
"user": 0.16
},
"hostname": "apache",
"load": {
"1": 170.11,
"15": 77.62,
"5": 167.51
"1": 2,
"15": 1.91,
"5": 1.85
},
"requests_per_sec": 0.0628019,
"requests_per_sec": 0.91958,
"scoreboard": {
"closing_connection": 0,
"dns_lookup": 0,
"gracefully_finishing": 0,
"idle_cleanup": 0,
"keepalive": 0,
"logging": 0,
"open_slot": 300,
"open_slot": 325,
"reading_request": 0,
"sending_reply": 1,
"starting_up": 0,
"total": 400,
"waiting_for_connection": 99
"waiting_for_connection": 74
},
"total_accesses": 26,
"total_kbytes": 2,
"total_accesses": 263,
"total_kbytes": 29,
"uptime": {
"server_uptime": 414,
"uptime": 414
"server_uptime": 286,
"uptime": 286
},
"workers": {
"busy": 1,
"idle": 99
"idle": 74
}
}
},
"beat": {
"hostname": "host.example.com",
"name": "host.example.com"
"event": {
"dataset": "apache.status",
"duration": 115000,
"module": "apache"
},
"metricset": {
"host": "apache",
"module": "apache",
"name": "status",
"rtt": 115
"name": "status"
},
"service": {
"address": "127.0.0.1:55555",
"type": "apache"
}
}
5 changes: 5 additions & 0 deletions metricbeat/module/apache/status/_meta/testdata/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type: http
url: "/server-status?auto="
suffix: plain
remove_fields_from_comparison:
- "apache.status.hostname"
31 changes: 31 additions & 0 deletions metricbeat/module/apache/status/_meta/testdata/docs.plain
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
172.26.0.2
ServerVersion: Apache/2.4.20 (Unix)
ServerMPM: event
Server Built: Jun 7 2016 17:55:26
CurrentTime: Thursday, 04-Apr-2019 09:32:31 UTC
RestartTime: Thursday, 04-Apr-2019 09:27:44 UTC
ParentServerConfigGeneration: 1
ParentServerMPMGeneration: 0
ServerUptimeSeconds: 286
ServerUptime: 4 minutes 46 seconds
Load1: 2.00
Load5: 1.85
Load15: 1.91
Total Accesses: 263
Total kBytes: 29
CPUUser: .16
CPUSystem: .2
CPUChildrenUser: 0
CPUChildrenSystem: 0
CPULoad: .125874
Uptime: 286
ReqPerSec: .91958
BytesPerSec: 103.832
BytesPerReq: 112.913
BusyWorkers: 1
IdleWorkers: 74
ConnsTotal: 0
ConnsAsyncWriting: 0
ConnsAsyncKeepAlive: 0
ConnsAsyncClosing: 0
Scoreboard: ______________________________________W____________________________________.....................................................................................................................................................................................................................................................................................................................................
68 changes: 68 additions & 0 deletions metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
[
{
"apache": {
"status": {
"bytes_per_request": 112.913,
"bytes_per_sec": 103.832,
"connections": {
"async": {
"closing": 0,
"keep_alive": 0,
"writing": 0
},
"total": 0
},
"cpu": {
"children_system": 0,
"children_user": 0,
"load": 0.125874,
"system": 0.2,
"user": 0.16
},
"hostname": "127.0.0.1:43985",
"load": {
"1": 2,
"15": 1.91,
"5": 1.85
},
"requests_per_sec": 0.91958,
"scoreboard": {
"closing_connection": 0,
"dns_lookup": 0,
"gracefully_finishing": 0,
"idle_cleanup": 0,
"keepalive": 0,
"logging": 0,
"open_slot": 325,
"reading_request": 0,
"sending_reply": 1,
"starting_up": 0,
"total": 400,
"waiting_for_connection": 74
},
"total_accesses": 263,
"total_kbytes": 29,
"uptime": {
"server_uptime": 286,
"uptime": 286
},
"workers": {
"busy": 1,
"idle": 74
}
}
},
"event": {
"dataset": "apache.status",
"duration": 115000,
"module": "apache"
},
"metricset": {
"name": "status"
},
"service": {
"address": "127.0.0.1:55555",
"type": "apache"
}
}
]
18 changes: 13 additions & 5 deletions metricbeat/module/apache/status/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
package status

import (
"github.com/elastic/beats/libbeat/common"
"github.com/pkg/errors"

"github.com/elastic/beats/libbeat/logp"
"github.com/elastic/beats/metricbeat/helper"
"github.com/elastic/beats/metricbeat/mb"
Expand Down Expand Up @@ -76,13 +77,20 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
}, nil
}

// Fetch makes an HTTP request to fetch status metrics from the mod_status endpoint.
func (m *MetricSet) Fetch() (common.MapStr, error) {
// Fetch methods implements the data gathering and data conversion to the right
// format. It publishes the event which is then forwarded to the output. In case
// of an error set the Error field of mb.Event or simply call report.Error().
func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
scanner, err := m.http.FetchScanner()
if err != nil {
return nil, err
return errors.Wrap(err, "error fetching data")
}

data, _ := eventMapping(scanner, m.Host())
return data, nil

if reported := reporter.Event(mb.Event{MetricSetFields: data}); !reported {
m.Logger().Error("error reporting event")
}

return nil
}
23 changes: 7 additions & 16 deletions metricbeat/module/apache/status/status_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,31 +32,22 @@ import (
func TestFetch(t *testing.T) {
compose.EnsureUp(t, "apache")

f := mbtest.NewEventFetcher(t, getConfig())
event, err := f.Fetch()
if !assert.NoError(t, err) {
t.FailNow()
f := mbtest.NewReportingMetricSetV2Error(t, getConfig())
events, errs := mbtest.ReportingFetchV2Error(f)
if len(errs) > 0 {
t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
}
assert.NotEmpty(t, events)
event := events[0]

t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)

// Check number of fields.
if len(event) < 11 {
if len(event.MetricSetFields) < 11 {
t.Fatal("Too few top-level elements in the event")
}
}

func TestData(t *testing.T) {
compose.EnsureUp(t, "apache")

f := mbtest.NewEventFetcher(t, getConfig())

err := mbtest.WriteEvent(f, t)
if err != nil {
t.Fatal("write", err)
}
}

func getConfig() map[string]interface{} {
return map[string]interface{}{
"module": "apache",
Expand Down
37 changes: 25 additions & 12 deletions metricbeat/module/apache/status/status_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
Expand Down Expand Up @@ -85,11 +86,13 @@ func TestFetchEventContents(t *testing.T) {
"hosts": []string{server.URL},
}

f := mbtest.NewEventFetcher(t, config)
event, err := f.Fetch()
if !assert.NoError(t, err) {
t.FailNow()
f := mbtest.NewReportingMetricSetV2Error(t, config)
events, errs := mbtest.ReportingFetchV2Error(f)
if len(errs) > 0 {
t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
}
assert.NotEmpty(t, events)
event := events[0].MetricSetFields

t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint())

Expand Down Expand Up @@ -162,13 +165,23 @@ func TestFetchTimeout(t *testing.T) {
"timeout": "50ms",
}

f := mbtest.NewEventFetcher(t, config)
f := mbtest.NewReportingMetricSetV2Error(t, config)

start := time.Now()
_, err := f.Fetch()
events, errs := mbtest.ReportingFetchV2Error(f)
if len(errs) == 0 {
t.Fatalf("Expected an error, had %d. %v\n", len(errs), errs)
}
assert.Empty(t, events)
elapsed := time.Since(start)
if assert.Error(t, err) {
assert.Contains(t, err.Error(), "request canceled (Client.Timeout exceeded")
var found bool
for _, err := range errs {
if strings.Contains(err.Error(), "request canceled (Client.Timeout exceeded") {
found = true
}
}
if !found {
assert.Failf(t, "", "expected an error containing 'request canceled (Client.Timeout exceeded'. Got %v", errs)
}

// Elapsed should be ~50ms, sometimes it can be up to 1s
Expand Down Expand Up @@ -201,12 +214,12 @@ func TestMultipleFetches(t *testing.T) {
"hosts": []string{server.URL},
}

f := mbtest.NewEventFetcher(t, config)
f := mbtest.NewReportingMetricSetV2Error(t, config)

for i := 0; i < 20; i++ {
_, err := f.Fetch()
if !assert.NoError(t, err) {
t.FailNow()
_, errs := mbtest.ReportingFetchV2Error(f)
if len(errs) > 0 {
t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
}
}

Expand Down