From 2e1defc515f57fb7f2bf23c150e42d35dcf7e355 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 28 Feb 2019 12:34:44 +0100 Subject: [PATCH 01/13] Atomic commit --- .../module/apache/status/_meta/data.json | 54 ++++++++++--------- metricbeat/module/apache/status/status.go | 18 +++++-- .../apache/status/status_integration_test.go | 20 ++++--- metricbeat/tests/system/test_apache.py | 2 +- 4 files changed, 57 insertions(+), 37 deletions(-) diff --git a/metricbeat/module/apache/status/_meta/data.json b/metricbeat/module/apache/status/_meta/data.json index e8e3d45845f..00d96b5f31b 100644 --- a/metricbeat/module/apache/status/_meta/data.json +++ b/metricbeat/module/apache/status/_meta/data.json @@ -1,9 +1,13 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, "apache": { "status": { - "bytes_per_request": 78.7692, - "bytes_per_sec": 4.94686, + "bytes_per_request": 105.931, + "bytes_per_sec": 102.4, "connections": { "async": { "closing": 0, @@ -15,17 +19,17 @@ "cpu": { "children_system": 0, "children_user": 0, - "load": 3.99758, - "system": 16.39, - "user": 0.16 + "load": 0.0666667, + "system": 0.01, + "user": 0.01 }, - "hostname": "apache", + "hostname": "172.26.0.2", "load": { - "1": 170.11, - "15": 77.62, - "5": 167.51 + "1": 1.71, + "15": 2.14, + "5": 1.89 }, - "requests_per_sec": 0.0628019, + "requests_per_sec": 0.966667, "scoreboard": { "closing_connection": 0, "dns_lookup": 0, @@ -33,33 +37,35 @@ "idle_cleanup": 0, "keepalive": 0, "logging": 0, - "open_slot": 300, + "open_slot": 325, "reading_request": 0, "sending_reply": 1, "starting_up": 0, "total": 400, - "waiting_for_connection": 99 + "waiting_for_connection": 74 }, - "total_accesses": 26, - "total_kbytes": 2, + "total_accesses": 29, + "total_kbytes": 3, "uptime": { - "server_uptime": 414, - "uptime": 414 + "server_uptime": 30, + "uptime": 30 }, "workers": { "busy": 1, - "idle": 99 + "idle": 74 } } }, - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" + "event": { + "dataset": "apache.status", + "duration": 115000, + "module": "apache" }, "metricset": { - "host": "apache", - "module": "apache", - "name": "status", - "rtt": 115 + "name": "status" + }, + "service": { + "address": "172.26.0.2", + "type": "apache" } } \ No newline at end of file diff --git a/metricbeat/module/apache/status/status.go b/metricbeat/module/apache/status/status.go index c1d6761e489..22f298048bf 100644 --- a/metricbeat/module/apache/status/status.go +++ b/metricbeat/module/apache/status/status.go @@ -19,7 +19,6 @@ package status import ( - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" @@ -49,6 +48,8 @@ var ( DefaultPath: defaultPath, QueryParams: autoQueryParam, }.Build() + + logger = logp.NewLogger("apache.status") ) func init() { @@ -76,13 +77,20 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { }, nil } -// Fetch makes an HTTP request to fetch status metrics from the mod_status endpoint. -func (m *MetricSet) Fetch() (common.MapStr, error) { +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { scanner, err := m.http.FetchScanner() if err != nil { - return nil, err + logger.Error(err) + reporter.Error(err) + return } data, _ := eventMapping(scanner, m.Host()) - return data, nil + + reporter.Event(mb.Event{MetricSetFields: data}) + + return } diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go index 01e7f70b53a..e0c18944ff0 100644 --- a/metricbeat/module/apache/status/status_integration_test.go +++ b/metricbeat/module/apache/status/status_integration_test.go @@ -32,11 +32,13 @@ import ( func TestFetch(t *testing.T) { compose.EnsureUp(t, "apache") - f := mbtest.NewEventFetcher(t, getConfig()) - event, err := f.Fetch() - if !assert.NoError(t, err) { - t.FailNow() + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) @@ -49,10 +51,14 @@ func TestFetch(t *testing.T) { func TestData(t *testing.T) { compose.EnsureUp(t, "apache") - f := mbtest.NewEventFetcher(t, getConfig()) + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) - err := mbtest.WriteEvent(f, t) - if err != nil { + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } diff --git a/metricbeat/tests/system/test_apache.py b/metricbeat/tests/system/test_apache.py index fd164518120..4cf84d3187b 100644 --- a/metricbeat/tests/system/test_apache.py +++ b/metricbeat/tests/system/test_apache.py @@ -55,7 +55,7 @@ def test_output(self): time.sleep(0.5) proc = self.start_beat() - self.wait_until(lambda: self.output_lines() > 0) + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) proc.check_kill_and_wait() self.assert_no_logged_warnings() From 822c9cef86dd43ab73f74c75db2482802887dd1e Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 28 Feb 2019 12:52:02 +0100 Subject: [PATCH 02/13] Update unit test too --- .../module/apache/status/status_test.go | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/metricbeat/module/apache/status/status_test.go b/metricbeat/module/apache/status/status_test.go index 32745626605..1e7073ad21a 100644 --- a/metricbeat/module/apache/status/status_test.go +++ b/metricbeat/module/apache/status/status_test.go @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -// +build !integration - package status import ( @@ -26,6 +24,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -85,11 +84,13 @@ func TestFetchEventContents(t *testing.T) { "hosts": []string{server.URL}, } - f := mbtest.NewEventFetcher(t, config) - event, err := f.Fetch() - if !assert.NoError(t, err) { - t.FailNow() + f := mbtest.NewReportingMetricSetV2(t, config) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) @@ -162,13 +163,23 @@ func TestFetchTimeout(t *testing.T) { "timeout": "50ms", } - f := mbtest.NewEventFetcher(t, config) + f := mbtest.NewReportingMetricSetV2(t, config) start := time.Now() - _, err := f.Fetch() + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) == 0 { + t.Fatalf("Expected an error, had %d. %v\n", len(errs), errs) + } + assert.Empty(t, events) elapsed := time.Since(start) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "request canceled (Client.Timeout exceeded") + var found bool + for _, err := range errs { + if strings.Contains(err.Error(), "request canceled (Client.Timeout exceeded") { + found = true + } + } + if !found { + assert.Failf(t, "", "expected an error containing 'request canceled (Client.Timeout exceeded'. Got %v", errs) } // Elapsed should be ~50ms, sometimes it can be up to 1s @@ -201,12 +212,12 @@ func TestMultipleFetches(t *testing.T) { "hosts": []string{server.URL}, } - f := mbtest.NewEventFetcher(t, config) + f := mbtest.NewReportingMetricSetV2(t, config) for i := 0; i < 20; i++ { - _, err := f.Fetch() - if !assert.NoError(t, err) { - t.FailNow() + _, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } } From 354ece6248f2472bcb5e7c1c6fce443bdc8f9ffa Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 13 Mar 2019 13:10:57 +0100 Subject: [PATCH 03/13] Use new error interface. Try to use new testing framework --- .../apache/status/_meta/testdata/config.yml | 3 + .../apache/status/_meta/testdata/input.plain | 354 ++++++++++++++++++ .../_meta/testdata/input.plain-expected.json | 19 + metricbeat/module/apache/status/status.go | 15 +- .../apache/status/status_integration_test.go | 16 +- .../module/apache/status/status_test.go | 2 + 6 files changed, 389 insertions(+), 20 deletions(-) create mode 100644 metricbeat/module/apache/status/_meta/testdata/config.yml create mode 100644 metricbeat/module/apache/status/_meta/testdata/input.plain create mode 100644 metricbeat/module/apache/status/_meta/testdata/input.plain-expected.json diff --git a/metricbeat/module/apache/status/_meta/testdata/config.yml b/metricbeat/module/apache/status/_meta/testdata/config.yml new file mode 100644 index 00000000000..80d43cee1e1 --- /dev/null +++ b/metricbeat/module/apache/status/_meta/testdata/config.yml @@ -0,0 +1,3 @@ +type: http +url: "/server-status" +suffix: plain diff --git a/metricbeat/module/apache/status/_meta/testdata/input.plain b/metricbeat/module/apache/status/_meta/testdata/input.plain new file mode 100644 index 00000000000..d09b1bac3a6 --- /dev/null +++ b/metricbeat/module/apache/status/_meta/testdata/input.plain @@ -0,0 +1,354 @@ + + +Apache Status + +

Apache Server Status for 172.26.0.2 (via 172.26.0.2)

+ +
Server Version: Apache/2.4.20 (Unix)
+
Server MPM: event
+
Server Built: Jun 7 2016 17:55:26 +
+
Current Time: Wednesday, 13-Mar-2019 12:04:31 UTC
+
Restart Time: Wednesday, 13-Mar-2019 12:02:59 UTC
+
Parent Server Config. Generation: 1
+
Parent Server MPM Generation: 0
+
Server uptime: 1 minute 32 seconds
+
Server load: 0.96 1.48 1.67
+
Total accesses: 86 - Total Traffic: 17 kB
+
CPU Usage: u.01 s.11 cu0 cs0 - .13% CPU load
+
.935 requests/sec - 189 B/second - 202 B/request
+
1 requests currently being processed, 74 idle workers
+
+ + + + + + + + + +
PIDConnectionsThreadsAsync connections
totalacceptingbusyidlewritingkeep-aliveclosing
70yes025000
80yes025000
90yes124000
Sum0 174000
+
_______________________________________________________W________
+___________.....................................................
+................................................................
+................................................................
+................................................................
+................................................................
+................
+

Scoreboard Key:
+"_" Waiting for Connection, +"S" Starting up, +"R" Reading Request,
+"W" Sending Reply, +"K" Keepalive (read), +"D" DNS Lookup,
+"C" Closing connection, +"L" Logging, +"G" Gracefully finishing,
+"I" Idle cleanup of worker, +"." Open slot with no current process
+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SrvPIDAccMCPU +SSReqConnChildSlotClientProtocolVHostRequest
0-070/1/1_ +0.004400.00.000.00 +
0-070/1/1_ +0.024000.00.000.00 +
0-070/1/1_ +0.004000.00.000.00 +
0-070/1/1_ +0.023700.00.000.00 +
0-070/1/1_ +0.003700.00.000.00 +
0-070/1/1_ +0.023500.00.000.00 +
0-070/1/1_ +0.003500.00.000.00 +
0-070/1/1_ +0.003400.00.000.00 +
0-070/1/1_ +0.023400.00.000.00 +
0-070/1/1_ +0.011100.00.000.00 +
0-070/1/1_ +0.032300.00.000.00 +
0-070/1/1_ +0.012300.00.000.00 +
0-070/1/1_ +0.041100.00.000.00 +
0-070/1/1_ +0.041000.00.000.00 +
0-070/1/1_ +0.011000.00.000.00 +
0-070/1/1_ +0.04600.00.000.00 +
0-070/1/1_ +0.01600.00.000.00 +
0-070/1/1_ +0.04200.00.000.00 +
0-070/1/1_ +0.01200.00.000.00 +
0-070/1/1_ +0.015400.00.000.00 +
0-070/1/1_ +0.015200.00.000.00 +
0-070/1/1_ +0.024400.00.010.01 +
1-080/1/1_ +0.004700.00.000.00 +
1-080/1/1_ +0.024300.00.000.00 +
1-080/1/1_ +0.004300.00.000.00 +
1-080/1/1_ +0.024200.00.000.00 +
1-080/1/1_ +0.004200.00.000.00 +
1-080/1/1_ +0.003600.00.000.00 +
1-080/1/1_ +0.003200.00.000.00 +
1-080/1/1_ +0.023600.00.000.00 +
1-080/1/1_ +0.023200.00.000.00 +
1-080/1/1_ +0.023100.00.000.00 +
1-080/1/1_ +0.003100.00.000.00 +
1-080/1/1_ +0.032700.00.000.00 +
1-080/1/1_ +0.002700.00.000.00 +
1-080/1/1_ +0.032500.00.000.00 +
1-080/1/1_ +0.012500.00.000.00 +
1-080/1/1_ +0.032200.00.000.00 +
1-080/1/1_ +0.012200.00.000.00 +
1-080/1/1_ +0.03800.00.000.00 +
1-080/1/1_ +0.01800.00.000.00 +
1-080/1/1_ +0.04300.00.000.00 +
1-080/1/1_ +0.01300.00.000.00 +
1-080/1/1_ +0.04100.00.000.00 +
1-080/1/1_ +0.01100.00.000.00 +
1-080/1/1_ +0.014700.00.000.00 +
2-090/2/2_ +0.02500.00.000.00 +
2-090/2/2_ +0.04000.00.000.00 +
2-090/2/2_ +0.04400.00.000.00 +
2-090/2/2_ +0.02400.00.000.00 +
2-090/2/2_ +0.03000.00.000.00 +
2-090/1/1W +0.01000.00.000.00 +172.26.0.1http/1.1172.26.0.2:80GET /server-status HTTP/1.1
2-090/2/2_ +0.032000.00.000.00 +
2-090/1/1_ +0.012000.00.000.00 +
2-090/2/2_ +0.031900.00.000.00 +
2-090/1/1_ +0.011900.00.000.00 +
2-090/2/2_ +0.031700.00.000.00 +
2-090/1/1_ +0.011700.00.000.00 +
2-090/2/2_ +0.031600.00.000.00 +
2-090/1/1_ +0.021600.00.000.00 +
2-090/2/2_ +0.031500.00.000.00 +
2-090/1/1_ +0.021500.00.000.00 +
2-090/2/2_ +0.031400.00.000.00 +
2-090/1/1_ +0.021400.00.000.00 +
2-090/2/2_ +0.031300.00.000.00 +
2-090/1/1_ +0.021300.00.000.00 +
2-090/2/2_ +0.031200.00.000.00 +
2-090/1/1_ +0.021200.00.000.00 +
2-090/2/2_ +0.03900.00.000.00 +
2-090/1/1_ +0.02900.00.000.00 +
2-090/2/2_ +0.04500.00.000.00 +
+

+ + + + + + + + + + +
SrvChild Server number - generation
PIDOS process ID
AccNumber of accesses this connection / this child / this slot
MMode of operation
CPUCPU usage, number of seconds
SSSeconds since beginning of most recent request
ReqMilliseconds required to process most recent request
ConnKilobytes transferred this connection
ChildMegabytes transferred this child
SlotTotal megabytes transferred this slot
+ diff --git a/metricbeat/module/apache/status/_meta/testdata/input.plain-expected.json b/metricbeat/module/apache/status/_meta/testdata/input.plain-expected.json new file mode 100644 index 00000000000..f31a5d0a920 --- /dev/null +++ b/metricbeat/module/apache/status/_meta/testdata/input.plain-expected.json @@ -0,0 +1,19 @@ +[ + { + "error": { + "message": "error fetching data: HTTP error 404 in status: 404 Not Found" + }, + "event": { + "dataset": "apache.status", + "duration": 115000, + "module": "apache" + }, + "metricset": { + "name": "status" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "apache" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/apache/status/status.go b/metricbeat/module/apache/status/status.go index 22f298048bf..b223e828b24 100644 --- a/metricbeat/module/apache/status/status.go +++ b/metricbeat/module/apache/status/status.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/pkg/errors" ) const ( @@ -48,8 +49,6 @@ var ( DefaultPath: defaultPath, QueryParams: autoQueryParam, }.Build() - - logger = logp.NewLogger("apache.status") ) func init() { @@ -80,17 +79,17 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(reporter mb.ReporterV2) { +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { scanner, err := m.http.FetchScanner() if err != nil { - logger.Error(err) - reporter.Error(err) - return + return errors.Wrap(err, "error fetching data") } data, _ := eventMapping(scanner, m.Host()) - reporter.Event(mb.Event{MetricSetFields: data}) + if reported := reporter.Event(mb.Event{MetricSetFields: data}); !reported { + m.Logger().Error("error reporting event") + } - return + return nil } diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go index e0c18944ff0..8b3ce7c6ab0 100644 --- a/metricbeat/module/apache/status/status_integration_test.go +++ b/metricbeat/module/apache/status/status_integration_test.go @@ -32,8 +32,8 @@ import ( func TestFetch(t *testing.T) { compose.EnsureUp(t, "apache") - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + events, errs := mbtest.ReportingFetchV2Error(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } @@ -49,16 +49,8 @@ func TestFetch(t *testing.T) { } func TestData(t *testing.T) { - compose.EnsureUp(t, "apache") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - - if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { t.Fatal("write", err) } } diff --git a/metricbeat/module/apache/status/status_test.go b/metricbeat/module/apache/status/status_test.go index 1e7073ad21a..9efe67a9fd2 100644 --- a/metricbeat/module/apache/status/status_test.go +++ b/metricbeat/module/apache/status/status_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build !integration + package status import ( From a92d59732ae43333eae857b20a2a8993be7ed15a Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 13 Mar 2019 13:23:54 +0100 Subject: [PATCH 04/13] make fmt --- metricbeat/module/apache/status/status.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/apache/status/status.go b/metricbeat/module/apache/status/status.go index b223e828b24..ddd132841de 100644 --- a/metricbeat/module/apache/status/status.go +++ b/metricbeat/module/apache/status/status.go @@ -19,11 +19,12 @@ package status import ( + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" - "github.com/pkg/errors" ) const ( From 5c646f607c59de4ff91f9cae8cc7c1d977b4bd70 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 4 Apr 2019 13:12:23 +0200 Subject: [PATCH 05/13] Revert timeout change --- metricbeat/tests/system/test_apache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/system/test_apache.py b/metricbeat/tests/system/test_apache.py index 4cf84d3187b..fd164518120 100644 --- a/metricbeat/tests/system/test_apache.py +++ b/metricbeat/tests/system/test_apache.py @@ -55,7 +55,7 @@ def test_output(self): time.sleep(0.5) proc = self.start_beat() - self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + self.wait_until(lambda: self.output_lines() > 0) proc.check_kill_and_wait() self.assert_no_logged_warnings() From 35304223634a8b38dca1f3ca4ebde56d68ea765b Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 4 Apr 2019 13:12:48 +0200 Subject: [PATCH 06/13] Rename input file name --- .../apache/status/_meta/testdata/docs.plain | 31 ++ .../apache/status/_meta/testdata/input.plain | 354 ------------------ 2 files changed, 31 insertions(+), 354 deletions(-) create mode 100644 metricbeat/module/apache/status/_meta/testdata/docs.plain delete mode 100644 metricbeat/module/apache/status/_meta/testdata/input.plain diff --git a/metricbeat/module/apache/status/_meta/testdata/docs.plain b/metricbeat/module/apache/status/_meta/testdata/docs.plain new file mode 100644 index 00000000000..85c68aa9f45 --- /dev/null +++ b/metricbeat/module/apache/status/_meta/testdata/docs.plain @@ -0,0 +1,31 @@ +172.26.0.2 +ServerVersion: Apache/2.4.20 (Unix) +ServerMPM: event +Server Built: Jun 7 2016 17:55:26 +CurrentTime: Thursday, 04-Apr-2019 09:32:31 UTC +RestartTime: Thursday, 04-Apr-2019 09:27:44 UTC +ParentServerConfigGeneration: 1 +ParentServerMPMGeneration: 0 +ServerUptimeSeconds: 286 +ServerUptime: 4 minutes 46 seconds +Load1: 2.00 +Load5: 1.85 +Load15: 1.91 +Total Accesses: 263 +Total kBytes: 29 +CPUUser: .16 +CPUSystem: .2 +CPUChildrenUser: 0 +CPUChildrenSystem: 0 +CPULoad: .125874 +Uptime: 286 +ReqPerSec: .91958 +BytesPerSec: 103.832 +BytesPerReq: 112.913 +BusyWorkers: 1 +IdleWorkers: 74 +ConnsTotal: 0 +ConnsAsyncWriting: 0 +ConnsAsyncKeepAlive: 0 +ConnsAsyncClosing: 0 +Scoreboard: ______________________________________W____________________________________..................................................................................................................................................................................................................................................................................................................................... diff --git a/metricbeat/module/apache/status/_meta/testdata/input.plain b/metricbeat/module/apache/status/_meta/testdata/input.plain deleted file mode 100644 index d09b1bac3a6..00000000000 --- a/metricbeat/module/apache/status/_meta/testdata/input.plain +++ /dev/null @@ -1,354 +0,0 @@ - - -Apache Status - -

Apache Server Status for 172.26.0.2 (via 172.26.0.2)

- -
Server Version: Apache/2.4.20 (Unix)
-
Server MPM: event
-
Server Built: Jun 7 2016 17:55:26 -
-
Current Time: Wednesday, 13-Mar-2019 12:04:31 UTC
-
Restart Time: Wednesday, 13-Mar-2019 12:02:59 UTC
-
Parent Server Config. Generation: 1
-
Parent Server MPM Generation: 0
-
Server uptime: 1 minute 32 seconds
-
Server load: 0.96 1.48 1.67
-
Total accesses: 86 - Total Traffic: 17 kB
-
CPU Usage: u.01 s.11 cu0 cs0 - .13% CPU load
-
.935 requests/sec - 189 B/second - 202 B/request
-
1 requests currently being processed, 74 idle workers
-
- - - - - - - - - -
PIDConnectionsThreadsAsync connections
totalacceptingbusyidlewritingkeep-aliveclosing
70yes025000
80yes025000
90yes124000
Sum0 174000
-
_______________________________________________________W________
-___________.....................................................
-................................................................
-................................................................
-................................................................
-................................................................
-................
-

Scoreboard Key:
-"_" Waiting for Connection, -"S" Starting up, -"R" Reading Request,
-"W" Sending Reply, -"K" Keepalive (read), -"D" DNS Lookup,
-"C" Closing connection, -"L" Logging, -"G" Gracefully finishing,
-"I" Idle cleanup of worker, -"." Open slot with no current process
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SrvPIDAccMCPU -SSReqConnChildSlotClientProtocolVHostRequest
0-070/1/1_ -0.004400.00.000.00 -
0-070/1/1_ -0.024000.00.000.00 -
0-070/1/1_ -0.004000.00.000.00 -
0-070/1/1_ -0.023700.00.000.00 -
0-070/1/1_ -0.003700.00.000.00 -
0-070/1/1_ -0.023500.00.000.00 -
0-070/1/1_ -0.003500.00.000.00 -
0-070/1/1_ -0.003400.00.000.00 -
0-070/1/1_ -0.023400.00.000.00 -
0-070/1/1_ -0.011100.00.000.00 -
0-070/1/1_ -0.032300.00.000.00 -
0-070/1/1_ -0.012300.00.000.00 -
0-070/1/1_ -0.041100.00.000.00 -
0-070/1/1_ -0.041000.00.000.00 -
0-070/1/1_ -0.011000.00.000.00 -
0-070/1/1_ -0.04600.00.000.00 -
0-070/1/1_ -0.01600.00.000.00 -
0-070/1/1_ -0.04200.00.000.00 -
0-070/1/1_ -0.01200.00.000.00 -
0-070/1/1_ -0.015400.00.000.00 -
0-070/1/1_ -0.015200.00.000.00 -
0-070/1/1_ -0.024400.00.010.01 -
1-080/1/1_ -0.004700.00.000.00 -
1-080/1/1_ -0.024300.00.000.00 -
1-080/1/1_ -0.004300.00.000.00 -
1-080/1/1_ -0.024200.00.000.00 -
1-080/1/1_ -0.004200.00.000.00 -
1-080/1/1_ -0.003600.00.000.00 -
1-080/1/1_ -0.003200.00.000.00 -
1-080/1/1_ -0.023600.00.000.00 -
1-080/1/1_ -0.023200.00.000.00 -
1-080/1/1_ -0.023100.00.000.00 -
1-080/1/1_ -0.003100.00.000.00 -
1-080/1/1_ -0.032700.00.000.00 -
1-080/1/1_ -0.002700.00.000.00 -
1-080/1/1_ -0.032500.00.000.00 -
1-080/1/1_ -0.012500.00.000.00 -
1-080/1/1_ -0.032200.00.000.00 -
1-080/1/1_ -0.012200.00.000.00 -
1-080/1/1_ -0.03800.00.000.00 -
1-080/1/1_ -0.01800.00.000.00 -
1-080/1/1_ -0.04300.00.000.00 -
1-080/1/1_ -0.01300.00.000.00 -
1-080/1/1_ -0.04100.00.000.00 -
1-080/1/1_ -0.01100.00.000.00 -
1-080/1/1_ -0.014700.00.000.00 -
2-090/2/2_ -0.02500.00.000.00 -
2-090/2/2_ -0.04000.00.000.00 -
2-090/2/2_ -0.04400.00.000.00 -
2-090/2/2_ -0.02400.00.000.00 -
2-090/2/2_ -0.03000.00.000.00 -
2-090/1/1W -0.01000.00.000.00 -172.26.0.1http/1.1172.26.0.2:80GET /server-status HTTP/1.1
2-090/2/2_ -0.032000.00.000.00 -
2-090/1/1_ -0.012000.00.000.00 -
2-090/2/2_ -0.031900.00.000.00 -
2-090/1/1_ -0.011900.00.000.00 -
2-090/2/2_ -0.031700.00.000.00 -
2-090/1/1_ -0.011700.00.000.00 -
2-090/2/2_ -0.031600.00.000.00 -
2-090/1/1_ -0.021600.00.000.00 -
2-090/2/2_ -0.031500.00.000.00 -
2-090/1/1_ -0.021500.00.000.00 -
2-090/2/2_ -0.031400.00.000.00 -
2-090/1/1_ -0.021400.00.000.00 -
2-090/2/2_ -0.031300.00.000.00 -
2-090/1/1_ -0.021300.00.000.00 -
2-090/2/2_ -0.031200.00.000.00 -
2-090/1/1_ -0.021200.00.000.00 -
2-090/2/2_ -0.03900.00.000.00 -
2-090/1/1_ -0.02900.00.000.00 -
2-090/2/2_ -0.04500.00.000.00 -
-

- - - - - - - - - - -
SrvChild Server number - generation
PIDOS process ID
AccNumber of accesses this connection / this child / this slot
MMode of operation
CPUCPU usage, number of seconds
SSSeconds since beginning of most recent request
ReqMilliseconds required to process most recent request
ConnKilobytes transferred this connection
ChildMegabytes transferred this child
SlotTotal megabytes transferred this slot
- From 9b99213b8345e3c9f668afc29706dbf0a3065a87 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 4 Apr 2019 13:21:57 +0200 Subject: [PATCH 07/13] Rename expected file --- .../_meta/testdata/docs.plain-expected.json | 68 +++++++++++++++++++ .../_meta/testdata/input.plain-expected.json | 19 ------ 2 files changed, 68 insertions(+), 19 deletions(-) create mode 100644 metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json delete mode 100644 metricbeat/module/apache/status/_meta/testdata/input.plain-expected.json diff --git a/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json b/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json new file mode 100644 index 00000000000..eb97ea5d6d5 --- /dev/null +++ b/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json @@ -0,0 +1,68 @@ +[ + { + "apache": { + "status": { + "bytes_per_request": 112.913, + "bytes_per_sec": 103.832, + "connections": { + "async": { + "closing": 0, + "keep_alive": 0, + "writing": 0 + }, + "total": 0 + }, + "cpu": { + "children_system": 0, + "children_user": 0, + "load": 0.125874, + "system": 0.2, + "user": 0.16 + }, + "hostname": "127.0.0.1:33377", + "load": { + "1": 2, + "15": 1.91, + "5": 1.85 + }, + "requests_per_sec": 0.91958, + "scoreboard": { + "closing_connection": 0, + "dns_lookup": 0, + "gracefully_finishing": 0, + "idle_cleanup": 0, + "keepalive": 0, + "logging": 0, + "open_slot": 325, + "reading_request": 0, + "sending_reply": 1, + "starting_up": 0, + "total": 400, + "waiting_for_connection": 74 + }, + "total_accesses": 263, + "total_kbytes": 29, + "uptime": { + "server_uptime": 286, + "uptime": 286 + }, + "workers": { + "busy": 1, + "idle": 74 + } + } + }, + "event": { + "dataset": "apache.status", + "duration": 115000, + "module": "apache" + }, + "metricset": { + "name": "status" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "apache" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/apache/status/_meta/testdata/input.plain-expected.json b/metricbeat/module/apache/status/_meta/testdata/input.plain-expected.json deleted file mode 100644 index f31a5d0a920..00000000000 --- a/metricbeat/module/apache/status/_meta/testdata/input.plain-expected.json +++ /dev/null @@ -1,19 +0,0 @@ -[ - { - "error": { - "message": "error fetching data: HTTP error 404 in status: 404 Not Found" - }, - "event": { - "dataset": "apache.status", - "duration": 115000, - "module": "apache" - }, - "metricset": { - "name": "status" - }, - "service": { - "address": "127.0.0.1:55555", - "type": "apache" - } - } -] \ No newline at end of file From 5b265d53d995c69a8aa27c4e192da0814d8530fb Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 4 Apr 2019 13:22:05 +0200 Subject: [PATCH 08/13] Generate data.json --- .../module/apache/status/_meta/data.json | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/metricbeat/module/apache/status/_meta/data.json b/metricbeat/module/apache/status/_meta/data.json index 00d96b5f31b..e7ddedf789d 100644 --- a/metricbeat/module/apache/status/_meta/data.json +++ b/metricbeat/module/apache/status/_meta/data.json @@ -1,35 +1,31 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, + "@timestamp": "2019-03-01T08:05:34.853Z", "apache": { "status": { - "bytes_per_request": 105.931, - "bytes_per_sec": 102.4, + "bytes_per_request": 112.913, + "bytes_per_sec": 103.832, "connections": { "async": { "closing": 0, - "keep_alive": 1, + "keep_alive": 0, "writing": 0 }, - "total": 1 + "total": 0 }, "cpu": { "children_system": 0, "children_user": 0, - "load": 0.0666667, - "system": 0.01, - "user": 0.01 + "load": 0.125874, + "system": 0.2, + "user": 0.16 }, - "hostname": "172.26.0.2", + "hostname": "127.0.0.1:33377", "load": { - "1": 1.71, - "15": 2.14, - "5": 1.89 + "1": 2, + "15": 1.91, + "5": 1.85 }, - "requests_per_sec": 0.966667, + "requests_per_sec": 0.91958, "scoreboard": { "closing_connection": 0, "dns_lookup": 0, @@ -44,11 +40,11 @@ "total": 400, "waiting_for_connection": 74 }, - "total_accesses": 29, - "total_kbytes": 3, + "total_accesses": 263, + "total_kbytes": 29, "uptime": { - "server_uptime": 30, - "uptime": 30 + "server_uptime": 286, + "uptime": 286 }, "workers": { "busy": 1, @@ -65,7 +61,7 @@ "name": "status" }, "service": { - "address": "172.26.0.2", + "address": "127.0.0.1:55555", "type": "apache" } } \ No newline at end of file From 2bb1ee30c58485bcfb4a975aef73c331b887817c Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 4 Apr 2019 13:23:37 +0200 Subject: [PATCH 09/13] Update config.yml file to make tests pass --- metricbeat/module/apache/status/_meta/testdata/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/apache/status/_meta/testdata/config.yml b/metricbeat/module/apache/status/_meta/testdata/config.yml index 80d43cee1e1..d9dbc048e8b 100644 --- a/metricbeat/module/apache/status/_meta/testdata/config.yml +++ b/metricbeat/module/apache/status/_meta/testdata/config.yml @@ -1,3 +1,3 @@ type: http -url: "/server-status" +url: "/server-status?auto=" suffix: plain From 05a5e750ae950b48fcab2eaf64faa6de52e3f1ad Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 8 Apr 2019 13:55:35 +0200 Subject: [PATCH 10/13] Rebase and update expected files --- metricbeat/module/apache/status/_meta/data.json | 2 +- .../apache/status/_meta/testdata/docs.plain-expected.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/apache/status/_meta/data.json b/metricbeat/module/apache/status/_meta/data.json index e7ddedf789d..137904ac4be 100644 --- a/metricbeat/module/apache/status/_meta/data.json +++ b/metricbeat/module/apache/status/_meta/data.json @@ -19,7 +19,7 @@ "system": 0.2, "user": 0.16 }, - "hostname": "127.0.0.1:33377", + "hostname": "127.0.0.1:33863", "load": { "1": 2, "15": 1.91, diff --git a/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json b/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json index eb97ea5d6d5..1355caf2a41 100644 --- a/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json +++ b/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json @@ -19,7 +19,7 @@ "system": 0.2, "user": 0.16 }, - "hostname": "127.0.0.1:33377", + "hostname": "127.0.0.1:33863", "load": { "1": 2, "15": 1.91, From 99ca5e62aa477f41013a6d2a26cf88ee55087e1f Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 9 Apr 2019 16:22:24 +0200 Subject: [PATCH 11/13] Add extra config --- metricbeat/module/apache/status/_meta/testdata/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metricbeat/module/apache/status/_meta/testdata/config.yml b/metricbeat/module/apache/status/_meta/testdata/config.yml index d9dbc048e8b..a9b46d84d32 100644 --- a/metricbeat/module/apache/status/_meta/testdata/config.yml +++ b/metricbeat/module/apache/status/_meta/testdata/config.yml @@ -1,3 +1,5 @@ type: http url: "/server-status?auto=" suffix: plain +remove_fields_from_comparison: + - "apache.status.hostname" From 1e9262c50b8e9b5a36251d1a1879a922b26b5d9c Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 10 Apr 2019 12:40:14 +0200 Subject: [PATCH 12/13] Fix unit tests --- metricbeat/module/apache/status/_meta/data.json | 1 - .../status/_meta/testdata/docs.plain-expected.json | 2 +- .../module/apache/status/status_integration_test.go | 4 ++-- metricbeat/module/apache/status/status_test.go | 12 ++++++------ 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/apache/status/_meta/data.json b/metricbeat/module/apache/status/_meta/data.json index 137904ac4be..b9b231bb74e 100644 --- a/metricbeat/module/apache/status/_meta/data.json +++ b/metricbeat/module/apache/status/_meta/data.json @@ -19,7 +19,6 @@ "system": 0.2, "user": 0.16 }, - "hostname": "127.0.0.1:33863", "load": { "1": 2, "15": 1.91, diff --git a/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json b/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json index 1355caf2a41..cfc827d23a4 100644 --- a/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json +++ b/metricbeat/module/apache/status/_meta/testdata/docs.plain-expected.json @@ -19,7 +19,7 @@ "system": 0.2, "user": 0.16 }, - "hostname": "127.0.0.1:33863", + "hostname": "127.0.0.1:43985", "load": { "1": 2, "15": 1.91, diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go index 8b3ce7c6ab0..ccb3a2421e9 100644 --- a/metricbeat/module/apache/status/status_integration_test.go +++ b/metricbeat/module/apache/status/status_integration_test.go @@ -38,12 +38,12 @@ func TestFetch(t *testing.T) { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } assert.NotEmpty(t, events) - event := events[0].MetricSetFields + event := events[0] t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) // Check number of fields. - if len(event) < 11 { + if len(event.MetricSetFields) < 11 { t.Fatal("Too few top-level elements in the event") } } diff --git a/metricbeat/module/apache/status/status_test.go b/metricbeat/module/apache/status/status_test.go index 9efe67a9fd2..6c07ef14f36 100644 --- a/metricbeat/module/apache/status/status_test.go +++ b/metricbeat/module/apache/status/status_test.go @@ -86,8 +86,8 @@ func TestFetchEventContents(t *testing.T) { "hosts": []string{server.URL}, } - f := mbtest.NewReportingMetricSetV2(t, config) - events, errs := mbtest.ReportingFetchV2(f) + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } @@ -165,10 +165,10 @@ func TestFetchTimeout(t *testing.T) { "timeout": "50ms", } - f := mbtest.NewReportingMetricSetV2(t, config) + f := mbtest.NewReportingMetricSetV2Error(t, config) start := time.Now() - events, errs := mbtest.ReportingFetchV2(f) + events, errs := mbtest.ReportingFetchV2Error(f) if len(errs) == 0 { t.Fatalf("Expected an error, had %d. %v\n", len(errs), errs) } @@ -214,10 +214,10 @@ func TestMultipleFetches(t *testing.T) { "hosts": []string{server.URL}, } - f := mbtest.NewReportingMetricSetV2(t, config) + f := mbtest.NewReportingMetricSetV2Error(t, config) for i := 0; i < 20; i++ { - _, errs := mbtest.ReportingFetchV2(f) + _, errs := mbtest.ReportingFetchV2Error(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } From 932ee4d26370cb64e530838dd9f2581cad2a648e Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 11 Apr 2019 10:37:46 +0200 Subject: [PATCH 13/13] Remove TestData test --- metricbeat/module/apache/status/status_integration_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go index ccb3a2421e9..96a60083146 100644 --- a/metricbeat/module/apache/status/status_integration_test.go +++ b/metricbeat/module/apache/status/status_integration_test.go @@ -48,13 +48,6 @@ func TestFetch(t *testing.T) { } } -func TestData(t *testing.T) { - f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) - if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { - t.Fatal("write", err) - } -} - func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "apache",