From a89ac5b26b53ce6b2c305d3231228ef047134033 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 28 Dec 2018 12:16:51 +0100 Subject: [PATCH 1/9] Atomic commit --- x-pack/metricbeat/include/list.go | 1 + x-pack/metricbeat/metricbeat.reference.yml | 1 + .../metricbeat/module/mssql/_meta/config.yml | 1 + x-pack/metricbeat/module/mssql/fetcher.go | 14 +-- x-pack/metricbeat/module/mssql/fields.go | 2 +- .../mssql/performance/_meta/docs.asciidoc | 13 +++ .../module/mssql/performance/_meta/fields.yml | 67 +++++++++++++ .../module/mssql/performance/data.go | 38 +++++++ .../performance/data_integration_test.go | 23 +++++ .../module/mssql/performance/performance.go | 76 ++++++++++++++ .../performance_integration_test.go | 99 +++++++++++++++++++ .../metricbeat/modules.d/mssql.yml.disabled | 1 + x-pack/metricbeat/tests/system/test_mssql.py | 26 +++++ 13 files changed, 354 insertions(+), 8 deletions(-) create mode 100644 x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/mssql/performance/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/mssql/performance/data.go create mode 100644 x-pack/metricbeat/module/mssql/performance/data_integration_test.go create mode 100644 x-pack/metricbeat/module/mssql/performance/performance.go create mode 100644 x-pack/metricbeat/module/mssql/performance/performance_integration_test.go diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index be12c4e52d8..5bf352e8b3e 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -10,4 +10,5 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/db" + _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" ) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 771a259f70e..db6d6640e2d 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -491,6 +491,7 @@ metricbeat.modules: - module: mssql metricsets: - "db" + - "performance" hosts: ["sqlserver://sa@localhost"] period: 10s diff --git a/x-pack/metricbeat/module/mssql/_meta/config.yml b/x-pack/metricbeat/module/mssql/_meta/config.yml index 4008489e10d..038955a319c 100644 --- a/x-pack/metricbeat/module/mssql/_meta/config.yml +++ b/x-pack/metricbeat/module/mssql/_meta/config.yml @@ -1,6 +1,7 @@ - module: mssql metricsets: - "db" + - "performance" hosts: ["sqlserver://sa@localhost"] period: 10s diff --git a/x-pack/metricbeat/module/mssql/fetcher.go b/x-pack/metricbeat/module/mssql/fetcher.go index 692d901ca9c..f822a075330 100644 --- a/x-pack/metricbeat/module/mssql/fetcher.go +++ b/x-pack/metricbeat/module/mssql/fetcher.go @@ -140,13 +140,13 @@ func (rr *rowsResultHandler) handle(s *s.Schema) (err error) { } } - result, err := s.Apply(mapOfResults) - if err != nil { - err = errors.Wrap(err, "error trying to apply schema") - logp.Error(err) - rr.reporter.Error(err) - continue - } + result, _ := s.Apply(mapOfResults) + //if err != nil { + // err = errors.Wrap(err, "error trying to apply schema") + // logp.Error(err) + // rr.reporter.Error(err) + // continue + //} rr.reporter.Event(mb.Event{MetricSetFields: result}) } diff --git a/x-pack/metricbeat/module/mssql/fields.go b/x-pack/metricbeat/module/mssql/fields.go index 390f0abf4b8..869f0623cc8 100644 --- a/x-pack/metricbeat/module/mssql/fields.go +++ b/x-pack/metricbeat/module/mssql/fields.go @@ -18,5 +18,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJyslEGPmzAQhe/8iqe97x/gUKnSXiq1hyq9R4MZghtjs55ho/TXVzghCwGiZJXcMn5+/ubNiFfs+ZijEXl3GaBWHed4Sf9fMqBkMdG2aoPP8WuDze+faELZOc6AyI5JOEfBShlQWXal5BkAvMJTw5/O/U+PLefYxdC158rE/tu5OLYZW5XFpbTkNfMrCzSs0RphxcE6h4rV1LC+CrGhXgQqQqdgMnUvr2JoQJB3B+H4wXFkfU01ISOlgoQnh2uMM863820cao4MrXngRk0fjILZn9C5TIhXZktkYzpbzo6uCP7UfOkBP94W5KdW9nw8hFguZuDCbistGd52QrsvRrHpDZAMJmOqQky5aCQvZFLNhd2DQWhQcqvNLcHNk+otIJ+YCxfWMMYoxVFZFhULoxH7jxGqlMC862kXLvi5YHi1E17ahUf6rxnBmK61XN7B9fQo1t6G9TeNboQzZmmN3kPyHS1Hw17p7wVilYwENOiHetrEdNqrbzJXLtAcagAW6w1vHYluCzL7+wZITehOLMMiJ2xvTh+f3i2xrTreXpinD/0x3gdW4X8AAAD//45S2Xs=" + return "eJzMmMFu4zYQhu9+isHesw/gQ4E2eymwBbZN7sKIHImsKVLhUHbdpy+Gsh1Zllw7ldHsZRGR/PnNPxMOmSfY0H4NDfObWwEkmxyt4Uv++csKQBOraNtkg1/Dby/w8vt3aILuHK0AIjlCpjWUlHAFUFlymtcrAIAn8NjQu7L8S/uW1lDH0LWHL2fyPx0+DmWGUro8fZrSutDTJTSUolVMCXbWOagoKQPWVyE2KJMAy9AlIFRGplcxNIDAbw6Y4pbiQHpMdUaGCUtkOhucY7zg/HZYDTtDkSAZOnKDwS1BSeR7dNIZcSQ2RTaks/piaETwaugUA/z6bWJ6H8qG9rsQ9aQHLtQFt6io6BjrD1rxIgKQBc7SVIWYfUkRPaPK31yo7zQihYRuNrgpuEunRAL4HXNiwRzGEKXcJ+LJGROpYfs3QaiyA5dRn0fhgr+ccNy1Y5qqhXviNwRBqa61pG/gWtyKub3B+qtCV8wZsrQq3ULyM7QUFfmEf54gZsmQAY/zj99zJeZRmX2VuXIBL6GOwGy9osIhp6JEtbktgdiErmc5FnLG9qo/fEQts80qXi+YxZN+H+8HSuGUfor5zPGK7ms1g4WDntOf2TzRcAT7x2DNswRIkW/sNy3WVHDrbBoHeeM5+9w1ncNktyToYD0n4fgKLybsQNuqYmnqO+k8LUUbtFUQCbX1NUMKYDX5ZKv9YZQlOVWkt06KXPCgx/t65xHNpP6tgn+8q0uqmVTweirZE4med9PZigr6qyUlTuw/ZqsUa69BGpJtaAAIyWACzH22dyhfSSI1aL3Mk5oou6qiCG0I4z61lHESJ7zHuYyDLqhNscPPVY7G1iaTgQo+yXjwDyjH77KDxD5O+H/ws2OKhQreU77qjKUmbiS+a0qKErashfm1tyJEUqFprcN5kf8xrfIMifQ0RHxAav8482DB/H52ax/s6/NjXB08DpZx9fgkWsDV4cNF/tvatH+As68DCxZ0tsSkTJHbO3++M/7A9UhffxEDjhst6mxu+IVCZagwNhVRfi0mHdahK934rXDe4A+Pkjq/P04XDYYqdF7ne3F/v8jbwc4mI5dRg1vra7FY7O7/CKItb1b/BAAA//8/AQXE" } diff --git a/x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc b/x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc new file mode 100644 index 00000000000..46b32a34150 --- /dev/null +++ b/x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc @@ -0,0 +1,13 @@ +`performance` Metricset fetches information from what's commonly known as https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-performance-counters-transact-sql?view=sql-server-2017[Performance Counters] in MSSQL. + +We fetch the following data: + +* Page splits per instance: Cumulative per instance. Show diffs between periodic readings to identify periods of frequent page splits. +* Page life expectancy in seconds: The expected time in seconds that a data page will remain in the buffer pool. +* Lock wait time in seconds: Cumulative per instance. Show diffs between periodic readings to identify periods of high lock contention. +* Total number of user connections. +* Cumulative (per instance) recompilations time in seconds: Show diffs between periodic readings to identify periods of high SQL re-compilations. +* Compilations time in seconds: Cumulative per instance. Show diffs between periodic readings to identify periods of high SQL compilations. +* Transactions time in seconds: Cumulative per database. Show diffs between periodic readings to identify periods of high transaction activity. +* Batch requests time in seconds: Cumulative per instance. Show diffs between periodic readings to identify periods of high request activity. +* Buffer Cache hit ratio: Percentage of data pages found in buffer cache without having to read from disk. diff --git a/x-pack/metricbeat/module/mssql/performance/_meta/fields.yml b/x-pack/metricbeat/module/mssql/performance/_meta/fields.yml new file mode 100644 index 00000000000..513ccfdc37d --- /dev/null +++ b/x-pack/metricbeat/module/mssql/performance/_meta/fields.yml @@ -0,0 +1,67 @@ +- name: performance + type: group + description: performance metricset fetches information about the Performance Counters + fields: + - name: page_splits + type: group + description: Cumulative per instance. Show diffs between periodic readings to identify periods of frequent page splits. + fields: + - name: sec + description: Page splits in seconds + type: long + + - name: page_life_expectancy + type: group + description: The expected time in seconds that a data page will remain in the buffer pool + fields: + - name: sec + description: Page life expectancy in seconds + type: long + + - name: lock_waits + type: group + description: Cumulative per instance. Show diffs between periodic readings to identify periods of high lock contention. + fields: + - name: sec + description: Lock wait time in seconds + type: long + + - name: user_connections + description: Total number of user connections + type: long + + - name: recompilations + type: group + description: Cumulative per instance. Show diffs between periodic readings to identify periods of high SQL re-compilations. + fields: + - name: sec + description: Recompilations time in seconds + type: long + + - name: compilations + type: group + description: Cumulative per instance. Show diffs between periodic readings to identify periods of high SQL compilations. + fields: + - name: sec + description: Compilations time in seconds + type: long + + - name: transactions + type: group + description: Cumulative per database. Show diffs between periodic readings to identify periods of high transaction activity. + fields: + - name: sec + description: Transactions time in seconds + type: long + + - name: batch_requests + type: group + description: Cumulative per instance. Show diffs between periodic readings to identify periods of high request activity. + fields: + - name: sec + description: Batch requests time in seconds + type: long + + - name: buffer_cache_hit_ratio + type: double + description: Percentage of data pages found in buffer cache without having to read from disk diff --git a/x-pack/metricbeat/module/mssql/performance/data.go b/x-pack/metricbeat/module/mssql/performance/data.go new file mode 100644 index 00000000000..ff97b7cd2e9 --- /dev/null +++ b/x-pack/metricbeat/module/mssql/performance/data.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package performance + +import ( + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstrstr" +) + +var ( + schema = s.Schema{ + "page_splits": s.Object{ + "sec": c.Int("page_splits_sec"), + }, + "page_life_expectancy": s.Object{ + "sec": c.Int("page_life_expectancy"), + }, + "lock_waits": s.Object{ + "sec": c.Int("lock_waits_sec"), + }, + "user_connections": c.Int("user_connections"), + "recompilations": s.Object{ + "sec": c.Int("recompilations_sec"), + }, + "compilations": s.Object{ + "sec": c.Int("compilations_sec"), + }, + "transactions": s.Object{ + "sec": c.Int("transactions_sec"), + }, + "batch_requests": s.Object{ + "sec": c.Int("batch_req_sec"), + }, + "buffer_cache_hit_ratio": c.Float("buffer_cache_hit_ratio"), + } +) diff --git a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go new file mode 100644 index 00000000000..0436116a77e --- /dev/null +++ b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package performance + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing" +) + +func TestData(t *testing.T) { + t.Skip("Skipping `data.json` generation test") + + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("performance")) + + err := mbtest.WriteEventsReporterV2(f, t, "") + if err != nil { + t.Fatal("write", err) + } +} diff --git a/x-pack/metricbeat/module/mssql/performance/performance.go b/x-pack/metricbeat/module/mssql/performance/performance.go new file mode 100644 index 00000000000..1f6fa048d73 --- /dev/null +++ b/x-pack/metricbeat/module/mssql/performance/performance.go @@ -0,0 +1,76 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package performance + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/mssql" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("mssql", "performance", New, + mb.DefaultMetricSet(), + mb.WithHostParser(mssql.HostParser)) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + log *logp.Logger + fetcher *mssql.Fetcher +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The mssql performance metricset is beta.") + + logger := logp.NewLogger("mssql.performance").With("host", base.HostData().SanitizedURI) + + fetcher, err := mssql.NewFetcher(base.HostData().URI, + []string{ + `SELECT [cntr_value] as page_life_expectancy FROM sys.dm_os_performance_counters WHERE [object_name] = 'SQLServer:Buffer Manager' AND [counter_name] = 'Page life expectancy'`, + `SELECT (a.cntr_value * 1.0 / b.cntr_value) * 100.0 as buffer_cache_hit_ratio FROM sys.dm_os_performance_counters a JOIN (SELECT cntr_value,OBJECT_NAME FROM sys.dm_os_performance_counters WHERE counter_name = 'Buffer cache hit ratio base' AND OBJECT_NAME = 'SQLServer:Buffer Manager') b ON a.OBJECT_NAME = b.OBJECT_NAME WHERE a.counter_name = 'Buffer cache hit ratio' AND a.OBJECT_NAME = 'SQLServer:Buffer Manager';`, + "SELECT cntr_value as batch_req_sec FROM sys.dm_os_performance_counters WHERE counter_name = 'Batch Requests/sec';", + "SELECT cntr_value as transactions_sec, instance_name as db FROM sys.dm_os_performance_counters where counter_name = 'Transactions/sec';", + "SELECT cntr_value as compilations_sec FROM sys.dm_os_performance_counters where counter_name = 'SQL Compilations/sec';", + "SELECT cntr_value as recompilations_sec FROM sys.dm_os_performance_counters where counter_name = 'SQL Re-Compilations/sec';", + "SELECT cntr_value as user_connections FROM sys.dm_os_performance_counters WHERE counter_name = 'User Connections';", + "SELECT cntr_value as lock_waits_sec FROM sys.dm_os_performance_counters WHERE counter_name = 'Lock Waits/sec' and instance_name = '_Total';", + "SELECT cntr_value as page_splits_sec FROM sys.dm_os_performance_counters WHERE counter_name = 'Page splits/sec'", + }, &schema, logger) + if err != nil { + return nil, errors.Wrap(err, "error creating fetcher") + } + + return &MetricSet{ + BaseMetricSet: base, + log: logger, + fetcher: fetcher, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { + m.fetcher.Report(reporter) +} + +// Close the connection to the server at the engine level +func (m *MetricSet) Close() error { + return m.fetcher.Close() +} diff --git a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go new file mode 100644 index 00000000000..71953373a2e --- /dev/null +++ b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go @@ -0,0 +1,99 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package performance + +import ( + "testing" + + "github.com/elastic/beats/libbeat/logp" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing" +) + +type keyAssertion struct { + key string + assertion func(v interface{}, key string) +} + +func TestFetch(t *testing.T) { + logp.TestingSetup() + compose.EnsureUp(t, "mssql") + + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("performance")) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + float64Assertion := func(f func(float64) bool) func(v interface{}, key string) { + return func(v interface{}, key string) { + value, ok := v.(float64) + if !ok { + t.Fatalf("%v is not a int64, but %T", key, v) + } + + assert.Truef(t, f(value), "Value '%d' on field '%s' wasn't higher than 0", value, key) + } + } + + int64Assertion := func(f func(int64) bool) func(v interface{}, key string) { + return func(v interface{}, key string) { + value, ok := v.(int64) + if !ok { + t.Fatalf("%v is not a int64, but %T", key, v) + } + + assert.Truef(t, f(value), "Value '%d' on field '%s' wasn't higher than 0", value, key) + } + } + + int64HigherThanZero := func(v int64) bool { + return v > 0 + } + + int64EqualZero := func(v int64) bool { + return v == 0 + } + + float64HigherThanZero := func(v float64) bool { + return v > 0 + } + + keys := []keyAssertion{ + {key: "page_splits.sec", assertion: int64Assertion(int64HigherThanZero)}, + {key: "page_life_expectancy.sec", assertion: int64Assertion(int64HigherThanZero)}, + {key: "lock_waits.sec", assertion: int64Assertion(int64HigherThanZero)}, + {key: "user_connections", assertion: int64Assertion(int64HigherThanZero)}, + {key: "recompilations.sec", assertion: int64Assertion(int64EqualZero)}, + {key: "compilations.sec", assertion: int64Assertion(int64HigherThanZero)}, + {key: "transactions.sec", assertion: int64Assertion(int64HigherThanZero)}, + {key: "batch_requests.sec", assertion: int64Assertion(int64HigherThanZero)}, + {key: "buffer_cache_hit_ratio", assertion: float64Assertion(float64HigherThanZero)}, + } + for _, keyAssertion := range keys { + var found bool + + for _, event := range events { + pageSplitsSeconds, err := event.MetricSetFields.GetValue(keyAssertion.key) + if err != nil { + continue + } + found = true + + keyAssertion.assertion(pageSplitsSeconds, keyAssertion.key) + } + + if !found { + t.Fatalf("Key '%s' not found", keyAssertion.key) + } + } +} diff --git a/x-pack/metricbeat/modules.d/mssql.yml.disabled b/x-pack/metricbeat/modules.d/mssql.yml.disabled index 4008489e10d..038955a319c 100644 --- a/x-pack/metricbeat/modules.d/mssql.yml.disabled +++ b/x-pack/metricbeat/modules.d/mssql.yml.disabled @@ -1,6 +1,7 @@ - module: mssql metricsets: - "db" + - "performance" hosts: ["sqlserver://sa@localhost"] period: 10s diff --git a/x-pack/metricbeat/tests/system/test_mssql.py b/x-pack/metricbeat/tests/system/test_mssql.py index ca2927f20c6..b5158a0c460 100644 --- a/x-pack/metricbeat/tests/system/test_mssql.py +++ b/x-pack/metricbeat/tests/system/test_mssql.py @@ -43,6 +43,32 @@ def test_status(self): self.assertTrue(evt["mssql"]["db"]["log_space_usage"]["used"]["pct"] > 0) self.assertTrue(evt["mssql"]["db"]["log_space_usage"]["used"]["bytes"] > 0) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_performance(self): + """ + MSSQL module outputs an event. + """ + self.render_config_template(modules=[{ + "name": "mssql", + "metricsets": ["performance"], + "hosts": self.get_hosts(), + "username": self.get_username(), + "password": self.get_password(), + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 8) + evt = output[0] + + self.assertItemsEqual(self.de_dot(MSSQL_FIELDS), evt.keys()) + self.assertTrue(evt["mssql"]["performance"]["page_life_expectancy"]["sec"] > 0) + self.assert_fields_are_documented(evt) def get_hosts(self): From cea896e6f1af12dd1ad2bd7efa3895f65442fe95 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 28 Dec 2018 15:10:33 +0100 Subject: [PATCH 2/9] Add missing check (removed by mistake) --- x-pack/metricbeat/tests/system/test_mssql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/metricbeat/tests/system/test_mssql.py b/x-pack/metricbeat/tests/system/test_mssql.py index b5158a0c460..d7bbb901d86 100644 --- a/x-pack/metricbeat/tests/system/test_mssql.py +++ b/x-pack/metricbeat/tests/system/test_mssql.py @@ -43,6 +43,8 @@ def test_status(self): self.assertTrue(evt["mssql"]["db"]["log_space_usage"]["used"]["pct"] > 0) self.assertTrue(evt["mssql"]["db"]["log_space_usage"]["used"]["bytes"] > 0) + self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @attr('integration') def test_performance(self): From d777639c5a91c9ac4777c3170c8b06dc631a7128 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 28 Dec 2018 15:12:02 +0100 Subject: [PATCH 3/9] data.json generated file --- .../module/mssql/performance/_meta/data.json | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 x-pack/metricbeat/module/mssql/performance/_meta/data.json diff --git a/x-pack/metricbeat/module/mssql/performance/_meta/data.json b/x-pack/metricbeat/module/mssql/performance/_meta/data.json new file mode 100644 index 00000000000..36361cbacfc --- /dev/null +++ b/x-pack/metricbeat/module/mssql/performance/_meta/data.json @@ -0,0 +1,32 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "dataset": "mssql.performance", + "duration": 115000, + "module": "mssql" + }, + "metricset": { + "name": "performance" + }, + "mssql": { + "performance": { + "batch_requests": {}, + "compilations": {}, + "lock_waits": {}, + "page_life_expectancy": { + "sec": 11 + }, + "page_splits": {}, + "recompilations": {}, + "transactions": {} + } + }, + "service": { + "address": "172.26.0.2", + "type": "mssql" + } +} \ No newline at end of file From 759f860b1d804bf2d625b32f3e39136407fb77b9 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 28 Dec 2018 15:14:44 +0100 Subject: [PATCH 4/9] Remove error check on schema apply --- x-pack/metricbeat/module/mssql/fetcher.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/metricbeat/module/mssql/fetcher.go b/x-pack/metricbeat/module/mssql/fetcher.go index f822a075330..7120ed77b9f 100644 --- a/x-pack/metricbeat/module/mssql/fetcher.go +++ b/x-pack/metricbeat/module/mssql/fetcher.go @@ -141,12 +141,6 @@ func (rr *rowsResultHandler) handle(s *s.Schema) (err error) { } result, _ := s.Apply(mapOfResults) - //if err != nil { - // err = errors.Wrap(err, "error trying to apply schema") - // logp.Error(err) - // rr.reporter.Error(err) - // continue - //} rr.reporter.Event(mb.Event{MetricSetFields: result}) } From 6c5a6347077fb41627ad86d8ac2f4f7816246e1e Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 9 Jan 2019 18:41:35 +0100 Subject: [PATCH 5/9] Use no abstractions and leave only instance performance metrics. --- CHANGELOG.next.asciidoc | 1 + x-pack/metricbeat/module/mssql/connection.go | 26 ++++++ x-pack/metricbeat/module/mssql/fields.go | 2 +- .../module/mssql/performance/_meta/data.json | 27 +++++-- .../mssql/performance/_meta/docs.asciidoc | 3 +- .../module/mssql/performance/_meta/fields.yml | 18 ++--- .../module/mssql/performance/data.go | 21 +++-- .../performance/data_integration_test.go | 46 ++++++++++- .../module/mssql/performance/performance.go | 80 ++++++++++++++----- .../performance_integration_test.go | 9 +-- 10 files changed, 174 insertions(+), 59 deletions(-) create mode 100644 x-pack/metricbeat/module/mssql/connection.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6eb0839fede..5e834945e5b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -109,6 +109,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `key` metricset to the Redis module. {issue}9582[9582] {pull}9657[9657] {pull}9746[9746] - Add `socket_summary` metricset to system defaults, removing experimental tag and supporting Windows {pull}9709[9709] - Add docker `event` metricset. {pull}9856[9856] +- Add 'performance' metricset to x-pack mssql module {pull}9826[9826] *Packetbeat* diff --git a/x-pack/metricbeat/module/mssql/connection.go b/x-pack/metricbeat/module/mssql/connection.go new file mode 100644 index 00000000000..7fe550f10e5 --- /dev/null +++ b/x-pack/metricbeat/module/mssql/connection.go @@ -0,0 +1,26 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package mssql + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func NewConnection(uri string) (*sql.DB, error) { + db, err := sql.Open("sqlserver", uri) + if err != nil { + return nil, errors.Wrap(err, "could not create db instance") + } + + // Check the connection before executing all queries to reduce the number + // of connection errors that we might encounter. + if err = db.Ping(); err != nil { + err = errors.Wrap(err, "error doing ping to db") + } + + return db, err +} diff --git a/x-pack/metricbeat/module/mssql/fields.go b/x-pack/metricbeat/module/mssql/fields.go index 869f0623cc8..61a95fddce2 100644 --- a/x-pack/metricbeat/module/mssql/fields.go +++ b/x-pack/metricbeat/module/mssql/fields.go @@ -18,5 +18,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzMmMFu4zYQhu9+isHesw/gQ4E2eymwBbZN7sKIHImsKVLhUHbdpy+Gsh1Zllw7ldHsZRGR/PnNPxMOmSfY0H4NDfObWwEkmxyt4Uv++csKQBOraNtkg1/Dby/w8vt3aILuHK0AIjlCpjWUlHAFUFlymtcrAIAn8NjQu7L8S/uW1lDH0LWHL2fyPx0+DmWGUro8fZrSutDTJTSUolVMCXbWOagoKQPWVyE2KJMAy9AlIFRGplcxNIDAbw6Y4pbiQHpMdUaGCUtkOhucY7zg/HZYDTtDkSAZOnKDwS1BSeR7dNIZcSQ2RTaks/piaETwaugUA/z6bWJ6H8qG9rsQ9aQHLtQFt6io6BjrD1rxIgKQBc7SVIWYfUkRPaPK31yo7zQihYRuNrgpuEunRAL4HXNiwRzGEKXcJ+LJGROpYfs3QaiyA5dRn0fhgr+ccNy1Y5qqhXviNwRBqa61pG/gWtyKub3B+qtCV8wZsrQq3ULyM7QUFfmEf54gZsmQAY/zj99zJeZRmX2VuXIBL6GOwGy9osIhp6JEtbktgdiErmc5FnLG9qo/fEQts80qXi+YxZN+H+8HSuGUfor5zPGK7ms1g4WDntOf2TzRcAT7x2DNswRIkW/sNy3WVHDrbBoHeeM5+9w1ncNktyToYD0n4fgKLybsQNuqYmnqO+k8LUUbtFUQCbX1NUMKYDX5ZKv9YZQlOVWkt06KXPCgx/t65xHNpP6tgn+8q0uqmVTweirZE4med9PZigr6qyUlTuw/ZqsUa69BGpJtaAAIyWACzH22dyhfSSI1aL3Mk5oou6qiCG0I4z61lHESJ7zHuYyDLqhNscPPVY7G1iaTgQo+yXjwDyjH77KDxD5O+H/ws2OKhQreU77qjKUmbiS+a0qKErashfm1tyJEUqFprcN5kf8xrfIMifQ0RHxAav8482DB/H52ax/s6/NjXB08DpZx9fgkWsDV4cNF/tvatH+As68DCxZ0tsSkTJHbO3++M/7A9UhffxEDjhst6mxu+IVCZagwNhVRfi0mHdahK934rXDe4A+Pkjq/P04XDYYqdF7ne3F/v8jbwc4mI5dRg1vra7FY7O7/CKItb1b/BAAA//8/AQXE" + return "eJzMl8GO4zYMhu95CmLvsw+QQ4F29rLAFph2ejdoiY7ZyJJHlJOmT19QTiaOY6fJNEEnt0gU9fEnIdJPsKbdEhqRN7cASJwcLeFL/v9lAWBJTOQ2cfBL+PUVXn/7AU2wnaMFQCRHKLSEkhIuAComZ2W5AAB4Ao8NHT3rL+1aWsIqhq7dr5y4/2m/OHQzdGXL96UpX2f+bAkNpchGKMGWnYOKkqmBfRVig2oEWIYuAaGp1byKoQEEeXMgFDcUB67HVCdkmLBEoZPNOcYzzm/707CtKRKkmg7cUOOGoCTyPTrZjDhyNkU2pGN7tjUi+KOm9xjg+7cJ8z6UNe22IdpJDVxYFdKioaITXH1Qild1ANnBSZqqELMuKaIXNHnNhdWNQqSQ0M0GNwV3rpS6ADliThyYwxiilLtEMmkxkRrhvwlClRU4j/o0Chf8ucHh1k5oqhZuib8mCMZ0LZO9guvuUszdDewvOrogzpClNekakp+hpWjIJ/zzHWKWDAXwYH9Yz5WYd9X6InPlAp5DHYCFvaHCoaSiRLO+LoHYhK5nORRyxvamf3zUW2ab9Xi5YO6e9Nt4P1AK7+mnmN8cb+i2VjM4OOg5/ZstEw1HsV8GZ541QIpyZb9pcUWFtI7TOMgr39nnrukcJt6QogN7ScrxFV7rsAXLVSXa1LfaeVqKHCwbiISW/UogBWBLPnG12++KJqeK9NZpkSse9Hhfb3yihcy/VfDL0bumWsgEb6eSPZHoeTUdV1TQXy0ZVWL3MVm1WHsfZCFxQwNASDUmwNxne4XySBKpQfZqpzVRdlVFEdoQxn3qXsJpnHCM8z4KumDWxRY/VznWvKozGZjgk+4H/4By/KE3aOzjhP8HPTuhWJjgPeVRZ+xqYiLxXVNS1LD1LMyfvRYhkglNyw7nnfyPadXPkEhPQ8QHpPb3Ew3umN/PLu2DdX1+jKolJlMXuQnJ53uJ9lygXy8bTrsH6PqLCnC46K7K5rZUGDQ1FTWPx9Ertf3uLRtMJLnVHSboVR6WtSEKVKHzdtQL86Ww5VTr4FTjhv1KhVbR+w92y7K+UczpOf+sl0etUWAZzOzHh7YHq3UMsbxhSxbK3QVLF8K6awXChuJxbq1oCxqaoLf9WIDGkMjk+NoLbUNXOlr8EwAA///t9ux6" } diff --git a/x-pack/metricbeat/module/mssql/performance/_meta/data.json b/x-pack/metricbeat/module/mssql/performance/_meta/data.json index 36361cbacfc..6384fe9e1ea 100644 --- a/x-pack/metricbeat/module/mssql/performance/_meta/data.json +++ b/x-pack/metricbeat/module/mssql/performance/_meta/data.json @@ -14,15 +14,28 @@ }, "mssql": { "performance": { - "batch_requests": {}, - "compilations": {}, - "lock_waits": {}, + "batch_requests": { + "sec": 109695 + }, + "buffer_cache_hit": { + "pct": 0.54 + }, + "compilations": { + "sec": 28814 + }, + "lock_waits": { + "sec": 3 + }, "page_life_expectancy": { - "sec": 11 + "sec": 31783 + }, + "page_splits": { + "sec": 166 + }, + "recompilations": { + "sec": 0 }, - "page_splits": {}, - "recompilations": {}, - "transactions": {} + "user_connections": 3 } }, "service": { diff --git a/x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc b/x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc index 46b32a34150..f414b12de15 100644 --- a/x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/mssql/performance/_meta/docs.asciidoc @@ -8,6 +8,5 @@ We fetch the following data: * Total number of user connections. * Cumulative (per instance) recompilations time in seconds: Show diffs between periodic readings to identify periods of high SQL re-compilations. * Compilations time in seconds: Cumulative per instance. Show diffs between periodic readings to identify periods of high SQL compilations. -* Transactions time in seconds: Cumulative per database. Show diffs between periodic readings to identify periods of high transaction activity. * Batch requests time in seconds: Cumulative per instance. Show diffs between periodic readings to identify periods of high request activity. -* Buffer Cache hit ratio: Percentage of data pages found in buffer cache without having to read from disk. +* Buffer Cache hit: Percentage of data pages found in buffer cache without having to read from disk. diff --git a/x-pack/metricbeat/module/mssql/performance/_meta/fields.yml b/x-pack/metricbeat/module/mssql/performance/_meta/fields.yml index 513ccfdc37d..5fd0a32a88f 100644 --- a/x-pack/metricbeat/module/mssql/performance/_meta/fields.yml +++ b/x-pack/metricbeat/module/mssql/performance/_meta/fields.yml @@ -46,14 +46,6 @@ description: Compilations time in seconds type: long - - name: transactions - type: group - description: Cumulative per database. Show diffs between periodic readings to identify periods of high transaction activity. - fields: - - name: sec - description: Transactions time in seconds - type: long - - name: batch_requests type: group description: Cumulative per instance. Show diffs between periodic readings to identify periods of high request activity. @@ -62,6 +54,10 @@ description: Batch requests time in seconds type: long - - name: buffer_cache_hit_ratio - type: double - description: Percentage of data pages found in buffer cache without having to read from disk + - name: buffer_cache_hit + type: group + description: Indicates the percentage of pages found in the buffer cache without having to read from disk + fields: + - name: pct + description: The ratio is the total number of cache hits divided by the total number of cache lookups over the last few thousand page accesses + type: double diff --git a/x-pack/metricbeat/module/mssql/performance/data.go b/x-pack/metricbeat/module/mssql/performance/data.go index ff97b7cd2e9..34c2d319fe5 100644 --- a/x-pack/metricbeat/module/mssql/performance/data.go +++ b/x-pack/metricbeat/module/mssql/performance/data.go @@ -12,27 +12,26 @@ import ( var ( schema = s.Schema{ "page_splits": s.Object{ - "sec": c.Int("page_splits_sec"), + "sec": c.Int("Page Splits/sec", s.Optional), }, "page_life_expectancy": s.Object{ - "sec": c.Int("page_life_expectancy"), + "sec": c.Int("Page life expectancy", s.Optional), }, "lock_waits": s.Object{ - "sec": c.Int("lock_waits_sec"), + "sec": c.Int("Lock Waits/sec", s.Optional), }, - "user_connections": c.Int("user_connections"), + "user_connections": c.Int("User Connections", s.Optional), "recompilations": s.Object{ - "sec": c.Int("recompilations_sec"), + "sec": c.Int("SQL Re-Compilations/sec", s.Optional), }, "compilations": s.Object{ - "sec": c.Int("compilations_sec"), - }, - "transactions": s.Object{ - "sec": c.Int("transactions_sec"), + "sec": c.Int("SQL Compilations/sec", s.Optional), }, "batch_requests": s.Object{ - "sec": c.Int("batch_req_sec"), + "sec": c.Int("Batch Requests/sec", s.Optional), + }, + "buffer_cache_hit": s.Object{ + "pct": c.Float("Buffer cache hit ratio", s.Optional), }, - "buffer_cache_hit_ratio": c.Float("buffer_cache_hit_ratio"), } ) diff --git a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go index 0436116a77e..9c688593783 100644 --- a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go +++ b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go @@ -5,19 +5,59 @@ package performance import ( + "net/url" "testing" + _ "github.com/denisenkom/go-mssqldb" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing" ) func TestData(t *testing.T) { t.Skip("Skipping `data.json` generation test") + _, config, err := getHostURI() + if err != nil { + t.Fatal("error getting config information", err.Error()) + } - f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("performance")) + f := mbtest.NewReportingMetricSetV2(t, config) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) - err := mbtest.WriteEventsReporterV2(f, t, "") - if err != nil { + if err = mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } + +func getHostURI() (string, map[string]interface{}, error) { + config := mtest.GetConfig("performance") + + host, ok := config["hosts"].([]string) + if !ok { + return "", nil, errors.New("error getting host name information") + } + + username, ok := config["username"].(string) + if !ok { + return "", nil, errors.New("error getting username information") + } + + password, ok := config["password"].(string) + if !ok { + return "", nil, errors.New("error getting password information") + } + + u := &url.URL{ + Scheme: "sqlserver", + User: url.UserPassword(username, password), + Host: host[0], + } + + return u.String(), config, nil +} diff --git a/x-pack/metricbeat/module/mssql/performance/performance.go b/x-pack/metricbeat/module/mssql/performance/performance.go index 1f6fa048d73..ea27babf944 100644 --- a/x-pack/metricbeat/module/mssql/performance/performance.go +++ b/x-pack/metricbeat/module/mssql/performance/performance.go @@ -5,6 +5,12 @@ package performance import ( + "database/sql" + "fmt" + "strings" + + "github.com/elastic/beats/libbeat/common" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common/cfgwarn" @@ -13,6 +19,13 @@ import ( "github.com/elastic/beats/x-pack/metricbeat/module/mssql" ) +type performanceCounter struct { + objectName string + instanceName string + counterName string + counterValue *int64 +} + // init registers the MetricSet with the central registry as soon as the program // starts. The New function will be called later to instantiate an instance of // the MetricSet for each host defined in the module's configuration. After the @@ -31,6 +44,7 @@ type MetricSet struct { mb.BaseMetricSet log *logp.Logger fetcher *mssql.Fetcher + db *sql.DB } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -40,26 +54,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { logger := logp.NewLogger("mssql.performance").With("host", base.HostData().SanitizedURI) - fetcher, err := mssql.NewFetcher(base.HostData().URI, - []string{ - `SELECT [cntr_value] as page_life_expectancy FROM sys.dm_os_performance_counters WHERE [object_name] = 'SQLServer:Buffer Manager' AND [counter_name] = 'Page life expectancy'`, - `SELECT (a.cntr_value * 1.0 / b.cntr_value) * 100.0 as buffer_cache_hit_ratio FROM sys.dm_os_performance_counters a JOIN (SELECT cntr_value,OBJECT_NAME FROM sys.dm_os_performance_counters WHERE counter_name = 'Buffer cache hit ratio base' AND OBJECT_NAME = 'SQLServer:Buffer Manager') b ON a.OBJECT_NAME = b.OBJECT_NAME WHERE a.counter_name = 'Buffer cache hit ratio' AND a.OBJECT_NAME = 'SQLServer:Buffer Manager';`, - "SELECT cntr_value as batch_req_sec FROM sys.dm_os_performance_counters WHERE counter_name = 'Batch Requests/sec';", - "SELECT cntr_value as transactions_sec, instance_name as db FROM sys.dm_os_performance_counters where counter_name = 'Transactions/sec';", - "SELECT cntr_value as compilations_sec FROM sys.dm_os_performance_counters where counter_name = 'SQL Compilations/sec';", - "SELECT cntr_value as recompilations_sec FROM sys.dm_os_performance_counters where counter_name = 'SQL Re-Compilations/sec';", - "SELECT cntr_value as user_connections FROM sys.dm_os_performance_counters WHERE counter_name = 'User Connections';", - "SELECT cntr_value as lock_waits_sec FROM sys.dm_os_performance_counters WHERE counter_name = 'Lock Waits/sec' and instance_name = '_Total';", - "SELECT cntr_value as page_splits_sec FROM sys.dm_os_performance_counters WHERE counter_name = 'Page splits/sec'", - }, &schema, logger) + db, err := mssql.NewConnection(base.HostData().URI) if err != nil { - return nil, errors.Wrap(err, "error creating fetcher") + return nil, errors.Wrap(err, "could not create connection to db") } return &MetricSet{ BaseMetricSet: base, log: logger, - fetcher: fetcher, + db: db, }, nil } @@ -67,10 +70,49 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. func (m *MetricSet) Fetch(reporter mb.ReporterV2) { - m.fetcher.Report(reporter) -} + var err error + var rows *sql.Rows + rows, err = m.db.Query("SELECT object_name, counter_name, instance_name, cntr_value FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Compilations/sec' OR counter_name = 'SQL Re-Compilations/sec' OR counter_name = 'User Connections' OR counter_name = 'Page splits/sec' OR (counter_name = 'Lock Waits/sec' AND instance_name = '_Total') OR counter_name = 'Page splits/sec' OR (object_name = 'SQLServer:Buffer Manager' AND counter_name = 'Page life expectancy') OR counter_name = 'Batch Requests/sec' OR (counter_name = 'Buffer cache hit ratio' AND object_name = 'SQLServer:Buffer Manager') OR (counter_name = 'Lock Waits/sec' and instance_name = '_Total')") + if err != nil { + reporter.Error(errors.Wrapf(err, "error closing rows")) + return + } + defer func() { + if err := rows.Close(); err != nil { + m.log.Error("error closing rows: %s", err.Error()) + } + }() + + mapStr := common.MapStr{} + for rows.Next() { + var row performanceCounter + if err = rows.Scan(&row.objectName, &row.counterName, &row.instanceName, &row.counterValue); err != nil { + reporter.Error(errors.Wrap(err, "error scanning rows")) + continue + } + + //cell values contains spaces at the beginning and at the end of the 'actual' value. They must be removed. + row.counterName = strings.TrimSpace(row.counterName) + row.instanceName = strings.TrimSpace(row.instanceName) + row.objectName = strings.TrimSpace(row.objectName) + + if row.counterName == "Buffer cache hit ratio" { + mapStr[row.counterName] = fmt.Sprintf("%v", float64(*row.counterValue)/100) + } else { + mapStr[row.counterName] = fmt.Sprintf("%v", *row.counterValue) + } + } + + res, err := schema.Apply(mapStr) + if err != nil { + m.log.Error(errors.Wrap(err, "error applying schema")) + return + } + + if isReported := reporter.Event(mb.Event{ + MetricSetFields: res, + }); !isReported { + m.log.Warn("event not reported") + } -// Close the connection to the server at the engine level -func (m *MetricSet) Close() error { - return m.fetcher.Close() } diff --git a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go index 71953373a2e..ad3b10250ec 100644 --- a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go +++ b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go @@ -38,7 +38,7 @@ func TestFetch(t *testing.T) { return func(v interface{}, key string) { value, ok := v.(float64) if !ok { - t.Fatalf("%v is not a int64, but %T", key, v) + t.Fatalf("%v is not a float64, but %T", key, v) } assert.Truef(t, f(value), "Value '%d' on field '%s' wasn't higher than 0", value, key) @@ -75,21 +75,20 @@ func TestFetch(t *testing.T) { {key: "user_connections", assertion: int64Assertion(int64HigherThanZero)}, {key: "recompilations.sec", assertion: int64Assertion(int64EqualZero)}, {key: "compilations.sec", assertion: int64Assertion(int64HigherThanZero)}, - {key: "transactions.sec", assertion: int64Assertion(int64HigherThanZero)}, {key: "batch_requests.sec", assertion: int64Assertion(int64HigherThanZero)}, - {key: "buffer_cache_hit_ratio", assertion: float64Assertion(float64HigherThanZero)}, + {key: "buffer_cache_hit.pct", assertion: float64Assertion(float64HigherThanZero)}, } for _, keyAssertion := range keys { var found bool for _, event := range events { - pageSplitsSeconds, err := event.MetricSetFields.GetValue(keyAssertion.key) + value, err := event.MetricSetFields.GetValue(keyAssertion.key) if err != nil { continue } found = true - keyAssertion.assertion(pageSplitsSeconds, keyAssertion.key) + keyAssertion.assertion(value, keyAssertion.key) } if !found { From 0f24e9debd6444d9fe020df6257eccaef883170b Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 9 Jan 2019 18:54:58 +0100 Subject: [PATCH 6/9] Add Close method to the metricset --- x-pack/metricbeat/module/mssql/performance/performance.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/metricbeat/module/mssql/performance/performance.go b/x-pack/metricbeat/module/mssql/performance/performance.go index ea27babf944..908f762e32c 100644 --- a/x-pack/metricbeat/module/mssql/performance/performance.go +++ b/x-pack/metricbeat/module/mssql/performance/performance.go @@ -114,5 +114,8 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { }); !isReported { m.log.Warn("event not reported") } +} +func (m *MetricSet) Close() error { + return m.db.Close() } From 960ef57344ad8605c7cb8748c32fd57e2cd1782e Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 9 Jan 2019 22:42:16 +0100 Subject: [PATCH 7/9] Fix broken test --- x-pack/metricbeat/tests/system/test_mssql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/tests/system/test_mssql.py b/x-pack/metricbeat/tests/system/test_mssql.py index d7bbb901d86..fdde45931d7 100644 --- a/x-pack/metricbeat/tests/system/test_mssql.py +++ b/x-pack/metricbeat/tests/system/test_mssql.py @@ -65,7 +65,7 @@ def test_performance(self): self.assert_no_logged_warnings() output = self.read_output_json() - self.assertEqual(len(output), 8) + self.assertEqual(len(output), 1) evt = output[0] self.assertItemsEqual(self.de_dot(MSSQL_FIELDS), evt.keys()) From 62a5626ab397fbee7fdb3634496b97e2da745f31 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 10 Jan 2019 12:19:54 +0100 Subject: [PATCH 8/9] Revert changes in fetcher --- x-pack/metricbeat/module/mssql/fetcher.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/mssql/fetcher.go b/x-pack/metricbeat/module/mssql/fetcher.go index 7120ed77b9f..692d901ca9c 100644 --- a/x-pack/metricbeat/module/mssql/fetcher.go +++ b/x-pack/metricbeat/module/mssql/fetcher.go @@ -140,7 +140,13 @@ func (rr *rowsResultHandler) handle(s *s.Schema) (err error) { } } - result, _ := s.Apply(mapOfResults) + result, err := s.Apply(mapOfResults) + if err != nil { + err = errors.Wrap(err, "error trying to apply schema") + logp.Error(err) + rr.reporter.Error(err) + continue + } rr.reporter.Event(mb.Event{MetricSetFields: result}) } From 6da41fd5198fed64778f65eb2036163529037a1f Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 10 Jan 2019 12:20:12 +0100 Subject: [PATCH 9/9] Make Hound happy. Use debug instead of warn in a message --- x-pack/metricbeat/module/mssql/connection.go | 1 + x-pack/metricbeat/module/mssql/performance/performance.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/mssql/connection.go b/x-pack/metricbeat/module/mssql/connection.go index 7fe550f10e5..3528e2371be 100644 --- a/x-pack/metricbeat/module/mssql/connection.go +++ b/x-pack/metricbeat/module/mssql/connection.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" ) +// NewConnection returns a connection already established with MSSQL func NewConnection(uri string) (*sql.DB, error) { db, err := sql.Open("sqlserver", uri) if err != nil { diff --git a/x-pack/metricbeat/module/mssql/performance/performance.go b/x-pack/metricbeat/module/mssql/performance/performance.go index 908f762e32c..26f87735437 100644 --- a/x-pack/metricbeat/module/mssql/performance/performance.go +++ b/x-pack/metricbeat/module/mssql/performance/performance.go @@ -112,10 +112,11 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { if isReported := reporter.Event(mb.Event{ MetricSetFields: res, }); !isReported { - m.log.Warn("event not reported") + m.log.Debug("event not reported") } } +// Close closes the db connection to MS SQL at the Metricset level func (m *MetricSet) Close() error { return m.db.Close() }