diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a1ce267b613..826b1f33414 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -262,6 +262,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for CouchDB v2 {issue}16352[16352] {pull}16455[16455] - Add dashboards for the azure container metricsets. {pull}17194[17194] - Replace vpc metricset into vpn, transitgateway and natgateway metricsets. {pull}16892[16892] +- Use Elasticsearch histogram type to store Prometheus histograms {pull}17061[17061] +- Allow to rate Prometheus counters when scraping them {pull}17061[17061] - Release Oracle module as GA. {issue}14279[14279] {pull}16833[16833] - Release vsphere module as GA. {issue}15798[15798] {pull}17119[17119] - Add Storage metricsets to GCP module {pull}15598[15598] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e4ff9ac5692..bfa22a28ff3 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66,6 +66,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -31639,6 +31640,53 @@ query metricset remote write metrics from Prometheus server +[[exported-fields-prometheus-xpack]] +== Prometheus typed metrics fields + +Stats scraped from a Prometheus endpoint. + + + +*`prometheus.*.value`*:: ++ +-- +Prometheus gauge metric + + +type: object + +-- + +*`prometheus.*.counter`*:: ++ +-- +Prometheus counter metric + + +type: object + +-- + +*`prometheus.*.rate`*:: ++ +-- +Prometheus rated counter metric + + +type: object + +-- + +*`prometheus.*.histogram`*:: ++ +-- +Prometheus histogram metric - release: ga + + +type: object + +-- + [[exported-fields-rabbitmq]] == RabbitMQ fields diff --git a/metricbeat/mb/registry.go b/metricbeat/mb/registry.go index 780213d2392..62ee4f6c1d0 100644 --- a/metricbeat/mb/registry.go +++ b/metricbeat/mb/registry.go @@ -68,6 +68,7 @@ type MetricSetRegistration struct { IsDefault bool HostParser HostParser Namespace string + Replace bool } // MetricSetOption sets an option for a MetricSetFactory that is being @@ -99,6 +100,15 @@ func WithNamespace(namespace string) MetricSetOption { } } +// MustReplace specifies that the MetricSetFactory must be replacing an existing +// metricset with the same name. An error will happen if there is no metricset +// defined with the same params. +func MustReplace() MetricSetOption { + return func(r *MetricSetRegistration) { + r.Replace = true + } +} + // Register contains the factory functions for creating new Modules and new // MetricSets. Registers are thread safe for concurrent usage. type Register struct { @@ -201,22 +211,28 @@ func (r *Register) addMetricSet(module, name string, factory MetricSetFactory, o module = strings.ToLower(module) name = strings.ToLower(name) + // Set the options. + msInfo := MetricSetRegistration{Name: name, Factory: factory} + for _, opt := range options { + opt(&msInfo) + } + if metricsets, ok := r.metricSets[module]; !ok { + if msInfo.Replace { + return fmt.Errorf("metricset '%s/%s' should be replacing an existing metricset, none found", module, name) + } + r.metricSets[module] = map[string]MetricSetRegistration{} } else if _, exists := metricsets[name]; exists { - return fmt.Errorf("metricset '%s/%s' is already registered", module, name) + if !msInfo.Replace { + return fmt.Errorf("metricset '%s/%s' is already registered", module, name) + } } if factory == nil { return fmt.Errorf("metricset '%s/%s' cannot be registered with a nil factory", module, name) } - // Set the options. - msInfo := MetricSetRegistration{Name: name, Factory: factory} - for _, opt := range options { - opt(&msInfo) - } - r.metricSets[module][name] = msInfo r.log.Infof("MetricSet registered: %s/%s", module, name) return nil diff --git a/metricbeat/mb/registry_test.go b/metricbeat/mb/registry_test.go index f4ba90f4dd4..98ed4ce8605 100644 --- a/metricbeat/mb/registry_test.go +++ b/metricbeat/mb/registry_test.go @@ -192,6 +192,14 @@ func TestDefaultMetricSet(t *testing.T) { assert.Contains(t, names, metricSetName) } +func TestMustReplaceMetricSet(t *testing.T) { + registry := NewRegister() + err := registry.addMetricSet(moduleName, metricSetName, fakeMetricSetFactory, MustReplace()) + if assert.Error(t, err) { + assert.Equal(t, "metricset 'mymodule/mymetricset' should be replacing an existing metricset, none found", err.Error()) + } +} + func TestMetricSetQuery(t *testing.T) { registry := NewRegister() err := registry.AddMetricSet(moduleName, metricSetName, fakeMetricSetFactory) diff --git a/metricbeat/mb/testing/testdata.go b/metricbeat/mb/testing/testdata.go index ca1005e57c3..a0cf85d4b01 100644 --- a/metricbeat/mb/testing/testdata.go +++ b/metricbeat/mb/testing/testdata.go @@ -336,9 +336,22 @@ func documentedFieldCheck(foundKeys common.MapStr, knownKeys map[string]interfac return nil } } - // If a field is defined as object it can also be defined as `status_codes.*` - // So this checks if such a key with the * exists by removing the last part. + // If a field is defined as object it can also have a * somewhere + // So this checks if such a key with the * exists by testing with it splits := strings.Split(foundKey, ".") + found := false + for pos := 1; pos < len(splits)-1; pos++ { + key := strings.Join(splits[0:pos], ".") + ".*." + strings.Join(splits[pos+1:len(splits)], ".") + if _, ok := knownKeys[key]; ok { + found = true + break + } + } + if found { + continue + } + + // last case `status_codes.*`: prefix := strings.Join(splits[0:len(splits)-1], ".") if _, ok := knownKeys[prefix+".*"]; ok { continue diff --git a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc index 82d58cecab2..e48e57ee209 100644 --- a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc @@ -24,6 +24,58 @@ to retrieve the metrics from (`/metrics` by default) can be configured with `met ------------------------------------------------------------------------------------- +[float] +[role="xpack"] +=== Histograms and types + +beta[] + +[source,yaml] +------------------------------------------------------------------------------------- +metricbeat.modules: +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + use_types: true + rate_counters: false +------------------------------------------------------------------------------------- + +`use_types` paramater (default: false) enables a different layout for metrics storage, leveraging Elasticsearch +types, including https://www.elastic.co/guide/en/elasticsearch/reference/current/histogram.html[histograms]. + +`rate_counters` paramater (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores +the counter increment since the last collection. This metric should make some aggregations easier and with better +performance. This parameter can only be enabled in combination with `use_types`. + +When `use_types` and `rate_counters` are enabled, metrics are stored like this: + +[source,json] +---- +{ + "prometheus": { + "labels": { + "instance": "172.27.0.2:9090", + "job": "prometheus" + }, + "prometheus_target_interval_length_seconds_count": { + "counter": 1, + "rate": 0 + }, + "prometheus_target_interval_length_seconds_sum": { + "counter": 15.000401344, + "rate": 0 + } + "prometheus_tsdb_compaction_chunk_range_seconds_bucket": { + "histogram": { + "values": [50, 300, 1000, 4000, 16000], + "counts": [10, 2, 34, 7] + } + } + }, +} +---- + + [float] === Scraping all metrics from a Prometheus server diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index fa78f64294d..690672ff829 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -19,6 +19,7 @@ package collector import ( "regexp" + "sync" "github.com/pkg/errors" dto "github.com/prometheus/client_model/go" @@ -35,20 +36,43 @@ const ( ) var ( - hostParser = parse.URLHostParserBuilder{ + // HostParser parses a Prometheus endpoint URL + HostParser = parse.URLHostParserBuilder{ DefaultScheme: defaultScheme, DefaultPath: defaultPath, PathConfigKey: "metrics_path", }.Build() + + upMetricName = "up" + upMetricType = dto.MetricType_GAUGE + upMetricInstanceLabel = "instance" + upMetricJobLabel = "job" + upMetricJobValue = "prometheus" ) func init() { - mb.Registry.MustAddMetricSet("prometheus", "collector", MetricSetBuilder("prometheus"), - mb.WithHostParser(hostParser), + mb.Registry.MustAddMetricSet("prometheus", "collector", + MetricSetBuilder("prometheus", DefaultPromEventsGeneratorFactory), + mb.WithHostParser(HostParser), mb.DefaultMetricSet(), ) } +// PromEventsGenerator converts a Prometheus metric family into a PromEvent list +type PromEventsGenerator interface { + // Start must be called before using the generator + Start() + + // converts a Prometheus metric family into a list of PromEvents + GeneratePromEvents(mf *dto.MetricFamily) []PromEvent + + // Stop must be called when the generator won't be used anymore + Stop() +} + +// PromEventsGeneratorFactory creates a PromEventsGenerator when instanciating a metricset +type PromEventsGeneratorFactory func(ms mb.BaseMetricSet) (PromEventsGenerator, error) + // MetricSet for fetching prometheus data type MetricSet struct { mb.BaseMetricSet @@ -56,10 +80,14 @@ type MetricSet struct { includeMetrics []*regexp.Regexp excludeMetrics []*regexp.Regexp namespace string + promEventsGen PromEventsGenerator + once sync.Once + host string } -// MetricSetBuilder returns a builder function for a new Prometheus metricset using the given namespace -func MetricSetBuilder(namespace string) func(base mb.BaseMetricSet) (mb.MetricSet, error) { +// MetricSetBuilder returns a builder function for a new Prometheus metricset using +// the given namespace and event generator +func MetricSetBuilder(namespace string, genFactory PromEventsGeneratorFactory) func(base mb.BaseMetricSet) (mb.MetricSet, error) { return func(base mb.BaseMetricSet) (mb.MetricSet, error) { config := defaultConfig if err := base.Module().UnpackConfig(&config); err != nil { @@ -70,11 +98,19 @@ func MetricSetBuilder(namespace string) func(base mb.BaseMetricSet) (mb.MetricSe return nil, err } + promEventsGen, err := genFactory(base) + if err != nil { + return nil, err + } + ms := &MetricSet{ BaseMetricSet: base, prometheus: prometheus, namespace: namespace, + promEventsGen: promEventsGen, } + // store host here to use it as a pointer when building `up` metric + ms.host = ms.Host() ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics) if err != nil { return nil, errors.Wrapf(err, "unable to compile exclude patterns") @@ -90,54 +126,52 @@ func MetricSetBuilder(namespace string) func(base mb.BaseMetricSet) (mb.MetricSe // Fetch fetches data and reports it func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + m.once.Do(m.promEventsGen.Start) + families, err := m.prometheus.GetFamilies() eventList := map[string]common.MapStr{} if err != nil { - m.addUpEvent(eventList, 0) - for _, evt := range eventList { - reporter.Event(mb.Event{ - RootFields: common.MapStr{m.namespace: evt}, - }) - } - return errors.Wrap(err, "unable to decode response from prometheus endpoint") + // send up event only + families = append(families, m.upMetricFamily(0.0)) + + // set the error to report it after sending the up event + err = errors.Wrap(err, "unable to decode response from prometheus endpoint") + } else { + // add up event to the list + families = append(families, m.upMetricFamily(1.0)) } for _, family := range families { if m.skipFamily(family) { continue } - promEvents := getPromEventsFromMetricFamily(family) + promEvents := m.promEventsGen.GeneratePromEvents(family) for _, promEvent := range promEvents { labelsHash := promEvent.LabelsHash() if _, ok := eventList[labelsHash]; !ok { - eventList[labelsHash] = common.MapStr{ - "metrics": common.MapStr{}, - } + eventList[labelsHash] = common.MapStr{} // Add default instance label if not already there - if exists, _ := promEvent.labels.HasKey("instance"); !exists { - promEvent.labels.Put("instance", m.Host()) + if exists, _ := promEvent.Labels.HasKey(upMetricInstanceLabel); !exists { + promEvent.Labels.Put(upMetricInstanceLabel, m.Host()) } // Add default job label if not already there - if exists, _ := promEvent.labels.HasKey("job"); !exists { - promEvent.labels.Put("job", m.Module().Name()) + if exists, _ := promEvent.Labels.HasKey("job"); !exists { + promEvent.Labels.Put("job", m.Module().Name()) } // Add labels - if len(promEvent.labels) > 0 { - eventList[labelsHash]["labels"] = promEvent.labels + if len(promEvent.Labels) > 0 { + eventList[labelsHash]["labels"] = promEvent.Labels } } - // Not checking anything here because we create these maps some lines before - metrics := eventList[labelsHash]["metrics"].(common.MapStr) - metrics.Update(promEvent.data) + // Accumulate metrics in the event + eventList[labelsHash].DeepUpdate(promEvent.Data) } } - m.addUpEvent(eventList, 1) - - // Converts hash list to slice + // Report events for _, e := range eventList { isOpen := reporter.Event(mb.Event{ RootFields: common.MapStr{m.namespace: e}, @@ -147,27 +181,36 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } } + return err +} + +// Close stops the metricset +func (m *MetricSet) Close() error { + m.promEventsGen.Stop() return nil } -func (m *MetricSet) addUpEvent(eventList map[string]common.MapStr, up int) { - metricName := "up" - if m.skipFamilyName(metricName) { - return +func (m *MetricSet) upMetricFamily(value float64) *dto.MetricFamily { + gauge := dto.Gauge{ + Value: &value, } - upPromEvent := PromEvent{ - labels: common.MapStr{ - "instance": m.Host(), - "job": "prometheus", - }, + label1 := dto.LabelPair{ + Name: &upMetricInstanceLabel, + Value: &m.host, } - eventList[upPromEvent.LabelsHash()] = common.MapStr{ - "metrics": common.MapStr{ - "up": up, - }, - "labels": upPromEvent.labels, + label2 := dto.LabelPair{ + Name: &upMetricJobLabel, + Value: &upMetricJobValue, + } + metric := dto.Metric{ + Gauge: &gauge, + Label: []*dto.LabelPair{&label1, &label2}, + } + return &dto.MetricFamily{ + Name: &upMetricName, + Type: &upMetricType, + Metric: []*dto.Metric{&metric}, } - } func (m *MetricSet) skipFamily(family *dto.MetricFamily) bool { diff --git a/metricbeat/module/prometheus/collector/collector_test.go b/metricbeat/module/prometheus/collector/collector_test.go index 88a361c623e..94477a0aa2b 100644 --- a/metricbeat/module/prometheus/collector/collector_test.go +++ b/metricbeat/module/prometheus/collector/collector_test.go @@ -63,10 +63,12 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ - "http_request_duration_microseconds": float64(10), + Data: common.MapStr{ + "metrics": common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, }, - labels: labels, + Labels: labels, }, }, }, @@ -85,10 +87,12 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ - "http_request_duration_microseconds": float64(10), + Data: common.MapStr{ + "metrics": common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, }, - labels: common.MapStr{}, + Labels: common.MapStr{}, }, }, }, @@ -114,17 +118,21 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ - "http_request_duration_microseconds_count": uint64(10), - "http_request_duration_microseconds_sum": float64(10), + Data: common.MapStr{ + "metrics": common.MapStr{ + "http_request_duration_microseconds_count": uint64(10), + "http_request_duration_microseconds_sum": float64(10), + }, }, - labels: common.MapStr{}, + Labels: common.MapStr{}, }, { - data: common.MapStr{ - "http_request_duration_microseconds": float64(10), + Data: common.MapStr{ + "metrics": common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, }, - labels: common.MapStr{ + Labels: common.MapStr{ "quantile": "0.99", }, }, @@ -152,17 +160,21 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ - "http_request_duration_microseconds_count": uint64(10), - "http_request_duration_microseconds_sum": float64(10), + Data: common.MapStr{ + "metrics": common.MapStr{ + "http_request_duration_microseconds_count": uint64(10), + "http_request_duration_microseconds_sum": float64(10), + }, }, - labels: common.MapStr{}, + Labels: common.MapStr{}, }, { - data: common.MapStr{ - "http_request_duration_microseconds_bucket": uint64(10), + Data: common.MapStr{ + "metrics": common.MapStr{ + "http_request_duration_microseconds_bucket": uint64(10), + }, }, - labels: common.MapStr{"le": "0.99"}, + Labels: common.MapStr{"le": "0.99"}, }, }, }, @@ -187,17 +199,20 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, Event: []PromEvent{ { - data: common.MapStr{ - "http_request_duration_microseconds": float64(10), + Data: common.MapStr{ + "metrics": common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, }, - labels: labels, + Labels: labels, }, }, }, } + p := promEventGenerator{} for _, test := range tests { - event := getPromEventsFromMetricFamily(test.Family) + event := p.GeneratePromEvents(test.Family) assert.Equal(t, test.Event, event) } } @@ -320,7 +335,7 @@ func TestSkipMetricFamily(t *testing.T) { metricsToKeep := 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { - metricsToKeep += 1 + metricsToKeep++ } } assert.Equal(t, metricsToKeep, len(testFamilies)) @@ -331,7 +346,7 @@ func TestSkipMetricFamily(t *testing.T) { metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { - metricsToKeep += 1 + metricsToKeep++ } } assert.Equal(t, metricsToKeep, 2) @@ -342,7 +357,7 @@ func TestSkipMetricFamily(t *testing.T) { metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { - metricsToKeep += 1 + metricsToKeep++ } } assert.Equal(t, len(testFamilies)-2, metricsToKeep) @@ -353,7 +368,7 @@ func TestSkipMetricFamily(t *testing.T) { metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { - metricsToKeep += 1 + metricsToKeep++ } } assert.Equal(t, 1, metricsToKeep) diff --git a/metricbeat/module/prometheus/collector/data.go b/metricbeat/module/prometheus/collector/data.go index 1651bf3752e..356e7f4c239 100644 --- a/metricbeat/module/prometheus/collector/data.go +++ b/metricbeat/module/prometheus/collector/data.go @@ -23,22 +23,35 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/helper/prometheus" + "github.com/elastic/beats/v7/metricbeat/mb" dto "github.com/prometheus/client_model/go" ) // PromEvent stores a set of one or more metrics with the same labels type PromEvent struct { - data common.MapStr - labels common.MapStr + Data common.MapStr + Labels common.MapStr } // LabelsHash returns a repeatable string that is unique for the set of labels in this event func (p *PromEvent) LabelsHash() string { - return prometheus.LabelHash(p.labels) + return prometheus.LabelHash(p.Labels) } -func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { +// DefaultPromEventsGeneratorFactory returns the default prometheus events generator +func DefaultPromEventsGeneratorFactory(ms mb.BaseMetricSet) (PromEventsGenerator, error) { + return &promEventGenerator{}, nil +} + +type promEventGenerator struct{} + +func (p *promEventGenerator) Start() {} +func (p *promEventGenerator) Stop() {} + +// DefaultPromEventsGenerator stores all Prometheus metrics using +// only double field type in Elasticsearch. +func (p *promEventGenerator) GeneratePromEvents(mf *dto.MetricFamily) []PromEvent { var events []PromEvent name := *mf.Name @@ -58,10 +71,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if counter != nil { if !math.IsNaN(counter.GetValue()) && !math.IsInf(counter.GetValue(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name: counter.GetValue(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name: counter.GetValue(), + }, }, - labels: labels, + Labels: labels, }) } } @@ -70,10 +85,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if gauge != nil { if !math.IsNaN(gauge.GetValue()) && !math.IsInf(gauge.GetValue(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name: gauge.GetValue(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name: gauge.GetValue(), + }, }, - labels: labels, + Labels: labels, }) } } @@ -82,11 +99,13 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if summary != nil { if !math.IsNaN(summary.GetSampleSum()) && !math.IsInf(summary.GetSampleSum(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name + "_sum": summary.GetSampleSum(), - name + "_count": summary.GetSampleCount(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name + "_sum": summary.GetSampleSum(), + name + "_count": summary.GetSampleCount(), + }, }, - labels: labels, + Labels: labels, }) } @@ -98,10 +117,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { quantileLabels := labels.Clone() quantileLabels["quantile"] = strconv.FormatFloat(quantile.GetQuantile(), 'f', -1, 64) events = append(events, PromEvent{ - data: common.MapStr{ - name: quantile.GetValue(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name: quantile.GetValue(), + }, }, - labels: quantileLabels, + Labels: quantileLabels, }) } } @@ -110,11 +131,13 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if histogram != nil { if !math.IsNaN(histogram.GetSampleSum()) && !math.IsInf(histogram.GetSampleSum(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name + "_sum": histogram.GetSampleSum(), - name + "_count": histogram.GetSampleCount(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name + "_sum": histogram.GetSampleSum(), + name + "_count": histogram.GetSampleCount(), + }, }, - labels: labels, + Labels: labels, }) } @@ -127,10 +150,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { bucketLabels["le"] = strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) events = append(events, PromEvent{ - data: common.MapStr{ - name + "_bucket": bucket.GetCumulativeCount(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name + "_bucket": bucket.GetCumulativeCount(), + }, }, - labels: bucketLabels, + Labels: bucketLabels, }) } } @@ -139,10 +164,12 @@ func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { if untyped != nil { if !math.IsNaN(untyped.GetValue()) && !math.IsInf(untyped.GetValue(), 0) { events = append(events, PromEvent{ - data: common.MapStr{ - name: untyped.GetValue(), + Data: common.MapStr{ + "metrics": common.MapStr{ + name: untyped.GetValue(), + }, }, - labels: labels, + Labels: labels, }) } } diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 04f5458895e..a46368da408 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -49,6 +49,8 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle/performance" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle/tablespace" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/redisenterprise" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql/query" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index acb13989e5b..8febfe39cd6 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1004,6 +1004,28 @@ metricbeat.modules: # Password to use when connecting to PostgreSQL. Empty by default. #password: pass +#----------------------- Prometheus Typed Metrics Module ----------------------- +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true #------------------------------ Prometheus Module ------------------------------ # Metrics collected from a Prometheus endpoint - module: prometheus diff --git a/x-pack/metricbeat/module/openmetrics/collector/collector.go b/x-pack/metricbeat/module/openmetrics/collector/collector.go index f0f7c960f65..362260ded07 100644 --- a/x-pack/metricbeat/module/openmetrics/collector/collector.go +++ b/x-pack/metricbeat/module/openmetrics/collector/collector.go @@ -24,6 +24,6 @@ var ( func init() { mb.Registry.MustAddMetricSet("openmetrics", "collector", - collector.MetricSetBuilder("openmetrics"), + collector.MetricSetBuilder("openmetrics", collector.DefaultPromEventsGeneratorFactory), mb.WithHostParser(hostParser)) } diff --git a/x-pack/metricbeat/module/prometheus/_meta/config.yml b/x-pack/metricbeat/module/prometheus/_meta/config.yml new file mode 100644 index 00000000000..89e819e0c21 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/_meta/config.yml @@ -0,0 +1,21 @@ +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/_meta/fields.yml new file mode 100644 index 00000000000..797d46d4e28 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/_meta/fields.yml @@ -0,0 +1,31 @@ +- key: prometheus-xpack + title: "Prometheus typed metrics" + description: > + Stats scraped from a Prometheus endpoint. + release: ga + settings: ["ssl", "http"] + fields: + - name: prometheus.*.value + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus gauge metric + - name: prometheus.*.counter + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus counter metric + - name: prometheus.*.rate + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus rated counter metric + - name: prometheus.*.histogram + type: object + object_type: histogram + object_type_mapping_type: "*" + description: > + Prometheus histogram metric diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/data.json b/x-pack/metricbeat/module/prometheus/collector/_meta/data.json new file mode 100644 index 00000000000..f0fed23d976 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/data.json @@ -0,0 +1,31 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "172.27.0.2:9090", + "interval": "15s", + "job": "prometheus" + }, + "prometheus_target_interval_length_seconds_count": { + "counter": 1, + "rate": 0 + }, + "prometheus_target_interval_length_seconds_sum": { + "counter": 15.000401344, + "rate": 0 + } + }, + "service": { + "address": "172.27.0.2:9090", + "type": "prometheus" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/collector/_meta/fields.yml new file mode 100644 index 00000000000..a927f3fc9f8 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/fields.yml @@ -0,0 +1 @@ +- release: ga diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml new file mode 100644 index 00000000000..e7b2a5e67b4 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml @@ -0,0 +1,13 @@ +type: http +url: "/metrics" +suffix: plain +omit_documented_fields_check: + # these are not mapped by this module but the oss one + - prometheus.labels.* + # histogram values & counts are not mapped (it's part of the type data) + - '*.histogram.values' + - '*.histogram.counts' +remove_fields_from_comparison: ["prometheus.labels.instance"] +module: + use_types: true + rate_counters: true \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain new file mode 100644 index 00000000000..47c3b38aedb --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain @@ -0,0 +1,11 @@ +# HELP node_network_carrier carrier value of /sys/class/net/. +# TYPE node_network_carrier gauge +node_network_carrier{device="br-0cb306323b90"} 0 +node_network_carrier{device="br-10229e3512d9"} 0 +node_network_carrier{device="br-210476dc4ef8"} 0 +node_network_carrier{device="br-33d819d5f834"} 0 +node_network_carrier{device="br-38425a39f36b"} 0 +node_network_carrier{device="br-38feb0aad6ab"} 0 +node_network_carrier{device="br-3a285aa5e58c"} 0 +node_network_carrier{device="br-425cb4c454a6"} 0 +node_network_carrier{device="br-4e623477470e"} 0 diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json new file mode 100644 index 00000000000..6810afce3f6 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json @@ -0,0 +1,251 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-210476dc4ef8", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-0cb306323b90", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-10229e3512d9", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-33d819d5f834", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "up": { + "value": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-38425a39f36b", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-3a285aa5e58c", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-38feb0aad6ab", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-4e623477470e", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "device": "br-425cb4c454a6", + "instance": "127.0.0.1:41313", + "job": "prometheus" + }, + "node_network_carrier": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain new file mode 100644 index 00000000000..ea88edb5047 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain @@ -0,0 +1,38 @@ +# HELP kafka_consumer_records_lag_records The latest lag of the partition +# TYPE kafka_consumer_records_lag_records gauge +kafka_consumer_records_lag_records{client_id="consumer1",} NaN +kafka_consumer_records_lag_records{client_id="consumer2",} +Inf +kafka_consumer_records_lag_records{client_id="consumer3",} -Inf +kafka_consumer_records_lag_records{client_id="consumer4",} 5 +# HELP http_failures Total number of http request failures +# TYPE http_failures counter +http_failures{method="GET"} 2 +http_failures{method="POST"} NaN +http_failures{method="DELETE"} +Inf +# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0",} NaN +go_gc_duration_seconds{quantile="0.25",} +Inf +go_gc_duration_seconds{quantile="0.5",} -Inf +go_gc_duration_seconds{quantile="0.75"} 9.8154e-05 +go_gc_duration_seconds{quantile="1",} 0.011689149 +go_gc_duration_seconds_sum 3.451780079 +go_gc_duration_seconds_count 13118 +# HELP http_request_duration_seconds request duration histogram +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.1"} +Inf +http_request_duration_seconds_bucket{le="0.2"} -Inf +http_request_duration_seconds_bucket{le="0.5"} NaN +http_request_duration_seconds_bucket{le="1"} 1 +http_request_duration_seconds_bucket{le="2"} 2 +http_request_duration_seconds_bucket{le="3"} 3 +http_request_duration_seconds_bucket{le="5"} 3 +http_request_duration_seconds_bucket{le="+Inf"} 3 +http_request_duration_seconds_sum 6 +http_request_duration_seconds_count 3 +# HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name. +# TYPE net_conntrack_listener_conn_accepted_total untyped +net_conntrack_listener_conn_accepted_total{listener_name="http"} 1568652315554 +# HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name. +# TYPE net_conntrack_listener_conn_closed_total untyped +net_conntrack_listener_conn_closed_total{listener_name="http"} NaN diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json new file mode 100644 index 00000000000..03ec694141a --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/metrics-with-naninf.plain-expected.json @@ -0,0 +1,199 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.000098154 + }, + "labels": { + "instance": "127.0.0.1:42061", + "job": "prometheus", + "quantile": "0.75" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.011689149 + }, + "labels": { + "instance": "127.0.0.1:42061", + "job": "prometheus", + "quantile": "1" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:42061", + "job": "prometheus" + }, + "up": { + "value": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds_count": { + "counter": 13118, + "rate": 0 + }, + "go_gc_duration_seconds_sum": { + "counter": 3.451780079, + "rate": 0 + }, + "http_request_duration_seconds": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.5, + 1.5, + 2.5, + 4, + 7 + ] + } + }, + "labels": { + "instance": "127.0.0.1:42061", + "job": "prometheus" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "kafka_consumer_records_lag_records": { + "value": 5 + }, + "labels": { + "client_id": "consumer4", + "instance": "127.0.0.1:42061", + "job": "prometheus" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "http_failures": { + "counter": 2, + "rate": 0 + }, + "labels": { + "instance": "127.0.0.1:42061", + "job": "prometheus", + "method": "GET" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:42061", + "job": "prometheus", + "listener_name": "http" + }, + "net_conntrack_listener_conn_accepted_total": { + "value": 1568652315554 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain new file mode 100644 index 00000000000..136196b72c2 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain @@ -0,0 +1,135 @@ +# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 3.8386e-05 +go_gc_duration_seconds{quantile="0.25"} 4.2803e-05 +go_gc_duration_seconds{quantile="0.5"} 6.0618e-05 +go_gc_duration_seconds{quantile="0.75"} 0.004392391 +go_gc_duration_seconds{quantile="1"} 0.004392391 +go_gc_duration_seconds_sum 0.004534198 +go_gc_duration_seconds_count 4 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 35 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.11.3"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 1.0558112e+07 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 1.408776e+07 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.447018e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 15673 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction 0.0008429952574435172 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 2.379776e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 1.0558112e+07 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 5.4042624e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 1.2214272e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 61771 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 0 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 6.6256896e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.5534303161488917e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 77444 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 6912 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 16384 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 127984 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 131072 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 1.8390112e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 1.201294e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 851968 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 851968 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 7.2284408e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 14 +# HELP net_conntrack_dialer_conn_attempted_total Total number of connections attempted by the given dialer a given name. +# TYPE net_conntrack_dialer_conn_attempted_total counter +net_conntrack_dialer_conn_attempted_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_attempted_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_attempted_total{dialer_name="prometheus"} 1 +# HELP net_conntrack_dialer_conn_closed_total Total number of connections closed which originated from the dialer of a given name. +# TYPE net_conntrack_dialer_conn_closed_total counter +net_conntrack_dialer_conn_closed_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="prometheus"} 0 +# HELP net_conntrack_dialer_conn_established_total Total number of connections successfully established by the given dialer a given name. +# TYPE net_conntrack_dialer_conn_established_total counter +net_conntrack_dialer_conn_established_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="prometheus"} 1 +# HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name. +# TYPE net_conntrack_listener_conn_accepted_total counter +net_conntrack_listener_conn_accepted_total{listener_name="http"} 3 +# HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name. +# TYPE net_conntrack_listener_conn_closed_total counter +net_conntrack_listener_conn_closed_total{listener_name="http"} 0 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 0.14 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 13 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 3.5934208e+07 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.5534303054e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 1.50646784e+08 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes -1 +# HELP prometheus_api_remote_read_queries The current number of remote read queries being executed or waiting. +# TYPE prometheus_api_remote_read_queries gauge +prometheus_api_remote_read_queries 0 diff --git a/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json new file mode 100644 index 00000000000..5d6410ea094 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json @@ -0,0 +1,444 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "dialer_name": "prometheus", + "instance": "127.0.0.1:34181", + "job": "prometheus" + }, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 1, + "rate": 0 + }, + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { + "counter": 1, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "listener_name": "http" + }, + "net_conntrack_listener_conn_accepted_total": { + "counter": 3, + "rate": 0 + }, + "net_conntrack_listener_conn_closed_total": { + "counter": 0, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds_count": { + "counter": 4, + "rate": 0 + }, + "go_gc_duration_seconds_sum": { + "counter": 0.004534198, + "rate": 0 + }, + "go_goroutines": { + "value": 35 + }, + "go_memstats_alloc_bytes": { + "value": 10558112 + }, + "go_memstats_alloc_bytes_total": { + "counter": 14087760, + "rate": 0 + }, + "go_memstats_buck_hash_sys_bytes": { + "value": 1447018 + }, + "go_memstats_frees_total": { + "counter": 15673, + "rate": 0 + }, + "go_memstats_gc_cpu_fraction": { + "value": 0.0008429952574435172 + }, + "go_memstats_gc_sys_bytes": { + "value": 2379776 + }, + "go_memstats_heap_alloc_bytes": { + "value": 10558112 + }, + "go_memstats_heap_idle_bytes": { + "value": 54042624 + }, + "go_memstats_heap_inuse_bytes": { + "value": 12214272 + }, + "go_memstats_heap_objects": { + "value": 61771 + }, + "go_memstats_heap_released_bytes": { + "value": 0 + }, + "go_memstats_heap_sys_bytes": { + "value": 66256896 + }, + "go_memstats_last_gc_time_seconds": { + "value": 1553430316.1488917 + }, + "go_memstats_lookups_total": { + "counter": 0, + "rate": 0 + }, + "go_memstats_mallocs_total": { + "counter": 77444, + "rate": 0 + }, + "go_memstats_mcache_inuse_bytes": { + "value": 6912 + }, + "go_memstats_mcache_sys_bytes": { + "value": 16384 + }, + "go_memstats_mspan_inuse_bytes": { + "value": 127984 + }, + "go_memstats_mspan_sys_bytes": { + "value": 131072 + }, + "go_memstats_next_gc_bytes": { + "value": 18390112 + }, + "go_memstats_other_sys_bytes": { + "value": 1201294 + }, + "go_memstats_stack_inuse_bytes": { + "value": 851968 + }, + "go_memstats_stack_sys_bytes": { + "value": 851968 + }, + "go_memstats_sys_bytes": { + "value": 72284408 + }, + "go_threads": { + "value": 14 + }, + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus" + }, + "process_cpu_seconds_total": { + "counter": 0.14, + "rate": 0 + }, + "process_max_fds": { + "value": 1048576 + }, + "process_open_fds": { + "value": 13 + }, + "process_resident_memory_bytes": { + "value": 35934208 + }, + "process_start_time_seconds": { + "value": 1553430305.4 + }, + "process_virtual_memory_bytes": { + "value": 150646784 + }, + "process_virtual_memory_max_bytes": { + "value": -1 + }, + "prometheus_api_remote_read_queries": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "dialer_name": "default", + "instance": "127.0.0.1:34181", + "job": "prometheus" + }, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { + "counter": 0, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_info": { + "value": 1 + }, + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "version": "go1.11.3" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "dialer_name": "alertmanager", + "instance": "127.0.0.1:34181", + "job": "prometheus" + }, + "net_conntrack_dialer_conn_attempted_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_closed_total": { + "counter": 0, + "rate": 0 + }, + "net_conntrack_dialer_conn_established_total": { + "counter": 0, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.004392391 + }, + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "quantile": "1" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.000038386 + }, + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "quantile": "0" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus" + }, + "up": { + "value": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.000060618 + }, + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "quantile": "0.5" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.000042803 + }, + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "quantile": "0.25" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "go_gc_duration_seconds": { + "value": 0.004392391 + }, + "labels": { + "instance": "127.0.0.1:34181", + "job": "prometheus", + "quantile": "0.75" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/collector/collector.go b/x-pack/metricbeat/module/prometheus/collector/collector.go new file mode 100644 index 00000000000..92f3552eebd --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/collector.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/prometheus/collector" +) + +func init() { + mb.Registry.MustAddMetricSet("prometheus", "collector", + collector.MetricSetBuilder("prometheus", promEventsGeneratorFactory), + mb.WithHostParser(collector.HostParser), + mb.DefaultMetricSet(), + + // must replace ensures that we are replacing the oss implementation with this one + // so we can make use of ES histograms (basic only) when use_types is enabled + mb.MustReplace(), + ) +} diff --git a/x-pack/metricbeat/module/prometheus/collector/collector_integration_test.go b/x-pack/metricbeat/module/prometheus/collector/collector_integration_test.go new file mode 100644 index 00000000000..179a97c75ce --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/collector_integration_test.go @@ -0,0 +1,37 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package collector + +import ( + "testing" + "time" + + "github.com/elastic/beats/v7/libbeat/tests/compose" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + service := compose.EnsureUp(t, "prometheus") + + config := map[string]interface{}{ + "module": "prometheus", + "metricsets": []string{"collector"}, + "hosts": []string{service.Host()}, + "use_types": true, + "rate_counters": true, + } + ms := mbtest.NewReportingMetricSetV2Error(t, config) + var err error + for retries := 0; retries < 3; retries++ { + err = mbtest.WriteEventsReporterV2Error(ms, t, "") + if err == nil { + return + } + time.Sleep(10 * time.Second) + } + t.Fatal("write", err) +} diff --git a/x-pack/metricbeat/module/prometheus/collector/collector_test.go b/x-pack/metricbeat/module/prometheus/collector/collector_test.go new file mode 100644 index 00000000000..0a3950bdc75 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/collector_test.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package collector + +import ( + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus" +) + +func TestData(t *testing.T) { + mbtest.TestDataFiles(t, "prometheus", "collector") +} diff --git a/x-pack/metricbeat/module/prometheus/collector/config.go b/x-pack/metricbeat/module/prometheus/collector/config.go new file mode 100644 index 00000000000..8813d5e3e59 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/config.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import "errors" + +type config struct { + UseTypes bool `config:"use_types"` + RateCounters bool `config:"rate_counters"` +} + +func (c *config) Validate() error { + if c.RateCounters && !c.UseTypes { + return errors.New("'rate_counters' can only be enabled when `use_types` is also enabled") + } + + return nil +} diff --git a/x-pack/metricbeat/module/prometheus/collector/counter.go b/x-pack/metricbeat/module/prometheus/collector/counter.go new file mode 100644 index 00000000000..f39e6de1763 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/counter.go @@ -0,0 +1,92 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "time" + + "github.com/elastic/beats/v7/libbeat/common" +) + +// CounterCache keeps a cache of the last value of all given counters +// and allows to calculate their rate since the last call. +// All methods are thread-unsafe and must not be called concurrently +type CounterCache interface { + // Start the cache cleanup worker. It mus be called once before start using + // the cache + Start() + + // Stop the cache cleanup worker. It mus be called when the cache is disposed + Stop() + + // RateUint64 returns, for a given counter name, the difference between the given value + // and the value that was given in a previous call. It will return 0 on the first call + RateUint64(counterName string, value uint64) uint64 + + // RateFloat64 returns, for a given counter name, the difference between the given value + // and the value that was given in a previous call. It will return 0.0 on the first call + RateFloat64(counterName string, value float64) float64 +} + +type counterCache struct { + ints *common.Cache + floats *common.Cache + timeout time.Duration +} + +// NewCounterCache initializes and returns a CounterCache. The timeout parameter will be +// used to automatically expire counters that hasn't been updated in a whole timeout period +func NewCounterCache(timeout time.Duration) CounterCache { + return &counterCache{ + ints: common.NewCache(timeout, 0), + floats: common.NewCache(timeout, 0), + timeout: timeout, + } +} + +// RateUint64 returns, for a given counter name, the difference between the given value +// and the value that was given in a previous call. It will return 0 on the first call +func (c *counterCache) RateUint64(counterName string, value uint64) uint64 { + prev := c.ints.PutWithTimeout(counterName, value, c.timeout) + if prev != nil { + if prev.(uint64) > value { + // counter reset + return 0 + } + return value - prev.(uint64) + } + + // first put for this value, return rate of 0 + return 0 +} + +// RateFloat64 returns, for a given counter name, the difference between the given value +// and the value that was given in a previous call. It will return 0.0 on the first call +func (c *counterCache) RateFloat64(counterName string, value float64) float64 { + prev := c.floats.PutWithTimeout(counterName, value, c.timeout) + if prev != nil { + if prev.(float64) > value { + // counter reset + return 0 + } + return value - prev.(float64) + } + + // first put for this value, return rate of 0 + return 0 +} + +// Start the cache cleanup worker. It mus be called once before start using +// the cache +func (c *counterCache) Start() { + c.ints.StartJanitor(c.timeout) + c.floats.StartJanitor(c.timeout) +} + +// Stop the cache cleanup worker. It mus be called when the cache is disposed +func (c *counterCache) Stop() { + c.ints.StopJanitor() + c.floats.StopJanitor() +} diff --git a/x-pack/metricbeat/module/prometheus/collector/counter_test.go b/x-pack/metricbeat/module/prometheus/collector/counter_test.go new file mode 100644 index 00000000000..3aed4a8d017 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/counter_test.go @@ -0,0 +1,65 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "testing" + "time" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func Test_CounterCache(t *testing.T) { + type fields struct { + ints *common.Cache + floats *common.Cache + timeout time.Duration + } + + tests := []struct { + name string + counterCache CounterCache + counterName string + valuesUint64 []uint64 + expectedUin64 []uint64 + valuesFloat64 []float64 + expectedFloat64 []float64 + }{ + { + name: "rates are calculated", + counterCache: NewCounterCache(1 * time.Second), + counterName: "test_counter", + valuesUint64: []uint64{10, 14, 17, 17, 28}, + expectedUin64: []uint64{0, 4, 3, 0, 11}, + valuesFloat64: []float64{1.0, 101.0, 102.0, 102.0, 1034.0}, + expectedFloat64: []float64{0.0, 100.0, 1.0, 0.0, 932.0}, + }, + { + name: "counter reset", + counterCache: NewCounterCache(1 * time.Second), + counterName: "test_counter", + valuesUint64: []uint64{10, 14, 17, 1, 3}, + expectedUin64: []uint64{0, 4, 3, 0, 2}, + valuesFloat64: []float64{1.0, 101.0, 2.0, 13.0}, + expectedFloat64: []float64{0.0, 100.0, 0.0, 11.0}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i, val := range tt.valuesUint64 { + want := tt.expectedUin64[i] + if got := tt.counterCache.RateUint64(tt.counterName, val); got != want { + t.Errorf("counterCache.RateUint64() = %v, want %v", got, want) + } + } + for i, val := range tt.valuesFloat64 { + want := tt.expectedFloat64[i] + if got := tt.counterCache.RateFloat64(tt.counterName, val); got != want { + t.Errorf("counterCache.RateFloat64() = %v, want %v", got, want) + } + } + }) + } +} diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go new file mode 100644 index 00000000000..8e205b66a2d --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -0,0 +1,191 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "math" + "strconv" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/prometheus/collector" + + dto "github.com/prometheus/client_model/go" +) + +func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGenerator, error) { + config := config{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + if config.UseTypes { + // use a counter cache with a timeout of 5x the period, as a safe value + // to make sure that all counters are available between fetches + counters := NewCounterCache(base.Module().Config().Period * 5) + + g := typedGenerator{ + counterCache: counters, + rateCounters: config.RateCounters, + } + + return &g, nil + } + + return collector.DefaultPromEventsGeneratorFactory(base) +} + +type typedGenerator struct { + counterCache CounterCache + rateCounters bool +} + +func (g *typedGenerator) Start() { + cfgwarn.Beta("Prometheus 'use_types' setting is beta") + + if g.rateCounters { + cfgwarn.Experimental("Prometheus 'rate_counters' setting is experimental") + } + + g.counterCache.Start() +} + +func (g *typedGenerator) Stop() { + g.counterCache.Stop() +} + +// GeneratePromEvents stores all Prometheus metrics using +// specific Elasticsearch data types. +func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.PromEvent { + var events []collector.PromEvent + + name := *mf.Name + metrics := mf.Metric + for _, metric := range metrics { + labels := common.MapStr{} + + if len(metric.Label) != 0 { + for _, label := range metric.Label { + if label.GetName() != "" && label.GetValue() != "" { + labels[label.GetName()] = label.GetValue() + } + } + } + + counter := metric.GetCounter() + if counter != nil { + if !math.IsNaN(counter.GetValue()) && !math.IsInf(counter.GetValue(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: g.rateCounterFloat64(name, labels, counter.GetValue()), + }, + Labels: labels, + }) + } + } + + gauge := metric.GetGauge() + if gauge != nil { + if !math.IsNaN(gauge.GetValue()) && !math.IsInf(gauge.GetValue(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: common.MapStr{ + "value": gauge.GetValue(), + }, + }, + Labels: labels, + }) + } + } + + summary := metric.GetSummary() + if summary != nil { + if !math.IsNaN(summary.GetSampleSum()) && !math.IsInf(summary.GetSampleSum(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name + "_sum": g.rateCounterFloat64(name, labels, summary.GetSampleSum()), + name + "_count": g.rateCounterUint64(name, labels, summary.GetSampleCount()), + }, + Labels: labels, + }) + } + + for _, quantile := range summary.GetQuantile() { + if math.IsNaN(quantile.GetValue()) || math.IsInf(quantile.GetValue(), 0) { + continue + } + + quantileLabels := labels.Clone() + quantileLabels["quantile"] = strconv.FormatFloat(quantile.GetQuantile(), 'f', -1, 64) + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: common.MapStr{ + "value": quantile.GetValue(), + }, + }, + Labels: quantileLabels, + }) + } + } + + histogram := metric.GetHistogram() + if histogram != nil { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: common.MapStr{ + "histogram": promHistogramToES(g.counterCache, name, labels, histogram), + }, + }, + Labels: labels, + }) + /* + TODO convert histogram to ES type + Send sum & count? not sure it's worth it + */ + } + + untyped := metric.GetUntyped() + if untyped != nil { + if !math.IsNaN(untyped.GetValue()) && !math.IsInf(untyped.GetValue(), 0) { + events = append(events, collector.PromEvent{ + Data: common.MapStr{ + name: common.MapStr{ + "value": untyped.GetValue(), + }, + }, + Labels: labels, + }) + } + } + } + return events +} + +// rateCounterUint64 fills a counter value and optionally adds the rate if rate_counters is enabled +func (g *typedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { + d := common.MapStr{ + "counter": value, + } + + if g.rateCounters { + d["rate"] = g.counterCache.RateUint64(name+labels.String(), value) + } + + return d +} + +// rateCounterFloat64 fills a counter value and optionally adds the rate if rate_counters is enabled +func (g *typedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { + d := common.MapStr{ + "counter": value, + } + + if g.rateCounters { + d["rate"] = g.counterCache.RateFloat64(name+labels.String(), value) + } + + return d +} diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram.go b/x-pack/metricbeat/module/prometheus/collector/histogram.go new file mode 100644 index 00000000000..8a62cbf8b97 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/collector/histogram.go @@ -0,0 +1,65 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package collector + +import ( + "fmt" + "math" + + "github.com/elastic/beats/v7/libbeat/common" + + dto "github.com/prometheus/client_model/go" +) + +// promHistogramToES takes a Prometheus histogram and converts it to an ES histogram: +// +// ES histograms look like this: +// +// "histogram_field" : { +// "values" : [0.1, 0.2, 0.3, 0.4, 0.5], +// "counts" : [3, 7, 23, 12, 6] +// } +// +// This code takes a Prometheus histogram and tries to accomodate it into an ES histogram by: +// - calculating centroids for each bucket (values) +// - undoing counters accumulation for each bucket (counts) +// +// https://www.elastic.co/guide/en/elasticsearch/reference/master/histogram.html +func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histogram *dto.Histogram) common.MapStr { + var values []float64 + var counts []uint64 + + // calculate centroids and rated counts + var lastUpper, prevUpper float64 + var sumCount uint64 + for _, bucket := range histogram.GetBucket() { + // Ignore non-numbers + if bucket.GetCumulativeCount() == uint64(math.NaN()) || bucket.GetCumulativeCount() == uint64(math.Inf(0)) { + continue + } + + if bucket.GetUpperBound() == math.Inf(0) { + // Report +Inf bucket as a point, interpolating its value + values = append(values, lastUpper+(lastUpper-prevUpper)) + } else { + // calculate bucket centroid + values = append(values, lastUpper+(bucket.GetUpperBound()-lastUpper)/2.0) + prevUpper = lastUpper + lastUpper = bucket.GetUpperBound() + } + + // take count for this period (rate) + deacumulate + count := cc.RateUint64(name+labels.String()+fmt.Sprintf("%f", bucket.GetUpperBound()), bucket.GetCumulativeCount()) - sumCount + counts = append(counts, count) + sumCount += count + } + + res := common.MapStr{ + "values": values, + "counts": counts, + } + + return res +} diff --git a/x-pack/metricbeat/module/prometheus/doc.go b/x-pack/metricbeat/module/prometheus/doc.go new file mode 100644 index 00000000000..d2a78cdb9b5 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/doc.go @@ -0,0 +1,6 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Package prometheus is a Metricbeat module that contains MetricSets. +package prometheus diff --git a/x-pack/metricbeat/module/prometheus/docker-compose.yml b/x-pack/metricbeat/module/prometheus/docker-compose.yml new file mode 100644 index 00000000000..e24deaf2ead --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/docker-compose.yml @@ -0,0 +1,7 @@ +version: '2.3' + +services: + prometheus: + extends: + file: ../../../../metricbeat/module/prometheus/docker-compose.yml + service: prometheus \ No newline at end of file diff --git a/x-pack/metricbeat/module/prometheus/fields.go b/x-pack/metricbeat/module/prometheus/fields.go new file mode 100644 index 00000000000..63330de9110 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package prometheus + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "prometheus", asset.ModuleFieldsPri, AssetPrometheus); err != nil { + panic(err) + } +} + +// AssetPrometheus returns asset data. +// This is the base64 encoded gzipped contents of module/prometheus. +func AssetPrometheus() string { + return "eJzEkktqAzEMhvdzih8vQ5MDeNEzFLosJSi24rjxC0tTmtuXZIYw9EGzKMQ7+/+QPgmvceSTRes1sx54lPVHI3ccAI2a2MI8XSPoqbFHZu3RiRkAz+J6bBprsXgcAOBZSQXiOp3Zfa8ZhEUNLr7VWHQzAJ0Tk7BFoAEQVo0liMWLEUnmAeag2szrAOwjJy/20mGNQpmXzpvV5p3SyJcYF02Luntjp/PTdNlOia/jLvH3ZJuptVjCjJmVmZkfxjyfxVSBxsDzZn6XdHUsyv1+mrPAn6Kd9I7LPHf3N7seomgNnfKNwl/5/3G+Vl36Ts7LX/4ZAAD//xMCFRw=" +} diff --git a/x-pack/metricbeat/modules.d/prometheus.yml.disabled b/x-pack/metricbeat/modules.d/prometheus.yml.disabled new file mode 100644 index 00000000000..344142c3ca5 --- /dev/null +++ b/x-pack/metricbeat/modules.d/prometheus.yml.disabled @@ -0,0 +1,24 @@ +# Module: prometheus +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-prometheus.html + +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + #metrics_filters: + # include: [] + # exclude: [] + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true \ No newline at end of file