From 9954a98030e3c5001c8e02aecd4684427ae8a8c9 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sat, 4 May 2024 22:53:39 +0300 Subject: [PATCH 01/12] remove collector labels fingerprint --- .../elasticsearch/ingest_pipeline/default.yml | 14 -------------- .../data_stream/collector/fields/fields.yml | 8 ++------ 2 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 packages/prometheus/data_stream/collector/elasticsearch/ingest_pipeline/default.yml diff --git a/packages/prometheus/data_stream/collector/elasticsearch/ingest_pipeline/default.yml b/packages/prometheus/data_stream/collector/elasticsearch/ingest_pipeline/default.yml deleted file mode 100644 index 3fb1fcb745d..00000000000 --- a/packages/prometheus/data_stream/collector/elasticsearch/ingest_pipeline/default.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -description: Pipeline for processing Prometheus collector metrics. -processors: - - fingerprint: - fields: ["prometheus.labels"] - target_field: "prometheus.labels_fingerprint" - ignore_missing: true -on_failure: - - set: - field: event.kind - value: pipeline_error - - append: - field: error.message - value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/prometheus/data_stream/collector/fields/fields.yml b/packages/prometheus/data_stream/collector/fields/fields.yml index ed592103e62..4311cfb1f91 100644 --- a/packages/prometheus/data_stream/collector/fields/fields.yml +++ b/packages/prometheus/data_stream/collector/fields/fields.yml @@ -2,14 +2,10 @@ type: group fields: - name: labels.* - type: object - object_type: keyword - description: | - Prometheus metric labels - - name: labels_fingerprint type: keyword dimension: true - description: Autogenerated ID representing the fingerprint of labels object + description: | + Prometheus metric labels - name: prometheus.metrics.* type: object object_type: double From 55c01ba71cb1a7ebb49e8feebd73bd3338f6a821 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sat, 4 May 2024 22:53:49 +0300 Subject: [PATCH 02/12] remove query labels fingerprint --- .../query/elasticsearch/ingest_pipeline/default.yml | 4 ---- packages/prometheus/data_stream/query/fields/fields.yml | 8 ++------ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/prometheus/data_stream/query/elasticsearch/ingest_pipeline/default.yml b/packages/prometheus/data_stream/query/elasticsearch/ingest_pipeline/default.yml index 4be76fe0ec4..8da5588e974 100644 --- a/packages/prometheus/data_stream/query/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prometheus/data_stream/query/elasticsearch/ingest_pipeline/default.yml @@ -8,10 +8,6 @@ processors: field: "prometheus.labels.query_name" value: "{{_ingest._key}}" description: Add query_name as a label to cover cases, when query returns a single value with no labels - - fingerprint: - fields: ["prometheus.labels"] - target_field: "prometheus.labels_fingerprint" - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/prometheus/data_stream/query/fields/fields.yml b/packages/prometheus/data_stream/query/fields/fields.yml index cb385f2bcb1..4c55309af12 100644 --- a/packages/prometheus/data_stream/query/fields/fields.yml +++ b/packages/prometheus/data_stream/query/fields/fields.yml @@ -2,14 +2,10 @@ type: group fields: - name: labels.* - type: object - object_type: keyword - description: | - Prometheus metric labels - - name: labels_fingerprint type: keyword dimension: true - description: Autogenerated ID representing the fingerprint of labels object and includes query name + description: | + Prometheus metric labels - name: prometheus.query.* type: object object_type: double From 88610b4d7815f2a5291259e7d995ca2e9e227108 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sat, 4 May 2024 22:54:00 +0300 Subject: [PATCH 03/12] remove remote_write labels fingerprint --- .../elasticsearch/ingest_pipeline/default.yml | 4 ---- .../prometheus/data_stream/remote_write/fields/fields.yml | 8 ++------ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml b/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml index 1cf77307100..711e6ab0362 100644 --- a/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml @@ -22,10 +22,6 @@ processors: field: "prometheus.labels.metrics_names" value: ["{{_ingest._key}}"] description: Add all keys of the 'prometheus' object to the earlier created field, it includes all metric names and key "labels" - - fingerprint: - fields: ["prometheus.labels"] - target_field: "prometheus.labels_fingerprint" - ignore_missing: true - remove: field: "prometheus.labels.metrics_names" description: Remove field as it contains a redundant information and can impact the documents size diff --git a/packages/prometheus/data_stream/remote_write/fields/fields.yml b/packages/prometheus/data_stream/remote_write/fields/fields.yml index b41673d627b..07d911f5611 100644 --- a/packages/prometheus/data_stream/remote_write/fields/fields.yml +++ b/packages/prometheus/data_stream/remote_write/fields/fields.yml @@ -2,14 +2,10 @@ type: group fields: - name: labels.* - type: object - object_type: keyword - description: | - Prometheus metric labels - - name: labels_fingerprint type: keyword dimension: true - description: Autogenerated ID representing the fingerprint of all labels and the list of metrics names + description: | + Prometheus metric labels - name: metrics.* type: object object_type: double From 2614fdd0d6b3a2e4f60e72d949604238651f6500 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sat, 4 May 2024 22:54:07 +0300 Subject: [PATCH 04/12] update docs --- packages/prometheus/docs/README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/prometheus/docs/README.md b/packages/prometheus/docs/README.md index c0483e0c695..85762db58ce 100644 --- a/packages/prometheus/docs/README.md +++ b/packages/prometheus/docs/README.md @@ -223,8 +223,7 @@ The fields reported are: | prometheus.\*.histogram | Prometheus histogram metric | object | | | prometheus.\*.rate | Prometheus rated counter metric | object | gauge | | prometheus.\*.value | Prometheus gauge metric | object | gauge | -| prometheus.labels.\* | Prometheus metric labels | object | | -| prometheus.labels_fingerprint | Autogenerated ID representing the fingerprint of labels object | keyword | | +| prometheus.labels.\* | Prometheus metric labels | keyword | | | prometheus.metrics.\* | Prometheus metric | object | gauge | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | @@ -437,8 +436,7 @@ The fields reported are: | prometheus.\*.histogram | Prometheus histogram metric | object | | | prometheus.\*.rate | Prometheus rated counter metric | object | gauge | | prometheus.\*.value | Prometheus gauge metric | object | gauge | -| prometheus.labels.\* | Prometheus metric labels | object | | -| prometheus.labels_fingerprint | Autogenerated ID representing the fingerprint of all labels and the list of metrics names | keyword | | +| prometheus.labels.\* | Prometheus metric labels | keyword | | | prometheus.metrics.\* | Prometheus metric | object | gauge | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | @@ -660,8 +658,7 @@ The fields reported are: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | host.os.version | Operating system version as a raw string. | keyword | | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| prometheus.labels.\* | Prometheus metric labels | object | | -| prometheus.labels_fingerprint | Autogenerated ID representing the fingerprint of labels object and includes query name | keyword | | +| prometheus.labels.\* | Prometheus metric labels | keyword | | | prometheus.query.\* | Prometheus value resulted from PromQL | object | gauge | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | From 149bf86aeaeb1e700596d0a2988b46aae55b38a7 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sat, 4 May 2024 22:55:28 +0300 Subject: [PATCH 05/12] bump package version --- packages/prometheus/changelog.yml | 5 +++++ packages/prometheus/manifest.yml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/prometheus/changelog.yml b/packages/prometheus/changelog.yml index b674dae62bb..816ced96587 100644 --- a/packages/prometheus/changelog.yml +++ b/packages/prometheus/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.15.2" + changes: + - description: Eliminate labels fingerprint for tsdb purposes and use keyword type instead. + type: enhancement + link: https://github.com/elastic/integrations/pull/999 - version: "1.15.1" changes: - description: Add a "period" parameter to the counter cache. diff --git a/packages/prometheus/manifest.yml b/packages/prometheus/manifest.yml index aa00925a2b2..5588130abf3 100644 --- a/packages/prometheus/manifest.yml +++ b/packages/prometheus/manifest.yml @@ -1,7 +1,7 @@ format_version: 2.10.0 name: prometheus title: Prometheus -version: 1.15.1 +version: 1.15.2 description: Collect metrics from Prometheus servers with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - monitoring - containers conditions: - kibana.version: "^8.12.1" + kibana.version: "^8.13.0" screenshots: - src: /img/prometheus-server-overview.png title: Metricbeat Prometheus Overview From 0a39469cd9f3257316b3c1775bb21e0000524093 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sat, 4 May 2024 23:03:02 +0300 Subject: [PATCH 06/12] fix PR link --- packages/prometheus/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/prometheus/changelog.yml b/packages/prometheus/changelog.yml index 816ced96587..fd1a88d430d 100644 --- a/packages/prometheus/changelog.yml +++ b/packages/prometheus/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Eliminate labels fingerprint for tsdb purposes and use keyword type instead. type: enhancement - link: https://github.com/elastic/integrations/pull/999 + link: https://github.com/elastic/integrations/pull/9785 - version: "1.15.1" changes: - description: Add a "period" parameter to the counter cache. From 25446aee22dcf1bb0138ed5e2fb3ba117414cd8e Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Thu, 16 May 2024 20:13:34 +0300 Subject: [PATCH 07/12] bump kibana version --- packages/prometheus/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/prometheus/manifest.yml b/packages/prometheus/manifest.yml index fccf03a50a4..e397591c4a3 100644 --- a/packages/prometheus/manifest.yml +++ b/packages/prometheus/manifest.yml @@ -9,7 +9,7 @@ categories: - monitoring - containers conditions: - kibana.version: "^8.13.0" + kibana.version: "^8.14.0" screenshots: - src: /img/prometheus-server-overview.png title: Metricbeat Prometheus Overview From f91a31679d902df53ccbab6fdd8066b6b1c52013 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Thu, 20 Jun 2024 15:20:21 +0300 Subject: [PATCH 08/12] fingerprint metrics_names for remote_write --- .../remote_write/elasticsearch/ingest_pipeline/default.yml | 4 ++++ .../prometheus/data_stream/remote_write/fields/fields.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml b/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml index 711e6ab0362..0dcf3b4b1c6 100644 --- a/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml @@ -22,6 +22,10 @@ processors: field: "prometheus.labels.metrics_names" value: ["{{_ingest._key}}"] description: Add all keys of the 'prometheus' object to the earlier created field, it includes all metric names and key "labels" + - fingerprint: + fields: ["prometheus.labels.metrics_names"] + target_field: "prometheus.labels_fingerprint" + ignore_missing: true - remove: field: "prometheus.labels.metrics_names" description: Remove field as it contains a redundant information and can impact the documents size diff --git a/packages/prometheus/data_stream/remote_write/fields/fields.yml b/packages/prometheus/data_stream/remote_write/fields/fields.yml index 07d911f5611..3a6281e0b79 100644 --- a/packages/prometheus/data_stream/remote_write/fields/fields.yml +++ b/packages/prometheus/data_stream/remote_write/fields/fields.yml @@ -6,6 +6,10 @@ dimension: true description: | Prometheus metric labels + - name: labels_fingerprint + type: keyword + dimension: true + description: Autogenerated ID representing the fingerprint of the list of metrics names - name: metrics.* type: object object_type: double From 83388dfff41ac72c266833eaa31717c88b065aed Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Thu, 20 Jun 2024 15:21:28 +0300 Subject: [PATCH 09/12] bump minor version --- packages/prometheus/changelog.yml | 4 ++-- packages/prometheus/manifest.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/prometheus/changelog.yml b/packages/prometheus/changelog.yml index 98ca1e067e0..1dda58149c7 100644 --- a/packages/prometheus/changelog.yml +++ b/packages/prometheus/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top -- version: "1.15.3" +- version: "1.16.0" changes: - - description: Eliminate labels fingerprint for tsdb purposes and use keyword type instead. + - description: Remove labels fingerprint for TSDB purposes. type: enhancement link: https://github.com/elastic/integrations/pull/9785 - version: "1.15.2" diff --git a/packages/prometheus/manifest.yml b/packages/prometheus/manifest.yml index e397591c4a3..b283a08f289 100644 --- a/packages/prometheus/manifest.yml +++ b/packages/prometheus/manifest.yml @@ -1,7 +1,7 @@ format_version: 2.10.0 name: prometheus title: Prometheus -version: 1.15.3 +version: 1.16.0 description: Collect metrics from Prometheus servers with Elastic Agent. type: integration categories: From 9588aa1ea4c8641b1729a6162d442d4314e3132f Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Thu, 20 Jun 2024 15:40:37 +0300 Subject: [PATCH 10/12] build readme --- packages/prometheus/docs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/prometheus/docs/README.md b/packages/prometheus/docs/README.md index 01488e73cf6..ced768ea43e 100644 --- a/packages/prometheus/docs/README.md +++ b/packages/prometheus/docs/README.md @@ -437,6 +437,7 @@ The fields reported are: | prometheus.\*.rate | Prometheus rated counter metric | object | gauge | | prometheus.\*.value | Prometheus gauge metric | object | gauge | | prometheus.labels.\* | Prometheus metric labels | keyword | | +| prometheus.labels_fingerprint | Autogenerated ID representing the fingerprint of the list of metrics names | keyword | | | prometheus.metrics.\* | Prometheus metric | object | gauge | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | From c6ab41b9844553fa3f4b331e956f9cdc8ff50317 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Thu, 20 Jun 2024 17:13:25 +0300 Subject: [PATCH 11/12] add metrics_names_fingerprint field --- .../remote_write/elasticsearch/ingest_pipeline/default.yml | 2 +- packages/prometheus/data_stream/remote_write/fields/fields.yml | 2 +- packages/prometheus/docs/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml b/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml index 0dcf3b4b1c6..5fc1f8b4738 100644 --- a/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prometheus/data_stream/remote_write/elasticsearch/ingest_pipeline/default.yml @@ -24,7 +24,7 @@ processors: description: Add all keys of the 'prometheus' object to the earlier created field, it includes all metric names and key "labels" - fingerprint: fields: ["prometheus.labels.metrics_names"] - target_field: "prometheus.labels_fingerprint" + target_field: "prometheus.metrics_names_fingerprint" ignore_missing: true - remove: field: "prometheus.labels.metrics_names" diff --git a/packages/prometheus/data_stream/remote_write/fields/fields.yml b/packages/prometheus/data_stream/remote_write/fields/fields.yml index 3a6281e0b79..04bac4803d4 100644 --- a/packages/prometheus/data_stream/remote_write/fields/fields.yml +++ b/packages/prometheus/data_stream/remote_write/fields/fields.yml @@ -6,7 +6,7 @@ dimension: true description: | Prometheus metric labels - - name: labels_fingerprint + - name: metrics_names_fingerprint type: keyword dimension: true description: Autogenerated ID representing the fingerprint of the list of metrics names diff --git a/packages/prometheus/docs/README.md b/packages/prometheus/docs/README.md index ced768ea43e..d02d7568954 100644 --- a/packages/prometheus/docs/README.md +++ b/packages/prometheus/docs/README.md @@ -437,8 +437,8 @@ The fields reported are: | prometheus.\*.rate | Prometheus rated counter metric | object | gauge | | prometheus.\*.value | Prometheus gauge metric | object | gauge | | prometheus.labels.\* | Prometheus metric labels | keyword | | -| prometheus.labels_fingerprint | Autogenerated ID representing the fingerprint of the list of metrics names | keyword | | | prometheus.metrics.\* | Prometheus metric | object | gauge | +| prometheus.metrics_names_fingerprint | Autogenerated ID representing the fingerprint of the list of metrics names | keyword | | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | From a8d7500961591194fdfa2cc9ce9e025c637e2822 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 26 Jun 2024 11:29:32 +0300 Subject: [PATCH 12/12] update changelog description --- packages/prometheus/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/prometheus/changelog.yml b/packages/prometheus/changelog.yml index 8d3f5c7bb54..602372419f9 100644 --- a/packages/prometheus/changelog.yml +++ b/packages/prometheus/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "1.16.0" changes: - - description: Remove labels fingerprint for TSDB purposes. + - description: Remove labels fingerprint, leading to better TSDB performance and storage reduction. type: enhancement link: https://github.com/elastic/integrations/pull/9785 - version: "1.15.3"