From bf752aaab93cbdbfa51174288b192dedab138a50 Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Wed, 19 Oct 2022 01:43:05 +0000 Subject: [PATCH 1/2] Copy packages from snapshot to snapshot --- .../elasticsearch/1.1.0-preview1/LICENSE.txt | 93 + .../1.1.0-preview1/changelog.yml | 34 + .../audit/agent/stream/log.yml.hbs | 46 + .../elasticsearch/ingest_pipeline/default.yml | 69 + .../ingest_pipeline/pipeline-json.yml | 219 +++ .../data_stream/audit/fields/base-fields.yml | 12 + .../data_stream/audit/fields/ecs.yml | 24 + .../data_stream/audit/fields/fields.yml | 46 + .../audit/fields/package-fields.yml | 27 + .../data_stream/audit/manifest.yml | 17 + .../ccr/agent/stream/stream.yml.hbs | 13 + .../data_stream/ccr/fields/base-fields.yml | 9 + .../data_stream/ccr/fields/ecs.yml | 48 + .../data_stream/ccr/fields/fields.yml | 117 ++ .../data_stream/ccr/fields/package-fields.yml | 145 ++ .../data_stream/ccr/manifest.yml | 12 + .../cluster_stats/agent/stream/stream.yml.hbs | 13 + .../cluster_stats/fields/base-fields.yml | 9 + .../data_stream/cluster_stats/fields/ecs.yml | 48 + .../cluster_stats/fields/fields.yml | 91 + .../cluster_stats/fields/package-fields.yml | 120 ++ .../data_stream/cluster_stats/manifest.yml | 12 + .../cluster_stats/sample_event.json | 88 + .../deprecation/agent/stream/log.yml.hbs | 16 + .../elasticsearch/ingest_pipeline/default.yml | 49 + .../ingest_pipeline/pipeline-json.yml | 23 + .../deprecation/fields/base-fields.yml | 12 + .../data_stream/deprecation/fields/fields.yml | 2 + .../deprecation/fields/package-fields.yml | 27 + .../data_stream/deprecation/manifest.yml | 17 + .../enrich/agent/stream/stream.yml.hbs | 13 + .../data_stream/enrich/fields/base-fields.yml | 9 + .../data_stream/enrich/fields/ecs.yml | 48 + .../data_stream/enrich/fields/fields.yml | 48 + .../enrich/fields/package-fields.yml | 49 + .../data_stream/enrich/manifest.yml | 12 + .../data_stream/enrich/sample_event.json | 83 + .../data_stream/gc/agent/stream/log.yml.hbs | 15 + .../elasticsearch/ingest_pipeline/default.yml | 70 + .../data_stream/gc/fields/base-fields.yml | 12 + .../data_stream/gc/fields/fields.yml | 97 + .../data_stream/gc/fields/package-fields.yml | 27 + .../data_stream/gc/manifest.yml | 18 + .../index/agent/stream/stream.yml.hbs | 13 + .../data_stream/index/fields/base-fields.yml | 9 + .../data_stream/index/fields/ecs.yml | 48 + .../data_stream/index/fields/fields.yml | 198 ++ .../index/fields/package-fields.yml | 241 +++ .../data_stream/index/manifest.yml | 12 + .../data_stream/index/sample_event.json | 88 + .../agent/stream/stream.yml.hbs | 13 + .../index_recovery/fields/base-fields.yml | 9 + .../data_stream/index_recovery/fields/ecs.yml | 48 + .../index_recovery/fields/fields.yml | 101 + .../index_recovery/fields/package-fields.yml | 58 + .../data_stream/index_recovery/manifest.yml | 20 + .../index_recovery/sample_event.json | 108 ++ .../index_summary/agent/stream/stream.yml.hbs | 13 + .../index_summary/fields/base-fields.yml | 9 + .../data_stream/index_summary/fields/ecs.yml | 48 + .../index_summary/fields/fields.yml | 122 ++ .../index_summary/fields/package-fields.yml | 85 + .../data_stream/index_summary/manifest.yml | 12 + .../index_summary/sample_event.json | 106 ++ .../ml_job/agent/stream/stream.yml.hbs | 13 + .../data_stream/ml_job/fields/base-fields.yml | 9 + .../data_stream/ml_job/fields/ecs.yml | 48 + .../data_stream/ml_job/fields/fields.yml | 31 + .../ml_job/fields/package-fields.yml | 55 + .../data_stream/ml_job/manifest.yml | 12 + .../data_stream/ml_job/sample_event.json | 37 + .../node/agent/stream/stream.yml.hbs | 13 + .../data_stream/node/fields/base-fields.yml | 9 + .../data_stream/node/fields/ecs.yml | 48 + .../data_stream/node/fields/fields.yml | 39 + .../node/fields/package-fields.yml | 49 + .../data_stream/node/manifest.yml | 12 + .../data_stream/node/sample_event.json | 97 + .../node_stats/agent/stream/stream.yml.hbs | 13 + .../node_stats/fields/base-fields.yml | 9 + .../data_stream/node_stats/fields/ecs.yml | 48 + .../data_stream/node_stats/fields/fields.yml | 309 ++++ .../node_stats/fields/package-fields.yml | 328 ++++ .../data_stream/node_stats/manifest.yml | 12 + .../data_stream/node_stats/sample_event.json | 168 ++ .../pending_tasks/agent/stream/stream.yml.hbs | 13 + .../pending_tasks/fields/base-fields.yml | 9 + .../data_stream/pending_tasks/fields/ecs.yml | 48 + .../pending_tasks/fields/fields.yml | 20 + .../pending_tasks/fields/package-fields.yml | 49 + .../data_stream/pending_tasks/manifest.yml | 12 + .../pending_tasks/sample_event.json | 74 + .../server/agent/stream/log.yml.hbs | 16 + .../elasticsearch/ingest_pipeline/default.yml | 96 + .../ingest_pipeline/pipeline-json.yml | 117 ++ .../data_stream/server/fields/base-fields.yml | 12 + .../data_stream/server/fields/fields.yml | 24 + .../server/fields/package-fields.yml | 27 + .../data_stream/server/manifest.yml | 17 + .../shard/agent/stream/stream.yml.hbs | 13 + .../data_stream/shard/fields/base-fields.yml | 9 + .../data_stream/shard/fields/ecs.yml | 48 + .../data_stream/shard/fields/fields.yml | 31 + .../shard/fields/package-fields.yml | 71 + .../data_stream/shard/manifest.yml | 12 + .../data_stream/shard/sample_event.json | 83 + .../slowlog/agent/stream/log.yml.hbs | 16 + .../elasticsearch/ingest_pipeline/default.yml | 66 + .../ingest_pipeline/pipeline-json.yml | 43 + .../slowlog/fields/base-fields.yml | 12 + .../data_stream/slowlog/fields/fields.yml | 42 + .../slowlog/fields/package-fields.yml | 27 + .../data_stream/slowlog/manifest.yml | 18 + .../1.1.0-preview1/docs/README.md | 1633 +++++++++++++++++ .../1.1.0-preview1/img/logo_elasticsearch.svg | 7 + .../elasticsearch/1.1.0-preview1/manifest.yml | 61 + 116 files changed, 7405 insertions(+) create mode 100755 packages/elasticsearch/1.1.0-preview1/LICENSE.txt create mode 100755 packages/elasticsearch/1.1.0-preview1/changelog.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/agent/stream/log.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ccr/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ccr/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/agent/stream/log.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/default.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/pipeline-json.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/enrich/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/enrich/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/enrich/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/gc/agent/stream/log.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/gc/elasticsearch/ingest_pipeline/default.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/gc/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/agent/stream/log.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/default.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/pipeline-json.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/shard/agent/stream/stream.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/shard/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/shard/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/agent/stream/log.yml.hbs create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/pipeline-json.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/base-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/package-fields.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/manifest.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/docs/README.md create mode 100755 packages/elasticsearch/1.1.0-preview1/img/logo_elasticsearch.svg create mode 100755 packages/elasticsearch/1.1.0-preview1/manifest.yml diff --git a/packages/elasticsearch/1.1.0-preview1/LICENSE.txt b/packages/elasticsearch/1.1.0-preview1/LICENSE.txt new file mode 100755 index 0000000000..809108b857 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/packages/elasticsearch/1.1.0-preview1/changelog.yml b/packages/elasticsearch/1.1.0-preview1/changelog.yml new file mode 100755 index 0000000000..f6d2302dff --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/changelog.yml @@ -0,0 +1,34 @@ +# newer versions go on top +- version: "1.0.0" + changes: + - description: Suffix `stack_monitoring` to the datasets + type: enhancement + link: https://github.com/elastic/integrations/pull/4018 + - description: Align metrics mappings with metricbeat + type: bugfix + link: https://github.com/elastic/integrations/pull/3928 +- version: "0.3.0" + changes: + - description: Add `scope` configuration option for metricsets + type: enhancement + link: https://github.com/elastic/integrations/pull/3275 +- version: "0.2.2" + changes: + - description: Add documentation for multi-fields + type: enhancement + link: https://github.com/elastic/integrations/pull/2916 +- version: "0.2.1" + changes: + - description: Fix version mapping in the index_recovery data stream. + type: bugfix + link: https://github.com/elastic/integrations/pull/2896 +- version: "0.2.0" + changes: + - description: Update to ECS 1.12.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1692 +- version: "0.1.0" + changes: + - description: initial release + type: enhancement + link: https://github.com/elastic/integrations/pull/1365 diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/agent/stream/log.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/agent/stream/log.yml.hbs new file mode 100755 index 0000000000..3e9a31838b --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/agent/stream/log.yml.hbs @@ -0,0 +1,46 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.10.0 + - if: + regexp: + message: "^{" + then: + - decode_json_fields: + fields: [ "message" ] + target: _json + - rename: + fields: + - from: _json.request.body + to: _request + ignore_missing: true + - drop_fields: + fields: [ "_json" ] + else: + - script: + lang: javascript + id: elasticsearch_audit + source: > + var requestRegex = new RegExp("request_body=\\\[(.*)\\\]$"); + function process(event) { + var message = event.Get("message"); + if (message !== null) { + var matches = message.match(requestRegex); + if (matches && matches.length > 1) { + event.Put("_request", matches[1]); + } + } + } + - detect_mime_type: + field: _request + target: http.request.mime_type + - drop_fields: + fields: ['_request'] + ignore_missing: true \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml new file mode 100755 index 0000000000..0cca384c70 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,69 @@ +--- +description: Pipeline for parsing elasticsearch audit logs +processors: + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + - set: + copy_from: "@timestamp" + field: event.created + - grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . + - drop: + if: ctx.first_char != '{' + - pipeline: + if: ctx.first_char == '{' + name: '{< IngestPipeline "pipeline-json" >}' + - set: + field: event.kind + value: event + - set: + field: event.category + value: database + - set: + if: "ctx?.elasticsearch?.audit?.event_type != null" + field: event.type + value: access + - script: + lang: painless + source: >- + def successEvents = ['authentication_success', 'access_granted', 'run_as_granted', 'connection_granted']; + if (ctx?.elasticsearch?.audit?.event_type != null && successEvents.contains(ctx.elasticsearch.audit.event_type)) { + ctx.event.outcome = 'success'; + } else { + ctx.event.outcome = 'failure'; + } + if (ctx?.event.action != null && successEvents.contains(ctx.event.action)) { + ctx.event.outcome = 'success'; + } else { + ctx.event.outcome = 'failure'; + } + - set: + field: host.id + value: "{{elasticsearch.node.id}}" + ignore_empty_value: true + - set: + field: host.name + value: "{{elasticsearch.node.name}}" + ignore_empty_value: true + - append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" + - append: + field: related.user + value: "{{user.effective.name}}" + if: "ctx?.user?.effective?.name != null" + - remove: + field: elasticsearch.audit.@timestamp + - remove: + field: + - first_char +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml new file mode 100755 index 0000000000..ad11496b27 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -0,0 +1,219 @@ +--- +description: Pipeline for parsing elasticsearch audit logs in JSON format +processors: + - json: + field: message + target_field: elasticsearch.audit + - dot_expander: + field: event.type + path: elasticsearch.audit + - drop: + if: ctx.elasticsearch.audit.containsKey('type') && ctx.elasticsearch.audit.type != 'audit' + - drop: + if: '!ctx.elasticsearch.audit.containsKey("type") && !["rest", "transport", "ip_filter", "security_config_change"].contains(ctx.elasticsearch?.audit?.event?.type)' + - remove: + field: elasticsearch.audit.type + ignore_missing: true + - date: + if: ctx.elasticsearch.audit['@timestamp'] != null && ctx.event.timezone != null + field: elasticsearch.audit.@timestamp + target_field: elasticsearch.audit.@timestamp + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + - yyyy-MM-dd'T'HH:mm:ss,SSSZ + timezone: "{{ event.timezone }}" + ignore_failure: true + - remove: + if: ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null + field: event.timezone + - rename: + field: elasticsearch.audit.timestamp + target_field: elasticsearch.audit.@timestamp + ignore_missing: true + - dot_expander: + field: event.action + path: elasticsearch.audit + - remove: + field: event.action + ignore_missing: true + - rename: + field: elasticsearch.audit.event.action + target_field: event.action + ignore_missing: true + - rename: + field: elasticsearch.audit.event.type + target_field: elasticsearch.audit.layer + ignore_missing: true + - dot_expander: + field: origin.address + path: elasticsearch.audit + - grok: + field: elasticsearch.audit.origin.address + patterns: + - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int} + - "%{IPORHOST:source.ip}:%{INT:source.port:int}" + ignore_missing: true + - remove: + field: source.address + ignore_missing: true + - rename: + field: elasticsearch.audit.origin.address + target_field: source.address + ignore_missing: true + - dot_expander: + field: url.path + path: elasticsearch.audit + - dot_expander: + field: url.query + path: elasticsearch.audit + - set: + if: ctx.elasticsearch.audit?.url?.query == null + field: url.original + value: "{{elasticsearch.audit.url.path}}" + ignore_empty_value: true + - set: + if: ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query != null + field: url.original + value: "{{elasticsearch.audit.url.path}}?{{elasticsearch.audit.url.query}}" + - remove: + if: ctx.elasticsearch.audit?.url?.path != null + field: elasticsearch.audit.url.path + - remove: + if: ctx.elasticsearch.audit?.url?.query != null + field: elasticsearch.audit.url.query + - dot_expander: + field: node.id + path: elasticsearch.audit + - dot_expander: + field: node.name + path: elasticsearch.audit + - remove: + field: elasticsearch.node + ignore_missing: true + - rename: + field: elasticsearch.audit.node + target_field: elasticsearch.node + - rename: + field: elasticsearch.audit.change.disable.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.change.enable.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.delete.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.put.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.put.user.full_name + target_field: user.full_name + ignore_missing: true + - rename: + field: elasticsearch.audit.put.user.email + target_field: user.email + ignore_missing: true + - remove: + field: elasticsearch.audit.put + ignore_missing: true + - rename: + field: elasticsearch.audit.invalidate.apikeys.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.invalidate.apikeys.user.realm + target_field: elasticsearch.audit.user.realm + ignore_missing: true + - dot_expander: + field: user.run_as.name + path: elasticsearch.audit + ignore_failure: true + - dot_expander: + field: user.run_as.realm + path: elasticsearch.audit + ignore_failure: true + - convert: + field: elasticsearch.audit.user.run_as.name + target_field: user.effective.name + type: string + ignore_failure: true + - dot_expander: + field: user.name + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.user.name + target_field: user.name + ignore_missing: true + - dot_expander: + field: user.email + path: elasticsearch.audit + - dot_expander: + field: request.method + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.request.method + target_field: http.request.method + ignore_missing: true + - dot_expander: + field: request.body + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.request.body + target_field: http.request.body.content + ignore_missing: true + - dot_expander: + field: request.id + path: elasticsearch.audit + - set: + field: http.request.id + value: "{{{elasticsearch.audit.request.id}}}" + ignore_empty_value: true + - dot_expander: + field: cluster.name + path: elasticsearch.audit + - dot_expander: + field: cluster.uuid + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.cluster.name + target_field: elasticsearch.cluster.name + ignore_missing: true + - rename: + field: elasticsearch.audit.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true + - rename: + field: elasticsearch.audit.level + target_field: log.level + ignore_missing: true + - set: + field: log.level + value: info + override: false + - dot_expander: + field: trace.id + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.trace.id + target_field: trace.id + ignore_missing: true + - remove: + field: elasticsearch.audit.trace.id + ignore_missing: true + - date: + field: elasticsearch.audit.@timestamp + target_field: "@timestamp" + formats: + - ISO8601 + ignore_failure: true + - set: + field: service.type + value: 'elasticsearch' +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/base-fields.yml new file mode 100755 index 0000000000..7c798f4534 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/ecs.yml new file mode 100755 index 0000000000..7b7a839904 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/ecs.yml @@ -0,0 +1,24 @@ +- description: The full HTTP request body. + multi_fields: + - name: text + type: match_only_text + name: http.request.body.content + type: wildcard +- description: IP address of the source (IPv4 or IPv6). + name: source.ip + type: ip +- description: |- + Unmodified original url as seen in the event source. + Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. + This field is meant to represent the URL as it was observed, complete or not. + multi_fields: + - name: text + type: match_only_text + name: url.original + type: wildcard +- description: Short name or login of the user. + multi_fields: + - name: text + type: match_only_text + name: user.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/fields.yml new file mode 100755 index 0000000000..b50e865553 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/fields.yml @@ -0,0 +1,46 @@ +- name: elasticsearch.audit + type: group + fields: + - name: layer + type: keyword + description: 'The layer from which this event originated: rest, transport or ip_filter' + - name: event_type + type: keyword + description: 'The type of event that occurred: anonymous_access_denied, authentication_failed, access_denied, access_granted, connection_granted, connection_denied, tampered_request, run_as_granted, run_as_denied' + - name: origin.type + type: keyword + description: 'Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)' + - name: realm + type: keyword + description: The authentication realm the authentication was validated against + - name: user.realm + type: keyword + description: The user's authentication realm, if authenticated + - name: user.roles + type: keyword + description: Roles to which the principal belongs + - name: user.run_as.name + type: keyword + - name: user.run_as.realm + type: keyword + - name: component + type: keyword + - name: action + type: keyword + description: The name of the action that was executed + - name: url.params + type: keyword + description: REST URI parameters + - name: indices + type: keyword + description: Indices accessed by action + - name: request.id + type: keyword + description: Unique ID of request + - name: request.name + type: keyword + description: The type of request that was executed + - name: message + type: text + - name: invalidate.apikeys.owned_by_authenticated_user + type: boolean diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/package-fields.yml new file mode 100755 index 0000000000..7ef974b1ab --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/package-fields.yml @@ -0,0 +1,27 @@ +- name: elasticsearch + type: group + fields: + - name: component + type: keyword + description: Elasticsearch component from where the log event originated + - name: cluster.uuid + type: keyword + description: UUID of the cluster + - name: cluster.name + type: keyword + description: Name of the cluster + - name: node.id + type: keyword + description: ID of the node + - name: node.name + type: keyword + description: Name of the node + - name: index.name + type: keyword + description: Index name + - name: index.id + type: keyword + description: Index id + - name: shard.id + type: keyword + description: Id of the shard diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/manifest.yml new file mode 100755 index 0000000000..6330111e84 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/manifest.yml @@ -0,0 +1,17 @@ +type: logs +title: Elasticsearch audit logs +release: experimental +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/elasticsearch/*_audit.json + template_path: log.yml.hbs + title: Audit logs + description: Collect Elasticsearch audit logs using log input diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..821b9184b1 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["ccr"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/fields.yml new file mode 100755 index 0000000000..dd84e6f94b --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/fields.yml @@ -0,0 +1,117 @@ +- name: elasticsearch.ccr + type: group + release: ga + fields: + - name: remote_cluster + type: keyword + - name: bytes_read + type: long + - name: last_requested_seq_no + type: long + - name: shard_id + type: integer + - name: total_time + type: group + fields: + - name: read.ms + type: long + - name: read.remote_exec.ms + type: long + - name: write.ms + type: long + - name: read_exceptions + type: nested + - name: requests + type: group + fields: + - name: successful + type: group + fields: + - name: read.count + type: long + - name: write.count + type: long + - name: failed + type: group + fields: + - name: read.count + type: long + - name: write.count + type: long + - name: outstanding + type: group + fields: + - name: read.count + type: long + - name: write.count + type: long + - name: write_buffer + type: group + fields: + - name: size.bytes + type: long + - name: operation.count + type: long + - name: auto_follow + type: group + fields: + - name: failed + type: group + fields: + - name: follow_indices.count + type: long + - name: remote_cluster_state_requests.count + type: long + - name: success + type: group + fields: + - name: follow_indices.count + type: long + - name: leader + type: group + fields: + - name: index + type: keyword + description: | + Name of leader index + - name: max_seq_no + type: long + description: | + Maximum sequence number of operation on the leader shard + - name: global_checkpoint + type: long + - name: follower + type: group + fields: + - name: index + type: keyword + description: | + Name of follower index + - name: shard.number + type: long + description: | + Number of the shard within the index + - name: operations_written + type: long + description: | + Number of operations indexed (replicated) into the follower shard from the leader shard + - name: time_since_last_read.ms + type: long + description: | + Time, in ms, since the follower last fetched from the leader + - name: global_checkpoint + type: long + description: | + Global checkpoint value on follower shard + - name: max_seq_no + type: long + description: | + Maximum sequence number of operation on the follower shard + - name: mapping_version + type: long + - name: settings_version + type: long + - name: aliases_version + type: long + - name: operations.read.count + type: long diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/package-fields.yml new file mode 100755 index 0000000000..299e55caf1 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/package-fields.yml @@ -0,0 +1,145 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: ccr_auto_follow_stats + type: group + fields: + - name: number_of_failed_follow_indices + type: alias + path: elasticsearch.ccr.auto_follow.failed.follow_indices.count + - name: number_of_failed_remote_cluster_state_requests + type: alias + path: elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count + - name: number_of_successful_follow_indices + type: alias + path: elasticsearch.ccr.auto_follow.success.follow_indices.count + - name: follower.failed_read_requests + type: alias + path: elasticsearch.ccr.requests.failed.read.count +- name: ccr_stats + type: group + fields: + - name: shard_id + type: alias + path: elasticsearch.ccr.follower.shard.number + - name: remote_cluster + type: alias + path: elasticsearch.ccr.remote_cluster + - name: leader_index + type: alias + path: elasticsearch.ccr.leader.index + - name: follower_index + type: alias + path: elasticsearch.ccr.follower.index + - name: leader_global_checkpoint + type: alias + path: elasticsearch.ccr.leader.global_checkpoint + - name: leader_max_seq_no + type: alias + path: elasticsearch.ccr.leader.max_seq_no + - name: follower_global_checkpoint + type: alias + path: elasticsearch.ccr.follower.global_checkpoint + - name: follower_max_seq_no + type: alias + path: elasticsearch.ccr.follower.max_seq_no + - name: last_requested_seq_no + type: alias + path: elasticsearch.ccr.last_requested_seq_no + - name: outstanding_read_requests + type: alias + path: elasticsearch.ccr.requests.outstanding.read.count + - name: outstanding_write_requests + type: alias + path: elasticsearch.ccr.requests.outstanding.write.count + - name: write_buffer_operation_count + type: alias + path: elasticsearch.ccr.write_buffer.operation.count + - name: write_buffer_size_in_bytes + type: alias + path: elasticsearch.ccr.write_buffer.size.bytes + - name: follower_mapping_version + type: alias + path: elasticsearch.ccr.follower.mapping_version + - name: follower_settings_version + type: alias + path: elasticsearch.ccr.follower.settings_version + - name: follower_aliases_version + type: alias + path: elasticsearch.ccr.follower.aliases_version + - name: total_read_time_millis + type: alias + path: elasticsearch.ccr.total_time.read.ms + - name: total_read_remote_exec_time_millis + type: alias + path: elasticsearch.ccr.total_time.read.remote_exec.ms + - name: successful_read_requests + type: alias + path: elasticsearch.ccr.requests.successful.read.count + - name: failed_read_requests + type: alias + path: elasticsearch.ccr.requests.failed.read.count + - name: operations_read + type: alias + path: elasticsearch.ccr.follower.operations.read.count + - name: operations_written + type: alias + path: elasticsearch.ccr.follower.operations_written + - name: bytes_read + type: alias + path: elasticsearch.ccr.bytes_read + - name: total_write_time_millis + type: alias + path: elasticsearch.ccr.total_time.write.ms + - name: successful_write_requests + type: alias + path: elasticsearch.ccr.requests.successful.write.count + - name: failed_write_requests + type: alias + path: elasticsearch.ccr.requests.failed.write.count +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/manifest.yml new file mode 100755 index 0000000000..9803fcfb27 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch ccr metrics +release: experimental +dataset: elasticsearch.stack_monitoring.ccr +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: CCR metrics + description: Collect Elasticsearch Cross Cluster Replication metrics diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..d9de79639c --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["cluster_stats"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/fields.yml new file mode 100755 index 0000000000..0d0e52f5be --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/fields.yml @@ -0,0 +1,91 @@ +- name: elasticsearch.cluster.stats + type: group + release: ga + fields: + - name: version + type: keyword + - name: state + type: group + fields: + - name: nodes_hash + type: keyword + - name: master_node + type: keyword + - name: version + type: keyword + - name: state_uuid + type: keyword + - name: status + type: keyword + description: Cluster status (green, yellow, red). + - name: nodes + type: group + fields: + - name: fs.total.bytes + type: long + - name: fs.available.bytes + type: long + - name: count + type: long + description: Total number of nodes in cluster. + - name: master + type: long + description: Number of master-eligible nodes in cluster. + - name: data + type: long + - name: stats.data + type: long + description: Number of data nodes in cluster. + - name: jvm + type: group + fields: + - name: max_uptime.ms + type: long + - name: memory.heap.max.bytes + type: long + - name: memory.heap.used.bytes + type: long + - name: indices + type: group + fields: + - name: store.size.bytes + type: long + - name: total + type: long + - name: docs.total + type: long + description: | + Total number of indices in cluster. + - name: shards + type: group + fields: + - name: count + type: long + description: | + Total number of shards in cluster. + - name: primaries + type: long + description: | + Total number of primary shards in cluster. + - name: fielddata.memory.bytes + type: long + description: | + Memory used for fielddata. + - name: license + type: group + fields: + - name: expiry_date_in_millis + type: long + - name: status + type: keyword + - name: type + type: keyword + - name: stack + type: group + fields: + - name: apm.found + type: boolean + - name: xpack.ccr.available + type: boolean + - name: xpack.ccr.enabled + type: boolean diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/package-fields.yml new file mode 100755 index 0000000000..898baed57d --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/package-fields.yml @@ -0,0 +1,120 @@ +- name: stack_stats + type: group + fields: + - name: apm.found + type: alias + path: elasticsearch.cluster.stats.stack.apm.found + - name: xpack.ccr.enabled + type: alias + path: elasticsearch.cluster.stats.stack.xpack.ccr.enabled + - name: xpack.ccr.available + type: alias + path: elasticsearch.cluster.stats.stack.xpack.ccr.available +- name: license + type: group + fields: + - name: status + type: alias + path: elasticsearch.cluster.stats.license.status + - name: type + type: alias + path: elasticsearch.cluster.stats.license.type +- name: cluster_stats + type: group + fields: + - name: indices + type: group + fields: + - name: count + type: alias + path: elasticsearch.cluster.stats.indices.total + - name: shards.total + type: alias + path: elasticsearch.cluster.stats.indices.shards.count + - name: nodes + type: group + fields: + - name: count.total + type: alias + path: elasticsearch.cluster.stats.nodes.count + - name: jvm + type: group + fields: + - name: max_uptime_in_millis + type: alias + path: elasticsearch.cluster.stats.nodes.jvm.max_uptime.ms + - name: mem.heap_used_in_bytes + type: alias + path: elasticsearch.cluster.stats.nodes.jvm.memory.heap.used.bytes + - name: mem.heap_max_in_bytes + type: alias + path: elasticsearch.cluster.stats.nodes.jvm.memory.heap.max.bytes +- name: cluster_state + type: group + fields: + - name: nodes_hash + type: alias + path: elasticsearch.cluster.stats.state.nodes_hash + - name: version + type: alias + path: elasticsearch.cluster.stats.state.version + - name: master_node + type: alias + path: elasticsearch.cluster.stats.state.master_node + - name: state_uuid + type: alias + path: elasticsearch.cluster.stats.state.state_uuid + - name: status + type: alias + path: elasticsearch.cluster.stats.status +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: elasticsearch + type: group + fields: + - name: version + type: keyword + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/manifest.yml new file mode 100755 index 0000000000..c1cc9efebe --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch cluster_stats metrics +release: experimental +dataset: elasticsearch.stack_monitoring.cluster_stats +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Cluster stats + description: Collect Elasticsearch cluster wide metrics diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/sample_event.json new file mode 100755 index 0000000000..0178d44ed3 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/sample_event.json @@ -0,0 +1,88 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "type": "metricbeat", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:47:15.382Z", + "elasticsearch": { + "cluster": { + "stats": { + "indices": { + "shards": { + "primaries": 39, + "count": 39 + }, + "total": 39, + "fielddata": { + "memory": { + "bytes": 288 + } + } + }, + "nodes": { + "data": 1, + "count": 1, + "master": 1 + }, + "status": "yellow" + }, + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.cluster_stats" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "cluster_stats" + }, + "event": { + "duration": 10597401, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:47:16.373264357Z", + "module": "elasticsearch", + "dataset": "elasticsearch.cluster_stats" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/agent/stream/log.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/agent/stream/log.yml.hbs new file mode 100755 index 0000000000..b0128abf9b --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/agent/stream/log.yml.hbs @@ -0,0 +1,16 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$","_slowlog.log$","_access.log$"] +multiline: + pattern: '^(\[[0-9]{4}-[0-9]{2}-[0-9]{2}|{)' + negate: true + match: after +processors: +# Locale for time zone is only needed in non-json logs +- add_locale.when.not.regexp.message: "^{" +- add_fields: + target: '' + fields: + ecs.version: 1.10.0 \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/default.yml new file mode 100755 index 0000000000..721368a023 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,49 @@ +--- +description: Pipeline for parsing elasticsearch deprecation logs +processors: + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + - set: + copy_from: "@timestamp" + field: event.created + - grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . + - drop: + if: ctx.first_char != '{' + - pipeline: + if: ctx.first_char == '{' + name: '{< IngestPipeline "pipeline-json" >}' + - set: + field: event.kind + value: event + - set: + field: event.category + value: database + - set: + field: event.type + value: info + - set: + field: host.id + value: "{{elasticsearch.node.id}}" + ignore_empty_value: true + - set: + field: host.name + value: "{{elasticsearch.node.name}}" + ignore_empty_value: true + - remove: + field: + - elasticsearch.deprecation.timestamp + - elasticsearch.deprecation.@timestamp + ignore_missing: true + - remove: + field: + - first_char +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/pipeline-json.yml new file mode 100755 index 0000000000..84ae736955 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -0,0 +1,23 @@ +--- +description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format. +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" +processors: + - json: + field: message + add_to_root: true + - dot_expander: + field: "*" + override: true + # Drop any non-deprecation logs that show up due to mixed log output configurations + - drop: + if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.event.dataset)' + # Normalize event.dataset value for kibana queries + - set: + field: event.dataset + value: elasticsearch.deprecation + - set: + field: service.type + value: 'elasticsearch' diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/base-fields.yml new file mode 100755 index 0000000000..7c798f4534 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/fields.yml new file mode 100755 index 0000000000..dfe88fe875 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/fields.yml @@ -0,0 +1,2 @@ +- name: elasticsearch.deprecation + type: group diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/package-fields.yml new file mode 100755 index 0000000000..7ef974b1ab --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/package-fields.yml @@ -0,0 +1,27 @@ +- name: elasticsearch + type: group + fields: + - name: component + type: keyword + description: Elasticsearch component from where the log event originated + - name: cluster.uuid + type: keyword + description: UUID of the cluster + - name: cluster.name + type: keyword + description: Name of the cluster + - name: node.id + type: keyword + description: ID of the node + - name: node.name + type: keyword + description: Name of the node + - name: index.name + type: keyword + description: Index name + - name: index.id + type: keyword + description: Index id + - name: shard.id + type: keyword + description: Id of the shard diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/manifest.yml new file mode 100755 index 0000000000..33f30c1c49 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/manifest.yml @@ -0,0 +1,17 @@ +type: logs +title: Elasticsearch deprecation logs +release: experimental +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/elasticsearch/*_deprecation.json + template_path: log.yml.hbs + title: Deprecation logs + description: Collect Elasticsearch deprecation logs using log input diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..36962e1aff --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["enrich"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/fields.yml new file mode 100755 index 0000000000..b1e626ceee --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/fields.yml @@ -0,0 +1,48 @@ +- name: elasticsearch.enrich + type: group + release: ga + fields: + - name: executing_policy + type: group + fields: + - name: name + type: keyword + - name: task + type: group + fields: + - name: id + type: long + - name: task + type: keyword + - name: action + type: keyword + - name: cancellable + type: boolean + - name: parent_task_id + type: keyword + - name: time + type: group + fields: + - name: start.ms + type: long + - name: running.nano + type: long + - name: queue.size + type: long + description: | + Number of search requests in the queue. + - name: executed_searches.total + type: long + description: | + Number of search requests that enrich processors have executed since node startup. + - name: remote_requests + type: group + fields: + - name: current + type: long + description: | + Current number of outstanding remote requests. + - name: total + type: long + description: | + Number of outstanding remote requests executed since node startup. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/package-fields.yml new file mode 100755 index 0000000000..4b4aaf1bec --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/package-fields.yml @@ -0,0 +1,49 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/manifest.yml new file mode 100755 index 0000000000..4c933add6b --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch enrich metrics +release: experimental +dataset: elasticsearch.stack_monitoring.enrich +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Enrich policies metrics + description: Collect Elasticsearch enrich policies stats diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/sample_event.json new file mode 100755 index 0000000000..db89aa8cea --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/sample_event.json @@ -0,0 +1,83 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:47:15.376Z", + "elasticsearch": { + "node": { + "id": "6XuAxHXaRbeX6LUrxIfAxg" + }, + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + }, + "enrich": { + "executed_searches": { + "total": 0 + }, + "remote_requests": { + "current": 0, + "total": 0 + }, + "queue": { + "size": 0 + } + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.enrich" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "enrich" + }, + "event": { + "duration": 2804362, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:47:16.373180707Z", + "module": "elasticsearch", + "dataset": "elasticsearch.enrich" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/agent/stream/log.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/agent/stream/log.yml.hbs new file mode 100755 index 0000000000..76304b3314 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/agent/stream/log.yml.hbs @@ -0,0 +1,15 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +exclude_lines: ["^(OpenJDK|Java HotSpot).* Server VM ", "^CommandLine flags: ", "^Memory: ", "^{"] # exclude JVM8 banner and JSON +multiline: + pattern: '^(\[?[0-9]{4}-[0-9]{2}-[0-9]{2}|{)' + negate: true + match: after +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.10.0 \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/elasticsearch/ingest_pipeline/default.yml new file mode 100755 index 0000000000..98d32286a5 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,70 @@ +--- +description: Pipeline for parsing Elasticsearch JVM garbage collection logs +processors: + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + - grok: + field: message + patterns: + - "(?:%{JVM8HEADER}|%{JVM9HEADER}) Total time for which application threads were + stopped: %{BASE10NUM:elasticsearch.gc.threads_total_stop_time_sec} seconds, + Stopping threads took: %{BASE10NUM:elasticsearch.gc.stopping_threads_time_sec} + seconds" + - '(?:%{JVM8HEADER}) \[GC \(%{DATA:elasticsearch.gc.phase.name}\) \[YG occupancy: + %{BASE10NUM:elasticsearch.gc.young_gen.used_kb} K \(%{BASE10NUM:elasticsearch.gc.young_gen.size_kb} + K\)\]%{BASE10NUM}: \[Rescan \(parallel\) , %{BASE10NUM:elasticsearch.gc.phase.parallel_rescan_time_sec} + secs\]%{BASE10NUM}: \[weak refs processing, %{BASE10NUM:elasticsearch.gc.phase.weak_refs_processing_time_sec} + secs\]%{BASE10NUM}: \[class unloading, %{BASE10NUM:elasticsearch.gc.phase.class_unload_time_sec} + secs\]%{BASE10NUM}: \[scrub symbol table, %{BASE10NUM:elasticsearch.gc.phase.scrub_symbol_table_time_sec} + secs\]%{BASE10NUM}: \[scrub string table, %{BASE10NUM:elasticsearch.gc.phase.scrub_string_table_time_sec} + secs\]\[1 CMS-remark: %{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)\] + %{BASE10NUM:elasticsearch.gc.heap.used_kb}K\(%{BASE10NUM:elasticsearch.gc.heap.size_kb}K\), + %{BASE10NUM:elasticsearch.gc.phase.duration_sec} secs\] %{PROCTIME}' + - '(?:%{JVM8HEADER}) \[GC \(%{DATA:elasticsearch.gc.phase.name}\) \[%{BASE10NUM} + CMS-initial-mark: %{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)\] + %{BASE10NUM:elasticsearch.gc.heap.used_kb}K\(%{BASE10NUM:elasticsearch.gc.heap.size_kb}K\), + %{BASE10NUM:elasticsearch.gc.phase.duration_sec} secs\] %{PROCTIME}' + - '%{JVM9HEADER} GC\(%{BASE10NUM}\) ParNew: %{BASE10NUM}K-\>%{BASE10NUM:elasticsearch.gc.young_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.young_gen.size_kb}K\)' + - '%{JVM9HEADER} GC\(%{BASE10NUM}\) Old: %{BASE10NUM}K-\>%{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)' + - (?:%{JVM8HEADER}|%{JVM9HEADER}) %{GREEDYMULTILINE:message} + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + JVM8HEADER: "%{TIMESTAMP_ISO8601:timestamp}: %{BASE10NUM:elasticsearch.gc.jvm_runtime_sec}:" + JVM9HEADER: \[%{TIMESTAMP_ISO8601:timestamp}\]\[%{POSINT:process.pid}\]\[%{DATA:elasticsearch.gc.tags}%{SPACE}\] + PROCTIME: + '\[Times: user=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.user_sec} + sys=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.sys_sec}, real=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.real_sec} + secs\]' + - set: + copy_from: "@timestamp" + field: event.created + - date: + field: timestamp + target_field: "@timestamp" + formats: + - ISO8601 + - remove: + field: timestamp + - set: + field: event.kind + value: metric + - set: + field: event.category + value: database + - set: + field: event.type + value: info + - split: + field: elasticsearch.gc.tags + separator: "," + ignore_missing: true + - set: + field: service.type + value: 'elasticsearch' +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/base-fields.yml new file mode 100755 index 0000000000..7c798f4534 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/fields.yml new file mode 100755 index 0000000000..782717a6f0 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/fields.yml @@ -0,0 +1,97 @@ +- name: elasticsearch.gc + type: group + fields: + - name: phase + type: group + fields: + - name: name + type: keyword + description: | + Name of the GC collection phase. + - name: duration_sec + type: float + description: | + Collection phase duration according to the Java virtual machine. + - name: scrub_symbol_table_time_sec + type: float + description: | + Pause time in seconds cleaning up symbol tables. + - name: scrub_string_table_time_sec + type: float + description: | + Pause time in seconds cleaning up string tables. + - name: weak_refs_processing_time_sec + type: float + description: | + Time spent processing weak references in seconds. + - name: parallel_rescan_time_sec + type: float + description: | + Time spent in seconds marking live objects while application is stopped. + - name: class_unload_time_sec + type: float + description: | + Time spent unloading unused classes in seconds. + - name: cpu_time + type: group + fields: + - name: user_sec + type: float + description: | + CPU time spent outside the kernel. + - name: sys_sec + type: float + description: "CPU time spent inside the kernel. \n" + - name: real_sec + type: float + description: | + Total elapsed CPU time spent to complete the collection from start to finish. + - name: jvm_runtime_sec + type: float + description: | + The time from JVM start up in seconds, as a floating point number. + - name: threads_total_stop_time_sec + type: float + description: | + Garbage collection threads total stop time seconds. + - name: stopping_threads_time_sec + type: float + description: | + Time took to stop threads seconds. + - name: tags + type: keyword + description: | + GC logging tags. + - name: heap + type: group + fields: + - name: size_kb + type: integer + description: | + Total heap size in kilobytes. + - name: used_kb + type: integer + description: | + Used heap in kilobytes. + - name: old_gen + type: group + fields: + - name: size_kb + type: integer + description: | + Total size of old generation in kilobytes. + - name: used_kb + type: integer + description: | + Old generation occupancy in kilobytes. + - name: young_gen + type: group + fields: + - name: size_kb + type: integer + description: | + Total size of young generation in kilobytes. + - name: used_kb + type: integer + description: | + Young generation occupancy in kilobytes. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/package-fields.yml new file mode 100755 index 0000000000..7ef974b1ab --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/package-fields.yml @@ -0,0 +1,27 @@ +- name: elasticsearch + type: group + fields: + - name: component + type: keyword + description: Elasticsearch component from where the log event originated + - name: cluster.uuid + type: keyword + description: UUID of the cluster + - name: cluster.name + type: keyword + description: Name of the cluster + - name: node.id + type: keyword + description: ID of the node + - name: node.name + type: keyword + description: Name of the node + - name: index.name + type: keyword + description: Index name + - name: index.id + type: keyword + description: Index id + - name: shard.id + type: keyword + description: Id of the shard diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/manifest.yml new file mode 100755 index 0000000000..901f05cfd3 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/manifest.yml @@ -0,0 +1,18 @@ +type: logs +title: Elasticsearch gc logs +release: experimental +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/elasticsearch/gc.log.[0-9]* + - /var/log/elasticsearch/gc.log + template_path: log.yml.hbs + title: Garbage Collection logs + description: Collect Elasticsearch gc logs using log input diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/index/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..126f414ea5 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["index"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/fields.yml new file mode 100755 index 0000000000..39ecdae58e --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/fields.yml @@ -0,0 +1,198 @@ +- name: elasticsearch.index + type: group + release: ga + fields: + - name: created + type: long + - name: hidden + type: boolean + - name: shards + type: group + fields: + - name: total + type: long + - name: uuid + type: keyword + - name: status + type: keyword + - name: name + type: keyword + description: | + Index name. + - name: primaries + type: group + fields: + - name: search + type: group + fields: + - name: query_total + type: long + - name: query_time_in_millis + type: long + - name: request_cache + type: group + fields: + - name: memory_size_in_bytes + type: long + - name: evictions + type: long + - name: hit_count + type: long + - name: miss_count + type: long + - name: query_cache + type: group + fields: + - name: memory_size_in_bytes + type: long + - name: hit_count + type: long + - name: miss_count + type: long + - name: store.size_in_bytes + type: long + - name: docs.count + type: long + - name: docs.deleted + type: long + - name: segments + type: group + fields: + - name: count + type: long + - name: memory_in_bytes + type: long + - name: terms_memory_in_bytes + type: long + - name: stored_fields_memory_in_bytes + type: long + - name: term_vectors_memory_in_bytes + type: long + - name: norms_memory_in_bytes + type: long + - name: points_memory_in_bytes + type: long + - name: doc_values_memory_in_bytes + type: long + - name: index_writer_memory_in_bytes + type: long + - name: version_map_memory_in_bytes + type: long + - name: fixed_bit_set_memory_in_bytes + type: long + - name: refresh.total_time_in_millis + type: long + - name: refresh.external_total_time_in_millis + type: long + - name: merges.total_size_in_bytes + type: long + - name: indexing + type: group + fields: + - name: index_total + type: long + - name: index_time_in_millis + type: long + - name: throttle_time_in_millis + type: long + - name: total + type: group + fields: + - name: docs.count + type: long + description: | + Total number of documents in the index. + - name: docs.deleted + type: long + description: | + Total number of deleted documents in the index. + - name: store.size.bytes + type: long + - name: store.size_in_bytes + type: long + format: bytes + description: | + Total size of the index in bytes. + - name: query_cache + type: group + fields: + - name: memory_size_in_bytes + type: long + - name: evictions + type: long + - name: hit_count + type: long + - name: miss_count + type: long + - name: fielddata.memory_size_in_bytes + type: long + - name: fielddata.evictions + type: long + - name: request_cache + type: group + fields: + - name: memory_size_in_bytes + type: long + - name: evictions + type: long + - name: hit_count + type: long + - name: miss_count + type: long + - name: merges.total_size_in_bytes + type: long + - name: refresh.total_time_in_millis + type: long + - name: refresh.external_total_time_in_millis + type: long + - name: segments + type: group + fields: + - name: memory_in_bytes + type: long + format: bytes + description: | + Total number of memory used by the segments in bytes. + - name: memory.bytes + type: long + format: bytes + description: | + Total number of memory used by the segments in bytes. + - name: terms_memory_in_bytes + type: long + - name: points_memory_in_bytes + type: long + - name: count + type: long + description: | + Total number of index segments. + - name: doc_values_memory_in_bytes + type: long + - name: norms_memory_in_bytes + type: long + - name: stored_fields_memory_in_bytes + type: long + - name: fixed_bit_set_memory_in_bytes + type: long + - name: term_vectors_memory_in_bytes + type: long + - name: version_map_memory_in_bytes + type: long + - name: index_writer_memory_in_bytes + type: long + - name: search + type: group + fields: + - name: query_total + type: long + - name: query_time_in_millis + type: long + - name: indexing + type: group + fields: + - name: index_total + type: long + - name: index_time_in_millis + type: long + - name: throttle_time_in_millis + type: long diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/package-fields.yml new file mode 100755 index 0000000000..3882adcf73 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/package-fields.yml @@ -0,0 +1,241 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: index_recovery + type: group + fields: + - name: shards.start_time_in_millis + type: alias + path: elasticsearch.index.recovery.start_time.ms + - name: shards.stop_time_in_millis + type: alias + path: elasticsearch.index.recovery.stop_time.ms +- name: indices_stats + type: group + fields: + - name: _all + type: group + fields: + - name: primaries + type: group + fields: + - name: indexing + type: group + fields: + - name: index_total + type: alias + path: elasticsearch.index.summary.primaries.indexing.index.count + - name: index_time_in_millis + type: alias + path: elasticsearch.index.summary.primaries.indexing.index.time.ms + - name: total + type: group + fields: + - name: search + type: group + fields: + - name: query_total + type: alias + path: elasticsearch.index.summary.total.search.query.count + - name: query_time_in_millis + type: alias + path: elasticsearch.index.summary.total.search.query.time.ms + - name: indexing + type: group + fields: + - name: index_total + type: alias + path: elasticsearch.index.summary.total.indexing.index.count +- name: index_stats + type: group + fields: + - name: primaries + type: group + fields: + - name: docs + type: group + fields: + - name: count + path: elasticsearch.index.primaries.docs.count + type: alias + - name: indexing + type: group + fields: + - name: throttle_time_in_millis + path: elasticsearch.index.primaries.indexing.throttle_time_in_millis + type: alias + - name: index_time_in_millis + path: elasticsearch.index.primaries.indexing.index_time_in_millis + type: alias + - name: index_total + path: elasticsearch.index.primaries.indexing.index_total + type: alias + - name: refresh + type: group + fields: + - name: total_time_in_millis + path: elasticsearch.index.primaries.refresh.total_time_in_millis + type: alias + - name: store + type: group + fields: + - name: size_in_bytes + path: elasticsearch.index.primaries.store.size_in_bytes + type: alias + - name: merges + type: group + fields: + - name: total_size_in_bytes + path: elasticsearch.index.primaries.merges.total_size_in_bytes + type: alias + - name: segments + type: group + fields: + - name: count + path: elasticsearch.index.primaries.segments.count + type: alias + - name: total + type: group + fields: + - name: search + type: group + fields: + - name: query_total + path: elasticsearch.index.total.search.query_total + type: alias + - name: query_time_in_millis + path: elasticsearch.index.total.search.query_time_in_millis + type: alias + - name: query_cache + type: group + fields: + - name: memory_size_in_bytes + path: elasticsearch.index.total.query_cache.memory_size_in_bytes + type: alias + - name: fielddata + type: group + fields: + - name: memory_size_in_bytes + path: elasticsearch.index.total.fielddata.memory_size_in_bytes + type: alias + - name: indexing + type: group + fields: + - name: throttle_time_in_millis + path: elasticsearch.index.total.indexing.throttle_time_in_millis + type: alias + - name: index_time_in_millis + path: elasticsearch.index.total.indexing.index_time_in_millis + type: alias + - name: index_total + path: elasticsearch.index.total.indexing.index_total + type: alias + - name: refresh + type: group + fields: + - name: total_time_in_millis + path: elasticsearch.index.total.refresh.total_time_in_millis + type: alias + - name: request_cache + type: group + fields: + - name: memory_size_in_bytes + path: elasticsearch.index.total.request_cache.memory_size_in_bytes + type: alias + - name: store + type: group + fields: + - name: size_in_bytes + path: elasticsearch.index.total.store.size_in_bytes + type: alias + - name: merges + type: group + fields: + - name: total_size_in_bytes + path: elasticsearch.index.total.merges.total_size_in_bytes + type: alias + - name: segments + type: group + fields: + - name: version_map_memory_in_bytes + path: elasticsearch.index.total.segments.version_map_memory_in_bytes + type: alias + - name: norms_memory_in_bytes + path: elasticsearch.index.total.segments.norms_memory_in_bytes + type: alias + - name: count + path: elasticsearch.index.total.segments.count + type: alias + - name: term_vectors_memory_in_bytes + path: elasticsearch.index.total.segments.term_vectors_memory_in_bytes + type: alias + - name: points_memory_in_bytes + path: elasticsearch.index.total.segments.points_memory_in_bytes + type: alias + - name: index_writer_memory_in_bytes + path: elasticsearch.index.total.segments.index_writer_memory_in_bytes + type: alias + - name: memory_in_bytes + path: elasticsearch.index.total.segments.memory_in_bytes + type: alias + - name: doc_values_memory_in_bytes + path: elasticsearch.index.total.segments.doc_values_memory_in_bytes + type: alias + - name: terms_memory_in_bytes + path: elasticsearch.index.total.segments.terms_memory_in_bytes + type: alias + - name: fixed_bit_set_memory_in_bytes + path: elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes + type: alias + - name: stored_fields_memory_in_bytes + path: elasticsearch.index.total.segments.stored_fields_memory_in_bytes + type: alias + - name: index + path: elasticsearch.index.name + type: alias +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index/manifest.yml new file mode 100755 index 0000000000..d38947b5f3 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch index metrics +release: experimental +dataset: elasticsearch.stack_monitoring.index +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Index metrics + description: Collect Elasticsearch index metrics diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/index/sample_event.json new file mode 100755 index 0000000000..5702f5392d --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/sample_event.json @@ -0,0 +1,88 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:46:47.831Z", + "ecs": { + "version": "1.10.0" + }, + "elasticsearch": { + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + }, + "index": { + "total": { + "docs": { + "deleted": 0, + "count": 13267 + }, + "store": { + "size": { + "bytes": 1490775 + } + }, + "segments": { + "memory": { + "bytes": 50388 + }, + "count": 5 + } + }, + "name": ".ds-metrics-elasticsearch.shard-default-2021.07.30-000001" + } + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.index" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "index" + }, + "event": { + "duration": 14394992, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:46:48.854674866Z", + "module": "elasticsearch", + "dataset": "elasticsearch.index" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..0f99ed20f3 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["index_recovery"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/fields.yml new file mode 100755 index 0000000000..1d67006406 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/fields.yml @@ -0,0 +1,101 @@ +- name: elasticsearch.index + type: group + release: ga + fields: + - name: name + type: keyword + - name: recovery + type: group + release: ga + fields: + - name: index + type: group + fields: + - name: files + type: group + fields: + - name: percent + type: keyword + - name: recovered + type: long + - name: reused + type: long + - name: total + type: long + - name: size + type: group + fields: + - name: recovered_in_bytes + type: long + - name: reused_in_bytes + type: long + - name: total_in_bytes + type: long + - name: name + type: keyword + - name: total_time.ms + type: long + - name: stop_time.ms + type: long + - name: start_time.ms + type: long + - name: id + type: long + description: | + Shard recovery id. + - name: type + type: keyword + description: | + Shard recovery type. + - name: primary + type: boolean + description: | + True if primary shard. + - name: stage + type: keyword + description: | + Recovery stage. + - name: translog + type: group + fields: + - name: percent + type: keyword + - name: total + type: long + - name: total_on_start + type: long + - name: target.transport_address + type: keyword + - name: target.id + type: keyword + description: | + Target node id. + - name: target.host + type: keyword + description: | + Target node host address (could be IP address or hostname). + - name: target.name + type: keyword + description: | + Target node name. + - name: source.transport_address + type: keyword + - name: source.id + type: keyword + description: | + Source node id. + - name: source.host + type: keyword + description: | + Source node host address (could be IP address or hostname). + - name: source.name + type: keyword + description: | + Source node name. + - name: verify_index + type: group + fields: + - name: check_index_time.ms + type: long + - name: total_time.ms + type: long diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/package-fields.yml new file mode 100755 index 0000000000..b2e824c300 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/package-fields.yml @@ -0,0 +1,58 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: index_recovery + type: group + fields: + - name: shards.start_time_in_millis + type: alias + path: elasticsearch.index.recovery.start_time.ms + - name: shards.stop_time_in_millis + type: alias + path: elasticsearch.index.recovery.stop_time.ms +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/manifest.yml new file mode 100755 index 0000000000..c122cbf48d --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/manifest.yml @@ -0,0 +1,20 @@ +type: metrics +title: Elasticsearch index_recovery metrics +release: experimental +dataset: elasticsearch.stack_monitoring.index_recovery +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Index recovery metrics + description: Returns information about ongoing and completed shard recoveries for one or more indices. + vars: + - name: active.only + type: bool + title: Fetch active only indices + multi: false + required: false + show_user: true + default: true diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/sample_event.json new file mode 100755 index 0000000000..293dab57f3 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/sample_event.json @@ -0,0 +1,108 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:41:17.832Z", + "ecs": { + "version": "1.10.0" + }, + "elasticsearch": { + "cluster": { + "id": "8l_zoGznQRmtoX9iSC-goA", + "name": "docker-cluster" + }, + "index": { + "name": ".kibana-event-log-8.0.0-000001", + "recovery": { + "id": 0, + "index": { + "files": { + "percent": "0.0%", + "recovered": 0, + "reused": 0, + "total": 0 + }, + "size": { + "recovered_in_bytes": 0, + "reused_in_bytes": 0, + "total_in_bytes": 0 + } + }, + "primary": true, + "source": {}, + "stage": "DONE", + "start_time": { + "ms": 1605819056123 + }, + "stop_time": { + "ms": 1605819058696 + }, + "target": { + "host": "127.0.0.1", + "id": "Fkj12lAFQOex0DwK0HMwHw", + "name": "082618b4bb36", + "transport_address": "127.0.0.1:9300" + }, + "translog": { + "percent": "100.0%", + "total": 0, + "total_on_start": 0 + }, + "type": "EMPTY_STORE" + } + } + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.index_recovery" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "type": "linux", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "index_recovery" + }, + "event": { + "duration": 4139652, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:41:18.844042490Z", + "module": "elasticsearch", + "dataset": "elasticsearch.index_recovery" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..6774f76148 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["index_summary"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/fields.yml new file mode 100755 index 0000000000..3f550d11f5 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/fields.yml @@ -0,0 +1,122 @@ +- name: elasticsearch.index.summary + type: group + release: ga + fields: + - name: primaries + type: group + fields: + - name: docs.count + type: long + description: | + Total number of documents in the index. + - name: docs.deleted + type: long + description: | + Total number of deleted documents in the index. + - name: store.size.bytes + type: long + format: bytes + description: | + Total size of the index in bytes. + - name: segments.count + type: long + description: | + Total number of index segments. + - name: segments.memory.bytes + type: long + format: bytes + description: | + Total number of memory used by the segments in bytes. + - name: indexing + type: group + fields: + - name: index.count + type: long + - name: index.time.ms + type: long + - name: search + type: group + fields: + - name: query + type: group + fields: + - name: count + type: long + - name: time.ms + type: long + - name: bulk + type: group + fields: + - name: operations.count + type: long + - name: size.bytes + type: long + - name: time + type: group + fields: + - name: count.ms + type: long + - name: avg.ms + type: long + - name: avg.bytes + type: long + - name: total + type: group + fields: + - name: docs.count + type: long + description: | + Total number of documents in the index. + - name: docs.deleted + type: long + description: | + Total number of deleted documents in the index. + - name: store.size.bytes + type: long + format: bytes + description: | + Total size of the index in bytes. + - name: segments.count + type: long + description: | + Total number of index segments. + - name: segments.memory.bytes + type: long + format: bytes + description: | + Total number of memory used by the segments in bytes. + - name: indexing + type: group + fields: + - name: index.count + type: long + - name: is_throttled + type: boolean + - name: throttle_time.ms + type: long + - name: index.time.ms + type: long + - name: search + type: group + fields: + - name: query + type: group + fields: + - name: count + type: long + - name: time.ms + type: long + - name: bulk + type: group + fields: + - name: operations.count + type: long + - name: size.bytes + type: long + - name: time + type: group + fields: + - name: avg.ms + type: long + - name: avg.bytes + type: long diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/package-fields.yml new file mode 100755 index 0000000000..8deaa38699 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/package-fields.yml @@ -0,0 +1,85 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: indices_stats + type: group + fields: + - name: _all + type: group + fields: + - name: primaries + type: group + fields: + - name: indexing + type: group + fields: + - name: index_total + type: alias + path: elasticsearch.index.summary.primaries.indexing.index.count + - name: index_time_in_millis + type: alias + path: elasticsearch.index.summary.primaries.indexing.index.time.ms + - name: total + type: group + fields: + - name: search + type: group + fields: + - name: query_total + type: alias + path: elasticsearch.index.summary.total.search.query.count + - name: query_time_in_millis + type: alias + path: elasticsearch.index.summary.total.search.query.time.ms + - name: indexing + type: group + fields: + - name: index_total + type: alias + path: elasticsearch.index.summary.total.indexing.index.count +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/manifest.yml new file mode 100755 index 0000000000..c5f272854d --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch index_summary metrics +release: experimental +dataset: elasticsearch.stack_monitoring.index_summary +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Index summary metrics + description: Collect summaries of Elasticsearch index metrics diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/sample_event.json new file mode 100755 index 0000000000..cd1c357bb8 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/sample_event.json @@ -0,0 +1,106 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "type": "metricbeat", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:47:15.391Z", + "elasticsearch": { + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + }, + "index": { + "summary": { + "primaries": { + "docs": { + "deleted": 7226, + "count": 50723 + }, + "store": { + "size": { + "bytes": 36769186 + } + }, + "segments": { + "memory": { + "bytes": 1790592 + }, + "count": 222 + } + }, + "total": { + "docs": { + "deleted": 7226, + "count": 50723 + }, + "store": { + "size": { + "bytes": 36769186 + } + }, + "segments": { + "memory": { + "bytes": 1790592 + }, + "count": 222 + } + } + } + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.index_summary" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "index_summary" + }, + "event": { + "duration": 12151260, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:47:16.373343461Z", + "module": "elasticsearch", + "dataset": "elasticsearch.index_summary" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..100d1b6bf4 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["ml_job"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/fields.yml new file mode 100755 index 0000000000..558fcb5a94 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/fields.yml @@ -0,0 +1,31 @@ +- name: elasticsearch.ml.job + type: group + release: ga + fields: + - name: id + type: keyword + description: | + Unique ml job id. + - name: state + type: keyword + description: | + Job state. + - name: forecasts_stats.total + type: long + - name: model_size + type: group + fields: + - name: memory_status + type: keyword + - name: data_counts + type: group + fields: + - name: invalid_date_count + type: long + - name: processed_record_count + type: long + description: Processed data events. + - name: data.invalid_date.count + type: long + description: | + The number of records with either a missing date field or a date that could not be parsed. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/package-fields.yml new file mode 100755 index 0000000000..8826f5f723 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/package-fields.yml @@ -0,0 +1,55 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: job_stats.job_id + type: alias + path: elasticsearch.ml.job.id +- name: job_stats.forecasts_stats.total + type: alias + path: elasticsearch.ml.job.forecasts_stats.total +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/manifest.yml new file mode 100755 index 0000000000..4163581733 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch ml_job metrics +release: experimental +dataset: elasticsearch.stack_monitoring.ml_job +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Anomaly detection machine learning job metrics + description: Collect usage information for anomaly detection Machine Learning jobs diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/sample_event.json new file mode 100755 index 0000000000..8c0853b148 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/sample_event.json @@ -0,0 +1,37 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "elasticsearch": { + "cluster": { + "id": "8l_zoGznQRmtoX9iSC-goA", + "name": "docker-cluster" + }, + "ml": { + "job": { + "data_counts": { + "invalid_date_count": 0, + "processed_record_count": 1216 + }, + "forecasts_stats": { + "total": 1 + }, + "id": "low_request_rate", + "model_size": { + "memory_status": "ok" + }, + "state": "opened" + } + }, + "node": { + "id": "a14cf47ef7f2" + } + }, + "event": { + "dataset": "elasticsearch.ml.job", + "duration": 115000, + "module": "elasticsearch" + }, + "metricset": { + "name": "ml_job", + "period": 10000 + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/node/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..94328c6415 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["node"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/fields.yml new file mode 100755 index 0000000000..98d8496a78 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/fields.yml @@ -0,0 +1,39 @@ +- name: elasticsearch.node + type: group + release: ga + fields: + - name: version + type: keyword + description: | + Node version. + - name: jvm + type: group + fields: + - name: version + type: keyword + description: | + JVM version. + - name: memory.heap.init.bytes + type: long + format: bytes + description: | + Heap init used by the JVM in bytes. + - name: memory.heap.max.bytes + type: long + format: bytes + description: | + Heap max used by the JVM in bytes. + - name: memory.nonheap.init.bytes + type: long + format: bytes + description: | + Non-Heap init used by the JVM in bytes. + - name: memory.nonheap.max.bytes + type: long + format: bytes + description: | + Non-Heap max used by the JVM in bytes. + - name: process.mlockall + type: boolean + description: | + If process locked in memory. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/package-fields.yml new file mode 100755 index 0000000000..4b4aaf1bec --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/package-fields.yml @@ -0,0 +1,49 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node/manifest.yml new file mode 100755 index 0000000000..3123c30c1d --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch node metrics +release: experimental +dataset: elasticsearch.stack_monitoring.node +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Node metrics + description: Collect node metrics from Elasticsearch diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/node/sample_event.json new file mode 100755 index 0000000000..116e96c809 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/sample_event.json @@ -0,0 +1,97 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "27d29977-878e-4309-81ed-8788662503ad", + "ephemeral_id": "f8f510e7-9503-4e3d-af7f-da2992648d31", + "type": "metricbeat", + "version": "7.15.0" + }, + "elastic_agent": { + "id": "27d29977-878e-4309-81ed-8788662503ad", + "version": "7.15.0", + "snapshot": true + }, + "@timestamp": "2021-08-03T12:27:26.083Z", + "elasticsearch": { + "cluster": { + "name": "docker-cluster", + "id": "icut8oAwR--oCfUTlFaPMg" + }, + "node": { + "jvm": { + "memory": { + "heap": { + "init": { + "bytes": 1073741824 + }, + "max": { + "bytes": 1073741824 + } + }, + "nonheap": { + "init": { + "bytes": 7667712 + }, + "max": { + "bytes": 0 + } + } + }, + "version": "16.0.1" + }, + "process": { + "mlockall": false + }, + "name": "2b8824139b92", + "id": "saWHxJSZQF6VqGZvEb45Uw", + "version": "7.15.0" + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.node" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.24.0.7" + ], + "name": "docker-fleet-agent", + "id": "1292624d19b2cee1a317ad634c9a8358", + "mac": [ + "02:42:ac:18:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "node" + }, + "event": { + "duration": 9853150, + "agent_id_status": "verified", + "ingested": "2021-08-03T12:27:27.080460943Z", + "module": "elasticsearch", + "dataset": "elasticsearch.node" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..4e66bd5640 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["node_stats"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/fields.yml new file mode 100755 index 0000000000..4f31817122 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/fields.yml @@ -0,0 +1,309 @@ +- name: elasticsearch.node.stats + type: group + release: ga + fields: + - name: indices + type: group + fields: + - name: docs.count + type: long + description: | + Total number of existing documents. + - name: docs.deleted + type: long + description: | + Total number of deleted documents. + - name: segments.count + type: long + description: | + Total number of segments. + - name: segments.memory.bytes + type: long + format: bytes + description: | + Total size of segments in bytes. + - name: store.size.bytes + type: long + description: | + Total size of the store in bytes. + - name: fielddata + type: group + fields: + - name: memory.bytes + type: long + format: bytes + - name: indexing + type: group + fields: + - name: index_time.ms + type: long + - name: index_total.count + type: long + - name: throttle_time.ms + type: long + - name: query_cache + type: group + fields: + - name: memory.bytes + type: long + format: bytes + - name: request_cache + type: group + fields: + - name: memory.bytes + type: long + format: bytes + - name: search + type: group + fields: + - name: query_time.ms + type: long + - name: query_total.count + type: long + - name: segments + type: group + fields: + - name: doc_values.memory.bytes + type: long + format: bytes + - name: fixed_bit_set.memory.bytes + type: long + format: bytes + - name: index_writer.memory.bytes + type: long + format: bytes + - name: norms.memory.bytes + type: long + format: bytes + - name: points.memory.bytes + type: long + format: bytes + - name: stored_fields.memory.bytes + type: long + format: bytes + - name: term_vectors.memory.bytes + type: long + format: bytes + - name: terms.memory.bytes + type: long + format: bytes + - name: version_map.memory.bytes + type: long + format: bytes + - name: jvm.mem.heap + type: group + fields: + - name: max.bytes + type: long + format: bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + - name: pct + type: double + format: percent + - name: jvm.mem.pools + type: group + fields: + - name: old + type: group + fields: + - name: max.bytes + type: long + format: bytes + description: | + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: | + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: | + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: | + Used bytes. + - name: young + type: group + fields: + - name: max.bytes + type: long + format: bytes + description: | + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: | + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: | + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: | + Used bytes. + - name: survivor + type: group + fields: + - name: max.bytes + type: long + format: bytes + description: | + Max bytes. + - name: peak.bytes + type: long + format: bytes + description: | + Peak bytes. + - name: peak_max.bytes + type: long + format: bytes + description: | + Peak max bytes. + - name: used.bytes + type: long + format: bytes + description: | + Used bytes. + - name: jvm.gc.collectors + type: group + fields: + - name: old.collection + type: group + fields: + - name: count + type: long + - name: ms + type: long + - name: young.collection + type: group + fields: + - name: count + type: long + - name: ms + type: long + - name: fs + type: group + fields: + - name: total + type: group + fields: + - name: total_in_bytes + type: long + - name: available_in_bytes + type: long + - name: summary + type: group + fields: + - name: total.bytes + type: long + format: bytes + - name: free.bytes + type: long + format: bytes + - name: available.bytes + type: long + format: bytes + - name: io_stats + type: group + fields: + - name: total + type: group + fields: + - name: operations.count + type: long + - name: read.operations.count + type: long + - name: write.operations.count + type: long + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: load_avg.1m + type: half_float + - name: cgroup + type: group + fields: + - name: cpuacct.usage.ns + type: long + - name: cpu + type: group + fields: + - name: cfs.quota.us + type: long + - name: stat + type: group + fields: + - name: elapsed_periods.count + type: long + - name: times_throttled.count + type: long + - name: time_throttled.ns + type: long + - name: memory + type: group + fields: + - name: control_group + type: keyword + - name: limit.bytes + type: keyword + format: bytes + - name: usage.bytes + type: keyword + format: bytes + - name: process.cpu.pct + type: double + format: percent + - name: thread_pool + type: group + fields: + - name: bulk + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + - name: get + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + - name: index + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + - name: search + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + - name: write + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/package-fields.yml new file mode 100755 index 0000000000..77bb0df01b --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/package-fields.yml @@ -0,0 +1,328 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: node_stats + type: group + fields: + - name: fs + type: group + fields: + - name: total + type: group + fields: + - name: available_in_bytes + type: alias + path: elasticsearch.node.stats.fs.summary.available.bytes + - name: total_in_bytes + type: alias + path: elasticsearch.node.stats.fs.summary.total.bytes + - name: summary + type: group + fields: + - name: available.bytes + type: alias + path: elasticsearch.node.stats.fs.summary.available.bytes + - name: total.bytes + type: alias + path: elasticsearch.node.stats.fs.summary.total.bytes + - name: io_stats + type: group + fields: + - name: total + type: group + fields: + - name: operations + type: alias + path: elasticsearch.node.stats.fs.io_stats.total.operations.count + - name: read_operations + type: alias + path: elasticsearch.node.stats.fs.io_stats.total.read.operations.count + - name: write_operations + type: alias + path: elasticsearch.node.stats.fs.io_stats.total.write.operations.count + - name: indices + type: group + fields: + - name: store + type: group + fields: + - name: size_in_bytes + type: alias + path: elasticsearch.node.stats.indices.store.size.bytes + - name: size.bytes + type: alias + path: elasticsearch.node.stats.indices.store.size.bytes + - name: docs.count + type: alias + path: elasticsearch.node.stats.indices.docs.count + - name: indexing + type: group + fields: + - name: index_time_in_millis + type: alias + path: elasticsearch.node.stats.indices.indexing.index_time.ms + - name: index_total + type: alias + path: elasticsearch.node.stats.indices.indexing.index_total.count + - name: throttle_time_in_millis + type: alias + path: elasticsearch.node.stats.indices.indexing.throttle_time.ms + - name: fielddata + type: group + fields: + - name: memory_size_in_bytes + type: alias + path: elasticsearch.node.stats.indices.fielddata.memory.bytes + - name: query_cache + type: group + fields: + - name: memory_size_in_bytes + type: alias + path: elasticsearch.node.stats.indices.query_cache.memory.bytes + - name: request_cache + type: group + fields: + - name: memory_size_in_bytes + type: alias + path: elasticsearch.node.stats.indices.request_cache.memory.bytes + - name: search + type: group + fields: + - name: query_time_in_millis + type: alias + path: elasticsearch.node.stats.indices.search.query_time.ms + - name: query_total + type: alias + path: elasticsearch.node.stats.indices.search.query_total.count + - name: segments + type: group + fields: + - name: count + type: alias + path: elasticsearch.node.stats.indices.segments.count + - name: doc_values_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.doc_values.memory.bytes + - name: fixed_bit_set_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes + - name: index_writer_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.index_writer.memory.bytes + - name: memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.memory.bytes + - name: norms_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.norms.memory.bytes + - name: points_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.points.memory.bytes + - name: stored_fields_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes + - name: term_vectors_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes + - name: terms_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.terms.memory.bytes + - name: version_map_memory_in_bytes + type: alias + path: elasticsearch.node.stats.indices.segments.version_map.memory.bytes + - name: jvm + type: group + fields: + - name: gc + type: group + fields: + - name: collectors + type: group + fields: + - name: old + type: group + fields: + - name: collection_count + type: alias + path: elasticsearch.node.stats.jvm.gc.collectors.old.collection.count + - name: collection_time_in_millis + type: alias + path: elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms + - name: young + type: group + fields: + - name: collection_count + type: alias + path: elasticsearch.node.stats.jvm.gc.collectors.young.collection.count + - name: collection_time_in_millis + type: alias + path: elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms + - name: mem + type: group + fields: + - name: heap_max_in_bytes + type: alias + path: elasticsearch.node.stats.jvm.mem.heap.max.bytes + - name: heap_used_in_bytes + type: alias + path: elasticsearch.node.stats.jvm.mem.heap.used.bytes + - name: heap_used_percent + type: alias + path: elasticsearch.node.stats.jvm.mem.heap.used.pct + - name: node_id + type: alias + path: elasticsearch.node.id + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: load_average + type: group + fields: + - name: 1m + type: alias + path: elasticsearch.node.stats.os.cpu.load_avg.1m + - name: cgroup + type: group + fields: + - name: cpuacct + type: group + fields: + - name: usage_nanos + type: alias + path: elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns + - name: cpu + type: group + fields: + - name: cfs_quota_micros + type: alias + path: elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: alias + path: elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count + - name: number_of_times_throttled + type: alias + path: elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count + - name: time_throttled_nanos + type: alias + path: elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns + - name: memory + type: group + fields: + - name: control_group + type: alias + path: elasticsearch.node.stats.os.cgroup.memory.control_group + - name: limit_in_bytes + type: alias + path: elasticsearch.node.stats.os.cgroup.memory.limit.bytes + - name: usage_in_bytes + type: alias + path: elasticsearch.node.stats.os.cgroup.memory.usage.bytes + - name: process + type: group + fields: + - name: cpu + type: group + fields: + - name: percent + type: alias + path: elasticsearch.node.stats.process.cpu.pct + - name: thread_pool + type: group + fields: + - name: bulk + type: group + fields: + - name: queue + type: alias + path: elasticsearch.node.stats.thread_pool.bulk.queue.count + - name: rejected + type: alias + path: elasticsearch.node.stats.thread_pool.bulk.rejected.count + - name: get + type: group + fields: + - name: queue + type: alias + path: elasticsearch.node.stats.thread_pool.get.queue.count + - name: rejected + type: alias + path: elasticsearch.node.stats.thread_pool.get.rejected.count + - name: index + type: group + fields: + - name: queue + type: alias + path: elasticsearch.node.stats.thread_pool.index.queue.count + - name: rejected + type: alias + path: elasticsearch.node.stats.thread_pool.index.rejected.count + - name: search + type: group + fields: + - name: queue + type: alias + path: elasticsearch.node.stats.thread_pool.search.queue.count + - name: rejected + type: alias + path: elasticsearch.node.stats.thread_pool.search.rejected.count + - name: write + type: group + fields: + - name: queue + type: alias + path: elasticsearch.node.stats.thread_pool.write.queue.count + - name: rejected + type: alias + path: elasticsearch.node.stats.thread_pool.write.rejected.count +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/manifest.yml new file mode 100755 index 0000000000..2336cf60f2 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch node_stats metrics +release: experimental +dataset: elasticsearch.stack_monitoring.node_stats +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Cluster nodes statistics + description: Collect node stats from Elasticsearch diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/sample_event.json new file mode 100755 index 0000000000..fc1542deec --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/sample_event.json @@ -0,0 +1,168 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:47:15.407Z", + "elasticsearch": { + "node": { + "stats": { + "jvm": { + "mem": { + "pools": { + "young": { + "max": { + "bytes": 0 + }, + "used": { + "bytes": 33554432 + }, + "peak": { + "bytes": 633339904 + }, + "peak_max": { + "bytes": 0 + } + }, + "old": { + "max": { + "bytes": 1073741824 + }, + "used": { + "bytes": 248498176 + }, + "peak": { + "bytes": 371192832 + }, + "peak_max": { + "bytes": 1073741824 + } + }, + "survivor": { + "max": { + "bytes": 0 + }, + "peak": { + "bytes": 67829936 + }, + "peak_max": { + "bytes": 0 + }, + "used": { + "bytes": 3283184 + } + } + } + }, + "gc": { + "collectors": { + "young": { + "collection": { + "ms": 6100, + "count": 425 + } + }, + "old": { + "collection": { + "ms": 0, + "count": 0 + } + } + } + } + }, + "indices": { + "docs": { + "deleted": 7226, + "count": 50805 + }, + "store": { + "size": { + "bytes": 37101213 + } + }, + "segments": { + "memory": { + "bytes": 1805548 + }, + "count": 227 + } + }, + "fs": { + "summary": { + "total": { + "bytes": 958613114880 + }, + "available": { + "bytes": 261931741184 + }, + "free": { + "bytes": 310698074112 + } + } + } + }, + "name": "e7e895f7c41e", + "id": "6XuAxHXaRbeX6LUrxIfAxg" + }, + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.node_stats" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "node_stats" + }, + "event": { + "duration": 32401229, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:47:16.373437564Z", + "module": "elasticsearch", + "dataset": "elasticsearch.node_stats" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..07f97a2108 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["pending_tasks"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/fields.yml new file mode 100755 index 0000000000..324d4327d9 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/fields.yml @@ -0,0 +1,20 @@ +- name: elasticsearch.cluster.pending_task + type: group + release: ga + fields: + - name: insert_order + type: long + description: | + Insert order + - name: priority + type: keyword + description: | + Priority + - name: source + type: keyword + description: | + Source. For example: put-mapping + - name: time_in_queue.ms + type: long + description: | + Time in queue diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/package-fields.yml new file mode 100755 index 0000000000..4b4aaf1bec --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/package-fields.yml @@ -0,0 +1,49 @@ +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/manifest.yml new file mode 100755 index 0000000000..c7320cf64a --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch pending_tasks metrics +release: experimental +dataset: elasticsearch.stack_monitoring.pending_tasks +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Pending tasks metrics + description: Collect cluster-level changes that have not yet been executed. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/sample_event.json new file mode 100755 index 0000000000..7bf7e262ad --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/sample_event.json @@ -0,0 +1,74 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:41:17.832Z", + "ecs": { + "version": "1.10.0" + }, + "elasticsearch": { + "cluster": { + "id": "3LbUkLkURz--FR-YO0wLNA", + "name": "es1", + "pending_task": { + "insert_order": 47, + "priority": "HIGH", + "source": "put-mapping", + "time_in_queue.ms": 34 + } + } + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.pending_task" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "type": "linux", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "pending_task" + }, + "event": { + "duration": 4139652, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:41:18.844042490Z", + "module": "elasticsearch", + "dataset": "elasticsearch.pending_task" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/agent/stream/log.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/server/agent/stream/log.yml.hbs new file mode 100755 index 0000000000..43cb1ceb34 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/agent/stream/log.yml.hbs @@ -0,0 +1,16 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$","_slowlog.log$","_access.log$","_deprecation.log$"] +multiline: + pattern: '^(\[[0-9]{4}-[0-9]{2}-[0-9]{2}|{)' + negate: true + match: after +processors: +# Locale for time zone is only needed in non-json logs +- add_locale.when.not.regexp.message: "^{" +- add_fields: + target: '' + fields: + ecs.version: 1.10.0 \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/default.yml new file mode 100755 index 0000000000..67dc3d2435 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,96 @@ +--- +description: Pipeline for parsing elasticsearch server logs +processors: + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + - set: + copy_from: "@timestamp" + field: event.created + - grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . + - drop: + if: ctx.first_char != '{' + - pipeline: + if: ctx.first_char == '{' + name: '{< IngestPipeline "pipeline-json" >}' + - script: + lang: painless + source: >- + if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.observation_duration != null) { + if (ctx.elasticsearch.server.gc.observation_duration.unit == params.seconds_unit) { + ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_s; + } + if (ctx.elasticsearch.server.gc.observation_duration.unit == params.milliseconds_unit) { + ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time; + } + if (ctx.elasticsearch.server.gc.observation_duration.unit == params.minutes_unit) { + ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_m; + } + } + if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.collection_duration != null) { + if (ctx.elasticsearch.server.gc.collection_duration.unit == params.seconds_unit) { + ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_s; + } + if (ctx.elasticsearch.server.gc.collection_duration.unit == params.milliseconds_unit) { + ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time; + } + if (ctx.elasticsearch.server.gc.collection_duration.unit == params.minutes_unit) { + ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_m; + } + } + params: + minutes_unit: m + seconds_unit: s + milliseconds_unit: ms + ms_in_one_s: 1000 + ms_in_one_m: 60000 + + - set: + field: event.kind + value: event + - set: + field: event.category + value: database + - script: + lang: painless + source: >- + def errorLevels = ['FATAL', 'ERROR']; + if (ctx?.log?.level != null) { + if (errorLevels.contains(ctx.log.level)) { + ctx.event.type = 'error'; + } else { + ctx.event.type = 'info'; + } + } + - set: + field: host.name + value: "{{elasticsearch.node.name}}" + ignore_empty_value: true + - set: + field: host.id + value: "{{elasticsearch.node.id}}" + ignore_empty_value: true + - remove: + field: + - elasticsearch.server.gc.collection_duration.time + - elasticsearch.server.gc.collection_duration.unit + - elasticsearch.server.gc.observation_duration.time + - elasticsearch.server.gc.observation_duration.unit + ignore_missing: true + - remove: + field: + - elasticsearch.server.timestamp + - elasticsearch.server.@timestamp + ignore_missing: true + - remove: + field: + - first_char +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/pipeline-json.yml new file mode 100755 index 0000000000..5d9b063ef2 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -0,0 +1,117 @@ +--- +description: Pipeline for parsing the Elasticsearch 8.0 server log file in JSON format. +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: + - json: + field: message + target_field: elasticsearch.server + - dot_expander: + field: event.dataset + path: elasticsearch.server + - drop: + if: ctx.elasticsearch.server.event.dataset != 'elasticsearch.server' + - set: + value: '{{ elasticsearch.server.event.dataset }}' + field: event.dataset + ignore_empty_value: true + - remove: + field: elasticsearch.server.event.dataset + - dot_expander: + field: ecs.version + path: elasticsearch.server + - set: + value: '{{ elasticsearch.server.ecs.version }}' + field: ecs.version + ignore_empty_value: true + - remove: + field: elasticsearch.server.ecs.version + - dot_expander: + field: service.name + path: elasticsearch.server + - rename: + field: elasticsearch.server.service.name + target_field: service.name + ignore_missing: true + - set: + field: service.type + value: 'elasticsearch' + - dot_expander: + field: elasticsearch.cluster.name + path: elasticsearch.server + - rename: + field: elasticsearch.server.elasticsearch.cluster.name + target_field: elasticsearch.cluster.name + - dot_expander: + field: elasticsearch.node.name + path: elasticsearch.server + - rename: + field: elasticsearch.server.elasticsearch.node.name + target_field: elasticsearch.node.name + - dot_expander: + field: elasticsearch.cluster.uuid + path: elasticsearch.server + - rename: + field: elasticsearch.server.elasticsearch.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true + - dot_expander: + field: elasticsearch.node.id + path: elasticsearch.server + - rename: + field: elasticsearch.server.elasticsearch.node.id + target_field: elasticsearch.node.id + ignore_missing: true + - dot_expander: + field: log.level + path: elasticsearch.server + - rename: + field: elasticsearch.server.log.level + target_field: log.level + ignore_missing: true + - dot_expander: + field: log.logger + path: elasticsearch.server + - rename: + field: elasticsearch.server.log.logger + target_field: log.logger + ignore_missing: true + - dot_expander: + field: process.thread.name + path: elasticsearch.server + - rename: + field: elasticsearch.server.process.thread.name + target_field: process.thread.name + ignore_missing: true + - grok: + field: elasticsearch.server.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + GC_ALL: + \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent + \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] + collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] + GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} + patterns: + - '%{GC_ALL}' + - '%{GC_YOUNG}' + - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} + - remove: + field: elasticsearch.server.message + - set: + field: '@timestamp' + value: '{{ elasticsearch.server.@timestamp }}' + ignore_empty_value: true + - remove: + field: elasticsearch.server.@timestamp + - date: + field: '@timestamp' + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/base-fields.yml new file mode 100755 index 0000000000..7c798f4534 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/fields.yml new file mode 100755 index 0000000000..2bee399d0c --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/fields.yml @@ -0,0 +1,24 @@ +- name: elasticsearch.server + type: group + fields: + - name: stacktrace + type: keyword + - name: gc + type: group + fields: + - name: young + type: group + fields: + - name: one + type: long + - name: two + type: long + - name: overhead_seq + type: long + description: Sequence number + - name: collection_duration.ms + type: float + description: Time spent in GC, in milliseconds + - name: observation_duration.ms + type: float + description: Total time over which collection was observed, in milliseconds diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/package-fields.yml new file mode 100755 index 0000000000..7ef974b1ab --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/package-fields.yml @@ -0,0 +1,27 @@ +- name: elasticsearch + type: group + fields: + - name: component + type: keyword + description: Elasticsearch component from where the log event originated + - name: cluster.uuid + type: keyword + description: UUID of the cluster + - name: cluster.name + type: keyword + description: Name of the cluster + - name: node.id + type: keyword + description: ID of the node + - name: node.name + type: keyword + description: Name of the node + - name: index.name + type: keyword + description: Index name + - name: index.id + type: keyword + description: Index id + - name: shard.id + type: keyword + description: Id of the shard diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/manifest.yml new file mode 100755 index 0000000000..c288597ff6 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/manifest.yml @@ -0,0 +1,17 @@ +type: logs +title: Elasticsearch server logs +release: experimental +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/elasticsearch/*_server.json + template_path: log.yml.hbs + title: Server logs + description: Collect server logs using log input diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/agent/stream/stream.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/agent/stream/stream.yml.hbs new file mode 100755 index 0000000000..b50111dbb8 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +metricsets: ["shard"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/base-fields.yml new file mode 100755 index 0000000000..a3e80e3a54 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/ecs.yml new file mode 100755 index 0000000000..26fee338b7 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/ecs.yml @@ -0,0 +1,48 @@ +- description: |- + Date/time when the event originated. + This is the date/time extracted from the event, typically representing when the event was generated by the source. + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + Required field for all events. + name: '@timestamp' + type: date +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Name of the dataset. + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. + name: event.dataset + type: keyword +- description: |- + Duration of the event in nanoseconds. + If event.start and event.end are known this value should be the difference between the end and start time. + name: event.duration + type: long +- description: |- + Name of the module this data is coming from. + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. + name: event.module + type: keyword +- description: |- + Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + name: host.name + type: keyword +- description: Service address + name: service.address + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/fields.yml new file mode 100755 index 0000000000..2931fa1974 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/fields.yml @@ -0,0 +1,31 @@ +- name: elasticsearch.shard + type: group + release: ga + fields: + - name: primary + type: boolean + description: | + True if this is the primary shard. + - name: number + type: long + description: | + The number of this shard. + - name: state + type: keyword + description: | + The state of this shard. + - name: relocating_node.name + type: keyword + description: | + The node the shard was relocated from. + - name: relocating_node.id + type: keyword + description: | + The node the shard was relocated from. It has the exact same value than relocating_node.name for compatibility purposes. + - name: source_node + type: group + fields: + - name: name + type: keyword + - name: uuid + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/package-fields.yml new file mode 100755 index 0000000000..7695a9cfff --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/package-fields.yml @@ -0,0 +1,71 @@ +- name: shard + type: group + fields: + - name: primary + type: alias + path: elasticsearch.shard.primary + - name: state + type: alias + path: elasticsearch.shard.state + - name: index + type: alias + path: elasticsearch.index.name + - name: node + type: alias + path: elasticsearch.node.id + - name: shard + type: alias + path: elasticsearch.shard.number +- name: timestamp + type: alias + path: '@timestamp' +- name: cluster_uuid + type: alias + path: elasticsearch.cluster.id +- name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name +- name: elasticsearch + type: group + fields: + - name: index.name + type: keyword + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name. + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id. + - name: cluster.state.id + type: keyword + description: | + Elasticsearch state id. + - name: cluster.stats.state.state_uuid + type: keyword + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name. + - name: master + type: boolean + description: | + Is the node the master node? + - name: mlockall + type: boolean + description: | + Is mlockall enabled on the node? diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/manifest.yml new file mode 100755 index 0000000000..70045164f1 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/manifest.yml @@ -0,0 +1,12 @@ +type: metrics +title: Elasticsearch shard metrics +release: experimental +dataset: elasticsearch.stack_monitoring.shard +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Shards metrics + description: Collect Elasticsearch shard metrics diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/sample_event.json new file mode 100755 index 0000000000..bf76eef883 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/sample_event.json @@ -0,0 +1,83 @@ +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:41:17.832Z", + "ecs": { + "version": "1.10.0" + }, + "elasticsearch": { + "node": { + "name": "6XuAxHXaRbeX6LUrxIfAxg" + }, + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q", + "state": { + "id": "mOYQ8E-ORnGSnnp9sB4BCw" + } + }, + "index": { + "name": ".async-search" + }, + "shard": { + "number": 0, + "relocating_node": {}, + "state": "STARTED", + "primary": true + } + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.shard" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "type": "linux", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "shard" + }, + "event": { + "duration": 4139652, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:41:18.844042490Z", + "module": "elasticsearch", + "dataset": "elasticsearch.shard" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/agent/stream/log.yml.hbs b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/agent/stream/log.yml.hbs new file mode 100755 index 0000000000..7c7b9c43b1 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/agent/stream/log.yml.hbs @@ -0,0 +1,16 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +multiline: + pattern: '^(\[?[0-9]{4}-[0-9]{2}-[0-9]{2}|{)' + negate: true + match: after +processors: +# Locale for time zone is only needed in non-json logs +- add_locale.when.not.regexp.message: "^{" +- add_fields: + target: '' + fields: + ecs.version: 1.10.0 \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml new file mode 100755 index 0000000000..76ba77ebd1 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,66 @@ +--- +description: Pipeline for parsing elasticsearch slow logs. +processors: + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + - set: + copy_from: "@timestamp" + field: event.created + - grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . + - drop: + if: ctx.first_char != '{' + - pipeline: + if: ctx.first_char == '{' + name: '{{ IngestPipeline "pipeline-json" }}' + - remove: + field: + - elasticsearch.slowlog.timestamp + - elasticsearch.server.@timestamp + ignore_missing: true + - script: + lang: painless + source: ctx.event.duration = Math.round(ctx.elasticsearch.slowlog.duration * params.scale) + params: + scale: 1000000 + if: ctx.elasticsearch.slowlog?.duration != null + - remove: + field: elasticsearch.slowlog.duration + ignore_missing: true + - set: + field: event.kind + value: event + - set: + field: event.category + value: database + - script: + lang: painless + source: >- + def errorLevels = ['FATAL', 'ERROR']; + if (ctx?.log?.level != null) { + if (errorLevels.contains(ctx.log.level)) { + ctx.event.type = 'error'; + } else { + ctx.event.type = 'info'; + } + } + - set: + field: host.name + value: "{{elasticsearch.node.name}}" + ignore_empty_value: true + - set: + field: host.id + value: "{{elasticsearch.node.id}}" + ignore_empty_value: true + - remove: + field: + - first_char +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/pipeline-json.yml new file mode 100755 index 0000000000..31d1b22fda --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -0,0 +1,43 @@ +--- +description: Pipeline for parsing the Elasticsearch slow logs in JSON format. +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" +processors: + - json: + field: message + add_to_root: true + - dot_expander: + field: "*" + override: true + # Drop any non-slowlog messages that show up due to mixed log output configurations + - drop: + if: '!["elasticsearch.slowlog", "elasticsearch.index_indexing_slowlog", "elasticsearch.index_search_slowlog"].contains(ctx.event.dataset)' + - convert: + field: elasticsearch.slowlog.took_millis + type: float + ignore_missing: true + - rename: + field: elasticsearch.slowlog.took_millis + target_field: elasticsearch.slowlog.duration + ignore_missing: true + - grok: + field: elasticsearch.slowlog.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: "[a-zA-Z0-9_.-]*" + patterns: + - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? + - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] + - set: + field: message + value: "{{ elasticsearch.slowlog.message }}" + ignore_empty_value: true + - remove: + field: elasticsearch.slowlog.message + - set: + field: service.type + value: 'elasticsearch' diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/base-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/base-fields.yml new file mode 100755 index 0000000000..7c798f4534 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/fields.yml new file mode 100755 index 0000000000..61f22cc5d0 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/fields.yml @@ -0,0 +1,42 @@ +- name: elasticsearch.slowlog + type: group + fields: + - name: logger + type: keyword + description: Logger name + - name: took + type: keyword + description: Time it took to execute the query + - name: types + type: keyword + description: Types + - name: stats + type: keyword + description: Stats groups + - name: search_type + type: keyword + description: Search type + - name: source_query + type: keyword + description: Slow query + - name: extra_source + type: keyword + description: Extra source information + - name: total_hits + type: keyword + description: Total hits + - name: total_shards + type: keyword + description: Total queried shards + - name: routing + type: keyword + description: Routing + - name: id + type: keyword + description: Id + - name: type + type: keyword + description: Type + - name: source + type: keyword + description: Source of document that was indexed diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/package-fields.yml new file mode 100755 index 0000000000..7ef974b1ab --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/package-fields.yml @@ -0,0 +1,27 @@ +- name: elasticsearch + type: group + fields: + - name: component + type: keyword + description: Elasticsearch component from where the log event originated + - name: cluster.uuid + type: keyword + description: UUID of the cluster + - name: cluster.name + type: keyword + description: Name of the cluster + - name: node.id + type: keyword + description: ID of the node + - name: node.name + type: keyword + description: Name of the node + - name: index.name + type: keyword + description: Index name + - name: index.id + type: keyword + description: Index id + - name: shard.id + type: keyword + description: Id of the shard diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/manifest.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/manifest.yml new file mode 100755 index 0000000000..77e438f421 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/manifest.yml @@ -0,0 +1,18 @@ +type: logs +title: Elasticsearch slowlog logs +release: experimental +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/elasticsearch/*_index_search_slowlog.json + - /var/log/elasticsearch/*_index_indexing_slowlog.json + template_path: log.yml.hbs + title: Slowlog logs + description: Collect Elasticsearch slowlog logs using log input diff --git a/packages/elasticsearch/1.1.0-preview1/docs/README.md b/packages/elasticsearch/1.1.0-preview1/docs/README.md new file mode 100755 index 0000000000..456dc33182 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/docs/README.md @@ -0,0 +1,1633 @@ +# Elasticsearch + +The `elasticsearch` package collects metrics and logs of Elasticsearch. + +## Compatibility + +The `elasticsearch` package can monitor Elasticsearch 6.7.0 and later. + +## Logs + +NOTE: If you're running against Elasticsearch >= 7.0.0, configure the +`var.paths` setting to point to JSON logs. Otherwise, configure it +to point to plain text logs. + +### Compatibility + +The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and newer. + +### Audit + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.audit.action | The name of the action that was executed | keyword | +| elasticsearch.audit.component | | keyword | +| elasticsearch.audit.event_type | The type of event that occurred: anonymous_access_denied, authentication_failed, access_denied, access_granted, connection_granted, connection_denied, tampered_request, run_as_granted, run_as_denied | keyword | +| elasticsearch.audit.indices | Indices accessed by action | keyword | +| elasticsearch.audit.invalidate.apikeys.owned_by_authenticated_user | | boolean | +| elasticsearch.audit.layer | The layer from which this event originated: rest, transport or ip_filter | keyword | +| elasticsearch.audit.message | | text | +| elasticsearch.audit.origin.type | Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request) | keyword | +| elasticsearch.audit.realm | The authentication realm the authentication was validated against | keyword | +| elasticsearch.audit.request.id | Unique ID of request | keyword | +| elasticsearch.audit.request.name | The type of request that was executed | keyword | +| elasticsearch.audit.url.params | REST URI parameters | keyword | +| elasticsearch.audit.user.realm | The user's authentication realm, if authenticated | keyword | +| elasticsearch.audit.user.roles | Roles to which the principal belongs | keyword | +| elasticsearch.audit.user.run_as.name | | keyword | +| elasticsearch.audit.user.run_as.realm | | keyword | +| elasticsearch.cluster.name | Name of the cluster | keyword | +| elasticsearch.cluster.uuid | UUID of the cluster | keyword | +| elasticsearch.component | Elasticsearch component from where the log event originated | keyword | +| elasticsearch.index.id | Index id | keyword | +| elasticsearch.index.name | Index name | keyword | +| elasticsearch.node.id | ID of the node | keyword | +| elasticsearch.node.name | Name of the node | keyword | +| elasticsearch.shard.id | Id of the shard | keyword | +| http | Fields related to HTTP activity. Use the `url` field set to store the url of the request. | group | +| http.request.body.content | The full HTTP request body. | wildcard | +| http.request.body.content.text | Multi-field of `http.request.body.content`. | match_only_text | +| source | Source fields capture details about the sender of a network exchange/packet. These fields are populated from a network event, packet, or other event containing details of a network transaction. Source fields are usually populated in conjunction with destination fields. The source and destination fields are considered the baseline and should always be filled if an event contains source and destination details from a network transaction. If the event also contains identification of the client and server roles, then the client and server fields should also be populated. | group | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| url | URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. | group | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.original.text | Multi-field of `url.original`. | match_only_text | +| user | The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. | group | +| user.name | Short name or login of the user. | keyword | +| user.name.text | Multi-field of `user.name`. | match_only_text | + + +### Deprecation + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.name | Name of the cluster | keyword | +| elasticsearch.cluster.uuid | UUID of the cluster | keyword | +| elasticsearch.component | Elasticsearch component from where the log event originated | keyword | +| elasticsearch.index.id | Index id | keyword | +| elasticsearch.index.name | Index name | keyword | +| elasticsearch.node.id | ID of the node | keyword | +| elasticsearch.node.name | Name of the node | keyword | +| elasticsearch.shard.id | Id of the shard | keyword | + + +### Garbage collection + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.name | Name of the cluster | keyword | +| elasticsearch.cluster.uuid | UUID of the cluster | keyword | +| elasticsearch.component | Elasticsearch component from where the log event originated | keyword | +| elasticsearch.gc.heap.size_kb | Total heap size in kilobytes. | integer | +| elasticsearch.gc.heap.used_kb | Used heap in kilobytes. | integer | +| elasticsearch.gc.jvm_runtime_sec | The time from JVM start up in seconds, as a floating point number. | float | +| elasticsearch.gc.old_gen.size_kb | Total size of old generation in kilobytes. | integer | +| elasticsearch.gc.old_gen.used_kb | Old generation occupancy in kilobytes. | integer | +| elasticsearch.gc.phase.class_unload_time_sec | Time spent unloading unused classes in seconds. | float | +| elasticsearch.gc.phase.cpu_time.real_sec | Total elapsed CPU time spent to complete the collection from start to finish. | float | +| elasticsearch.gc.phase.cpu_time.sys_sec | CPU time spent inside the kernel. | float | +| elasticsearch.gc.phase.cpu_time.user_sec | CPU time spent outside the kernel. | float | +| elasticsearch.gc.phase.duration_sec | Collection phase duration according to the Java virtual machine. | float | +| elasticsearch.gc.phase.name | Name of the GC collection phase. | keyword | +| elasticsearch.gc.phase.parallel_rescan_time_sec | Time spent in seconds marking live objects while application is stopped. | float | +| elasticsearch.gc.phase.scrub_string_table_time_sec | Pause time in seconds cleaning up string tables. | float | +| elasticsearch.gc.phase.scrub_symbol_table_time_sec | Pause time in seconds cleaning up symbol tables. | float | +| elasticsearch.gc.phase.weak_refs_processing_time_sec | Time spent processing weak references in seconds. | float | +| elasticsearch.gc.stopping_threads_time_sec | Time took to stop threads seconds. | float | +| elasticsearch.gc.tags | GC logging tags. | keyword | +| elasticsearch.gc.threads_total_stop_time_sec | Garbage collection threads total stop time seconds. | float | +| elasticsearch.gc.young_gen.size_kb | Total size of young generation in kilobytes. | integer | +| elasticsearch.gc.young_gen.used_kb | Young generation occupancy in kilobytes. | integer | +| elasticsearch.index.id | Index id | keyword | +| elasticsearch.index.name | Index name | keyword | +| elasticsearch.node.id | ID of the node | keyword | +| elasticsearch.node.name | Name of the node | keyword | +| elasticsearch.shard.id | Id of the shard | keyword | + + +### Server + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.name | Name of the cluster | keyword | +| elasticsearch.cluster.uuid | UUID of the cluster | keyword | +| elasticsearch.component | Elasticsearch component from where the log event originated | keyword | +| elasticsearch.index.id | Index id | keyword | +| elasticsearch.index.name | Index name | keyword | +| elasticsearch.node.id | ID of the node | keyword | +| elasticsearch.node.name | Name of the node | keyword | +| elasticsearch.server.gc.collection_duration.ms | Time spent in GC, in milliseconds | float | +| elasticsearch.server.gc.observation_duration.ms | Total time over which collection was observed, in milliseconds | float | +| elasticsearch.server.gc.overhead_seq | Sequence number | long | +| elasticsearch.server.gc.young.one | | long | +| elasticsearch.server.gc.young.two | | long | +| elasticsearch.server.stacktrace | | keyword | +| elasticsearch.shard.id | Id of the shard | keyword | + + +### Slowlog + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.name | Name of the cluster | keyword | +| elasticsearch.cluster.uuid | UUID of the cluster | keyword | +| elasticsearch.component | Elasticsearch component from where the log event originated | keyword | +| elasticsearch.index.id | Index id | keyword | +| elasticsearch.index.name | Index name | keyword | +| elasticsearch.node.id | ID of the node | keyword | +| elasticsearch.node.name | Name of the node | keyword | +| elasticsearch.shard.id | Id of the shard | keyword | +| elasticsearch.slowlog.extra_source | Extra source information | keyword | +| elasticsearch.slowlog.id | Id | keyword | +| elasticsearch.slowlog.logger | Logger name | keyword | +| elasticsearch.slowlog.routing | Routing | keyword | +| elasticsearch.slowlog.search_type | Search type | keyword | +| elasticsearch.slowlog.source | Source of document that was indexed | keyword | +| elasticsearch.slowlog.source_query | Slow query | keyword | +| elasticsearch.slowlog.stats | Stats groups | keyword | +| elasticsearch.slowlog.took | Time it took to execute the query | keyword | +| elasticsearch.slowlog.total_hits | Total hits | keyword | +| elasticsearch.slowlog.total_shards | Total queried shards | keyword | +| elasticsearch.slowlog.type | Type | keyword | +| elasticsearch.slowlog.types | Types | keyword | + + +## Metrics + +### Usage for Stack Monitoring + +The `elasticsearch` package can be used to collect logs and metrics shown in our Stack Monitoring +UI in Kibana. + +### Metric-specific configuration notes + +Like other package, `elasticsearch` metrics collection accepts a `hosts` configuration setting. +This setting can contain a list of entries. The related `scope` setting determines how each entry in +the `hosts` list is interpreted by the module. + +* If `scope` is set to `node` (default), each entry in the `hosts` list indicates a distinct node in an + Elasticsearch cluster. +* If `scope` is set to `cluster`, each entry in the `hosts` list indicates a single endpoint for a distinct + Elasticsearch cluster (for example, a load-balancing proxy fronting the cluster). + +### Cross Cluster Replication + +CCR It uses the Cross-Cluster Replication Stats API endpoint to fetch metrics about cross-cluster +replication from the Elasticsearch clusters that are participating in cross-cluster +replication. + +If the Elasticsearch cluster does not have cross-cluster replication enabled, this package +will not collect metrics. A DEBUG log message about this will be emitted in the log. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.ccr.auto_follow.failed.follow_indices.count | | long | +| elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count | | long | +| elasticsearch.ccr.auto_follow.success.follow_indices.count | | long | +| elasticsearch.ccr.bytes_read | | long | +| elasticsearch.ccr.follower.aliases_version | | long | +| elasticsearch.ccr.follower.global_checkpoint | Global checkpoint value on follower shard | long | +| elasticsearch.ccr.follower.index | Name of follower index | keyword | +| elasticsearch.ccr.follower.mapping_version | | long | +| elasticsearch.ccr.follower.max_seq_no | Maximum sequence number of operation on the follower shard | long | +| elasticsearch.ccr.follower.operations.read.count | | long | +| elasticsearch.ccr.follower.operations_written | Number of operations indexed (replicated) into the follower shard from the leader shard | long | +| elasticsearch.ccr.follower.settings_version | | long | +| elasticsearch.ccr.follower.shard.number | Number of the shard within the index | long | +| elasticsearch.ccr.follower.time_since_last_read.ms | Time, in ms, since the follower last fetched from the leader | long | +| elasticsearch.ccr.last_requested_seq_no | | long | +| elasticsearch.ccr.leader.global_checkpoint | | long | +| elasticsearch.ccr.leader.index | Name of leader index | keyword | +| elasticsearch.ccr.leader.max_seq_no | Maximum sequence number of operation on the leader shard | long | +| elasticsearch.ccr.read_exceptions | | nested | +| elasticsearch.ccr.remote_cluster | | keyword | +| elasticsearch.ccr.requests.failed.read.count | | long | +| elasticsearch.ccr.requests.failed.write.count | | long | +| elasticsearch.ccr.requests.outstanding.read.count | | long | +| elasticsearch.ccr.requests.outstanding.write.count | | long | +| elasticsearch.ccr.requests.successful.read.count | | long | +| elasticsearch.ccr.requests.successful.write.count | | long | +| elasticsearch.ccr.shard_id | | integer | +| elasticsearch.ccr.total_time.read.ms | | long | +| elasticsearch.ccr.total_time.read.remote_exec.ms | | long | +| elasticsearch.ccr.total_time.write.ms | | long | +| elasticsearch.ccr.write_buffer.operation.count | | long | +| elasticsearch.ccr.write_buffer.size.bytes | | long | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | + +### Cluster Stats + +`cluster_stats` interrogates the +[Cluster Stats API endpoint](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html) +to fetch information about the Elasticsearch cluster. + +An example event for `cluster_stats` looks as following: + +```json +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "type": "metricbeat", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:47:15.382Z", + "elasticsearch": { + "cluster": { + "stats": { + "indices": { + "shards": { + "primaries": 39, + "count": 39 + }, + "total": 39, + "fielddata": { + "memory": { + "bytes": 288 + } + } + }, + "nodes": { + "data": 1, + "count": 1, + "master": 1 + }, + "status": "yellow" + }, + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.cluster_stats" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "cluster_stats" + }, + "event": { + "duration": 10597401, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:47:16.373264357Z", + "module": "elasticsearch", + "dataset": "elasticsearch.cluster_stats" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.cluster.stats.indices.docs.total | Total number of indices in cluster. | long | +| elasticsearch.cluster.stats.indices.fielddata.memory.bytes | Memory used for fielddata. | long | +| elasticsearch.cluster.stats.indices.shards.count | Total number of shards in cluster. | long | +| elasticsearch.cluster.stats.indices.shards.primaries | Total number of primary shards in cluster. | long | +| elasticsearch.cluster.stats.indices.store.size.bytes | | long | +| elasticsearch.cluster.stats.indices.total | | long | +| elasticsearch.cluster.stats.license.expiry_date_in_millis | | long | +| elasticsearch.cluster.stats.license.status | | keyword | +| elasticsearch.cluster.stats.license.type | | keyword | +| elasticsearch.cluster.stats.nodes.count | Total number of nodes in cluster. | long | +| elasticsearch.cluster.stats.nodes.data | | long | +| elasticsearch.cluster.stats.nodes.fs.available.bytes | | long | +| elasticsearch.cluster.stats.nodes.fs.total.bytes | | long | +| elasticsearch.cluster.stats.nodes.jvm.max_uptime.ms | | long | +| elasticsearch.cluster.stats.nodes.jvm.memory.heap.max.bytes | | long | +| elasticsearch.cluster.stats.nodes.jvm.memory.heap.used.bytes | | long | +| elasticsearch.cluster.stats.nodes.master | Number of master-eligible nodes in cluster. | long | +| elasticsearch.cluster.stats.nodes.stats.data | Number of data nodes in cluster. | long | +| elasticsearch.cluster.stats.stack.apm.found | | boolean | +| elasticsearch.cluster.stats.stack.xpack.ccr.available | | boolean | +| elasticsearch.cluster.stats.stack.xpack.ccr.enabled | | boolean | +| elasticsearch.cluster.stats.state.master_node | | keyword | +| elasticsearch.cluster.stats.state.nodes_hash | | keyword | +| elasticsearch.cluster.stats.state.state_uuid | | keyword | +| elasticsearch.cluster.stats.state.version | | keyword | +| elasticsearch.cluster.stats.status | Cluster status (green, yellow, red). | keyword | +| elasticsearch.cluster.stats.version | | keyword | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | +| elasticsearch.version | | keyword | + +### Enrich + +Enrch interrogates the [Enrich Stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-apis.html) +endpoint to fetch information about Enrich coordinator nodesin the Elasticsearch cluster that are participating in +ingest-time enrichment. + +An example event for `enrich` looks as following: + +```json +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:47:15.376Z", + "elasticsearch": { + "node": { + "id": "6XuAxHXaRbeX6LUrxIfAxg" + }, + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + }, + "enrich": { + "executed_searches": { + "total": 0 + }, + "remote_requests": { + "current": 0, + "total": 0 + }, + "queue": { + "size": 0 + } + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.enrich" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "enrich" + }, + "event": { + "duration": 2804362, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:47:16.373180707Z", + "module": "elasticsearch", + "dataset": "elasticsearch.enrich" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.enrich.executed_searches.total | Number of search requests that enrich processors have executed since node startup. | long | +| elasticsearch.enrich.executing_policy.name | | keyword | +| elasticsearch.enrich.executing_policy.task.action | | keyword | +| elasticsearch.enrich.executing_policy.task.cancellable | | boolean | +| elasticsearch.enrich.executing_policy.task.id | | long | +| elasticsearch.enrich.executing_policy.task.parent_task_id | | keyword | +| elasticsearch.enrich.executing_policy.task.task | | keyword | +| elasticsearch.enrich.executing_policy.task.time.running.nano | | long | +| elasticsearch.enrich.executing_policy.task.time.start.ms | | long | +| elasticsearch.enrich.queue.size | Number of search requests in the queue. | long | +| elasticsearch.enrich.remote_requests.current | Current number of outstanding remote requests. | long | +| elasticsearch.enrich.remote_requests.total | Number of outstanding remote requests executed since node startup. | long | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | + +### Index + +An example event for `index` looks as following: + +```json +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:46:47.831Z", + "ecs": { + "version": "1.10.0" + }, + "elasticsearch": { + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + }, + "index": { + "total": { + "docs": { + "deleted": 0, + "count": 13267 + }, + "store": { + "size": { + "bytes": 1490775 + } + }, + "segments": { + "memory": { + "bytes": 50388 + }, + "count": 5 + } + }, + "name": ".ds-metrics-elasticsearch.shard-default-2021.07.30-000001" + } + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.index" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "index" + }, + "event": { + "duration": 14394992, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:46:48.854674866Z", + "module": "elasticsearch", + "dataset": "elasticsearch.index" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.index.created | | long | +| elasticsearch.index.hidden | | boolean | +| elasticsearch.index.name | Index name. | keyword | +| elasticsearch.index.primaries.docs.count | | long | +| elasticsearch.index.primaries.docs.deleted | | long | +| elasticsearch.index.primaries.indexing.index_time_in_millis | | long | +| elasticsearch.index.primaries.indexing.index_total | | long | +| elasticsearch.index.primaries.indexing.throttle_time_in_millis | | long | +| elasticsearch.index.primaries.merges.total_size_in_bytes | | long | +| elasticsearch.index.primaries.query_cache.hit_count | | long | +| elasticsearch.index.primaries.query_cache.memory_size_in_bytes | | long | +| elasticsearch.index.primaries.query_cache.miss_count | | long | +| elasticsearch.index.primaries.refresh.external_total_time_in_millis | | long | +| elasticsearch.index.primaries.refresh.total_time_in_millis | | long | +| elasticsearch.index.primaries.request_cache.evictions | | long | +| elasticsearch.index.primaries.request_cache.hit_count | | long | +| elasticsearch.index.primaries.request_cache.memory_size_in_bytes | | long | +| elasticsearch.index.primaries.request_cache.miss_count | | long | +| elasticsearch.index.primaries.search.query_time_in_millis | | long | +| elasticsearch.index.primaries.search.query_total | | long | +| elasticsearch.index.primaries.segments.count | | long | +| elasticsearch.index.primaries.segments.doc_values_memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.fixed_bit_set_memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.index_writer_memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.norms_memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.points_memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.stored_fields_memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.term_vectors_memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.terms_memory_in_bytes | | long | +| elasticsearch.index.primaries.segments.version_map_memory_in_bytes | | long | +| elasticsearch.index.primaries.store.size_in_bytes | | long | +| elasticsearch.index.shards.total | | long | +| elasticsearch.index.status | | keyword | +| elasticsearch.index.total.docs.count | Total number of documents in the index. | long | +| elasticsearch.index.total.docs.deleted | Total number of deleted documents in the index. | long | +| elasticsearch.index.total.fielddata.evictions | | long | +| elasticsearch.index.total.fielddata.memory_size_in_bytes | | long | +| elasticsearch.index.total.indexing.index_time_in_millis | | long | +| elasticsearch.index.total.indexing.index_total | | long | +| elasticsearch.index.total.indexing.throttle_time_in_millis | | long | +| elasticsearch.index.total.merges.total_size_in_bytes | | long | +| elasticsearch.index.total.query_cache.evictions | | long | +| elasticsearch.index.total.query_cache.hit_count | | long | +| elasticsearch.index.total.query_cache.memory_size_in_bytes | | long | +| elasticsearch.index.total.query_cache.miss_count | | long | +| elasticsearch.index.total.refresh.external_total_time_in_millis | | long | +| elasticsearch.index.total.refresh.total_time_in_millis | | long | +| elasticsearch.index.total.request_cache.evictions | | long | +| elasticsearch.index.total.request_cache.hit_count | | long | +| elasticsearch.index.total.request_cache.memory_size_in_bytes | | long | +| elasticsearch.index.total.request_cache.miss_count | | long | +| elasticsearch.index.total.search.query_time_in_millis | | long | +| elasticsearch.index.total.search.query_total | | long | +| elasticsearch.index.total.segments.count | Total number of index segments. | long | +| elasticsearch.index.total.segments.doc_values_memory_in_bytes | | long | +| elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes | | long | +| elasticsearch.index.total.segments.index_writer_memory_in_bytes | | long | +| elasticsearch.index.total.segments.memory_in_bytes | Total number of memory used by the segments in bytes. | long | +| elasticsearch.index.total.segments.norms_memory_in_bytes | | long | +| elasticsearch.index.total.segments.points_memory_in_bytes | | long | +| elasticsearch.index.total.segments.stored_fields_memory_in_bytes | | long | +| elasticsearch.index.total.segments.term_vectors_memory_in_bytes | | long | +| elasticsearch.index.total.segments.terms_memory_in_bytes | | long | +| elasticsearch.index.total.segments.version_map_memory_in_bytes | | long | +| elasticsearch.index.total.store.size_in_bytes | Total size of the index in bytes. | long | +| elasticsearch.index.uuid | | keyword | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | + +### Index recovery + +By default only data about indices which are under active recovery are fetched. +To gather data about all indices set `active_only: false`. + +An example event for `index_recovery` looks as following: + +```json +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:41:17.832Z", + "ecs": { + "version": "1.10.0" + }, + "elasticsearch": { + "cluster": { + "id": "8l_zoGznQRmtoX9iSC-goA", + "name": "docker-cluster" + }, + "index": { + "name": ".kibana-event-log-8.0.0-000001", + "recovery": { + "id": 0, + "index": { + "files": { + "percent": "0.0%", + "recovered": 0, + "reused": 0, + "total": 0 + }, + "size": { + "recovered_in_bytes": 0, + "reused_in_bytes": 0, + "total_in_bytes": 0 + } + }, + "primary": true, + "source": {}, + "stage": "DONE", + "start_time": { + "ms": 1605819056123 + }, + "stop_time": { + "ms": 1605819058696 + }, + "target": { + "host": "127.0.0.1", + "id": "Fkj12lAFQOex0DwK0HMwHw", + "name": "082618b4bb36", + "transport_address": "127.0.0.1:9300" + }, + "translog": { + "percent": "100.0%", + "total": 0, + "total_on_start": 0 + }, + "type": "EMPTY_STORE" + } + } + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.index_recovery" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "type": "linux", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "index_recovery" + }, + "event": { + "duration": 4139652, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:41:18.844042490Z", + "module": "elasticsearch", + "dataset": "elasticsearch.index_recovery" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.index.name | | keyword | +| elasticsearch.index.recovery.id | Shard recovery id. | long | +| elasticsearch.index.recovery.index.files.percent | | keyword | +| elasticsearch.index.recovery.index.files.recovered | | long | +| elasticsearch.index.recovery.index.files.reused | | long | +| elasticsearch.index.recovery.index.files.total | | long | +| elasticsearch.index.recovery.index.size.recovered_in_bytes | | long | +| elasticsearch.index.recovery.index.size.reused_in_bytes | | long | +| elasticsearch.index.recovery.index.size.total_in_bytes | | long | +| elasticsearch.index.recovery.name | | keyword | +| elasticsearch.index.recovery.primary | True if primary shard. | boolean | +| elasticsearch.index.recovery.source.host | Source node host address (could be IP address or hostname). | keyword | +| elasticsearch.index.recovery.source.id | Source node id. | keyword | +| elasticsearch.index.recovery.source.name | Source node name. | keyword | +| elasticsearch.index.recovery.source.transport_address | | keyword | +| elasticsearch.index.recovery.stage | Recovery stage. | keyword | +| elasticsearch.index.recovery.start_time.ms | | long | +| elasticsearch.index.recovery.stop_time.ms | | long | +| elasticsearch.index.recovery.target.host | Target node host address (could be IP address or hostname). | keyword | +| elasticsearch.index.recovery.target.id | Target node id. | keyword | +| elasticsearch.index.recovery.target.name | Target node name. | keyword | +| elasticsearch.index.recovery.target.transport_address | | keyword | +| elasticsearch.index.recovery.total_time.ms | | long | +| elasticsearch.index.recovery.translog.percent | | keyword | +| elasticsearch.index.recovery.translog.total | | long | +| elasticsearch.index.recovery.translog.total_on_start | | long | +| elasticsearch.index.recovery.type | Shard recovery type. | keyword | +| elasticsearch.index.recovery.verify_index.check_index_time.ms | | long | +| elasticsearch.index.recovery.verify_index.total_time.ms | | long | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | +| version | | long | + +### Index summary + +An example event for `index_summary` looks as following: + +```json +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "type": "metricbeat", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:47:15.391Z", + "elasticsearch": { + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + }, + "index": { + "summary": { + "primaries": { + "docs": { + "deleted": 7226, + "count": 50723 + }, + "store": { + "size": { + "bytes": 36769186 + } + }, + "segments": { + "memory": { + "bytes": 1790592 + }, + "count": 222 + } + }, + "total": { + "docs": { + "deleted": 7226, + "count": 50723 + }, + "store": { + "size": { + "bytes": 36769186 + } + }, + "segments": { + "memory": { + "bytes": 1790592 + }, + "count": 222 + } + } + } + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.index_summary" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "index_summary" + }, + "event": { + "duration": 12151260, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:47:16.373343461Z", + "module": "elasticsearch", + "dataset": "elasticsearch.index_summary" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.index.summary.primaries.bulk.operations.count | | long | +| elasticsearch.index.summary.primaries.bulk.size.bytes | | long | +| elasticsearch.index.summary.primaries.bulk.time.avg.bytes | | long | +| elasticsearch.index.summary.primaries.bulk.time.avg.ms | | long | +| elasticsearch.index.summary.primaries.bulk.time.count.ms | | long | +| elasticsearch.index.summary.primaries.docs.count | Total number of documents in the index. | long | +| elasticsearch.index.summary.primaries.docs.deleted | Total number of deleted documents in the index. | long | +| elasticsearch.index.summary.primaries.indexing.index.count | | long | +| elasticsearch.index.summary.primaries.indexing.index.time.ms | | long | +| elasticsearch.index.summary.primaries.search.query.count | | long | +| elasticsearch.index.summary.primaries.search.query.time.ms | | long | +| elasticsearch.index.summary.primaries.segments.count | Total number of index segments. | long | +| elasticsearch.index.summary.primaries.segments.memory.bytes | Total number of memory used by the segments in bytes. | long | +| elasticsearch.index.summary.primaries.store.size.bytes | Total size of the index in bytes. | long | +| elasticsearch.index.summary.total.bulk.operations.count | | long | +| elasticsearch.index.summary.total.bulk.size.bytes | | long | +| elasticsearch.index.summary.total.bulk.time.avg.bytes | | long | +| elasticsearch.index.summary.total.bulk.time.avg.ms | | long | +| elasticsearch.index.summary.total.docs.count | Total number of documents in the index. | long | +| elasticsearch.index.summary.total.docs.deleted | Total number of deleted documents in the index. | long | +| elasticsearch.index.summary.total.indexing.index.count | | long | +| elasticsearch.index.summary.total.indexing.index.time.ms | | long | +| elasticsearch.index.summary.total.indexing.is_throttled | | boolean | +| elasticsearch.index.summary.total.indexing.throttle_time.ms | | long | +| elasticsearch.index.summary.total.search.query.count | | long | +| elasticsearch.index.summary.total.search.query.time.ms | | long | +| elasticsearch.index.summary.total.segments.count | Total number of index segments. | long | +| elasticsearch.index.summary.total.segments.memory.bytes | Total number of memory used by the segments in bytes. | long | +| elasticsearch.index.summary.total.store.size.bytes | Total size of the index in bytes. | long | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | + +### Machine Learning Jobs + +If you have Machine Learning jobs, this data stream will interrogate the +[Machine Learning Anomaly Detection API](https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-apis.html) +and requires [Machine Learning](https://www.elastic.co/products/x-pack/machine-learning) to be enabled. + +An example event for `ml_job` looks as following: + +```json +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "elasticsearch": { + "cluster": { + "id": "8l_zoGznQRmtoX9iSC-goA", + "name": "docker-cluster" + }, + "ml": { + "job": { + "data_counts": { + "invalid_date_count": 0, + "processed_record_count": 1216 + }, + "forecasts_stats": { + "total": 1 + }, + "id": "low_request_rate", + "model_size": { + "memory_status": "ok" + }, + "state": "opened" + } + }, + "node": { + "id": "a14cf47ef7f2" + } + }, + "event": { + "dataset": "elasticsearch.ml.job", + "duration": 115000, + "module": "elasticsearch" + }, + "metricset": { + "name": "ml_job", + "period": 10000 + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.ml.job.data.invalid_date.count | The number of records with either a missing date field or a date that could not be parsed. | long | +| elasticsearch.ml.job.data_counts.invalid_date_count | | long | +| elasticsearch.ml.job.data_counts.processed_record_count | Processed data events. | long | +| elasticsearch.ml.job.forecasts_stats.total | | long | +| elasticsearch.ml.job.id | Unique ml job id. | keyword | +| elasticsearch.ml.job.model_size.memory_status | | keyword | +| elasticsearch.ml.job.state | Job state. | keyword | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | + +### Node + +`node` interrogates the +https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-info.html[Cluster API endpoint] of +Elasticsearch to get cluster nodes information. It only fetches the data from the `_local` node so it must +run on each Elasticsearch node. + +An example event for `node` looks as following: + +```json +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "27d29977-878e-4309-81ed-8788662503ad", + "ephemeral_id": "f8f510e7-9503-4e3d-af7f-da2992648d31", + "type": "metricbeat", + "version": "7.15.0" + }, + "elastic_agent": { + "id": "27d29977-878e-4309-81ed-8788662503ad", + "version": "7.15.0", + "snapshot": true + }, + "@timestamp": "2021-08-03T12:27:26.083Z", + "elasticsearch": { + "cluster": { + "name": "docker-cluster", + "id": "icut8oAwR--oCfUTlFaPMg" + }, + "node": { + "jvm": { + "memory": { + "heap": { + "init": { + "bytes": 1073741824 + }, + "max": { + "bytes": 1073741824 + } + }, + "nonheap": { + "init": { + "bytes": 7667712 + }, + "max": { + "bytes": 0 + } + } + }, + "version": "16.0.1" + }, + "process": { + "mlockall": false + }, + "name": "2b8824139b92", + "id": "saWHxJSZQF6VqGZvEb45Uw", + "version": "7.15.0" + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.node" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.24.0.7" + ], + "name": "docker-fleet-agent", + "id": "1292624d19b2cee1a317ad634c9a8358", + "mac": [ + "02:42:ac:18:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "node" + }, + "event": { + "duration": 9853150, + "agent_id_status": "verified", + "ingested": "2021-08-03T12:27:27.080460943Z", + "module": "elasticsearch", + "dataset": "elasticsearch.node" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.jvm.memory.heap.init.bytes | Heap init used by the JVM in bytes. | long | +| elasticsearch.node.jvm.memory.heap.max.bytes | Heap max used by the JVM in bytes. | long | +| elasticsearch.node.jvm.memory.nonheap.init.bytes | Non-Heap init used by the JVM in bytes. | long | +| elasticsearch.node.jvm.memory.nonheap.max.bytes | Non-Heap max used by the JVM in bytes. | long | +| elasticsearch.node.jvm.version | JVM version. | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | +| elasticsearch.node.process.mlockall | If process locked in memory. | boolean | +| elasticsearch.node.version | Node version. | keyword | + +### Node stats + +`node_stats` interrogates the +https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-stats.html[Cluster API endpoint] of +Elasticsearch to get the cluster nodes statistics. The data received is only for the local node so the Agent has +to be run on each Elasticsearch node. + +NOTE: The indices stats are node-specific. That means for example the total number of docs reported by all nodes together is not the total number of documents in all indices as there can also be replicas. + +An example event for `node_stats` looks as following: + +```json +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:47:15.407Z", + "elasticsearch": { + "node": { + "stats": { + "jvm": { + "mem": { + "pools": { + "young": { + "max": { + "bytes": 0 + }, + "used": { + "bytes": 33554432 + }, + "peak": { + "bytes": 633339904 + }, + "peak_max": { + "bytes": 0 + } + }, + "old": { + "max": { + "bytes": 1073741824 + }, + "used": { + "bytes": 248498176 + }, + "peak": { + "bytes": 371192832 + }, + "peak_max": { + "bytes": 1073741824 + } + }, + "survivor": { + "max": { + "bytes": 0 + }, + "peak": { + "bytes": 67829936 + }, + "peak_max": { + "bytes": 0 + }, + "used": { + "bytes": 3283184 + } + } + } + }, + "gc": { + "collectors": { + "young": { + "collection": { + "ms": 6100, + "count": 425 + } + }, + "old": { + "collection": { + "ms": 0, + "count": 0 + } + } + } + } + }, + "indices": { + "docs": { + "deleted": 7226, + "count": 50805 + }, + "store": { + "size": { + "bytes": 37101213 + } + }, + "segments": { + "memory": { + "bytes": 1805548 + }, + "count": 227 + } + }, + "fs": { + "summary": { + "total": { + "bytes": 958613114880 + }, + "available": { + "bytes": 261931741184 + }, + "free": { + "bytes": 310698074112 + } + } + } + }, + "name": "e7e895f7c41e", + "id": "6XuAxHXaRbeX6LUrxIfAxg" + }, + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q" + } + }, + "ecs": { + "version": "1.10.0" + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.node_stats" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "type": "linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "node_stats" + }, + "event": { + "duration": 32401229, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:47:16.373437564Z", + "module": "elasticsearch", + "dataset": "elasticsearch.node_stats" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | +| elasticsearch.node.stats.fs.io_stats.total.operations.count | | long | +| elasticsearch.node.stats.fs.io_stats.total.read.operations.count | | long | +| elasticsearch.node.stats.fs.io_stats.total.write.operations.count | | long | +| elasticsearch.node.stats.fs.summary.available.bytes | | long | +| elasticsearch.node.stats.fs.summary.free.bytes | | long | +| elasticsearch.node.stats.fs.summary.total.bytes | | long | +| elasticsearch.node.stats.fs.total.available_in_bytes | | long | +| elasticsearch.node.stats.fs.total.total_in_bytes | | long | +| elasticsearch.node.stats.indices.docs.count | Total number of existing documents. | long | +| elasticsearch.node.stats.indices.docs.deleted | Total number of deleted documents. | long | +| elasticsearch.node.stats.indices.fielddata.memory.bytes | | long | +| elasticsearch.node.stats.indices.indexing.index_time.ms | | long | +| elasticsearch.node.stats.indices.indexing.index_total.count | | long | +| elasticsearch.node.stats.indices.indexing.throttle_time.ms | | long | +| elasticsearch.node.stats.indices.query_cache.memory.bytes | | long | +| elasticsearch.node.stats.indices.request_cache.memory.bytes | | long | +| elasticsearch.node.stats.indices.search.query_time.ms | | long | +| elasticsearch.node.stats.indices.search.query_total.count | | long | +| elasticsearch.node.stats.indices.segments.count | Total number of segments. | long | +| elasticsearch.node.stats.indices.segments.doc_values.memory.bytes | | long | +| elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes | | long | +| elasticsearch.node.stats.indices.segments.index_writer.memory.bytes | | long | +| elasticsearch.node.stats.indices.segments.memory.bytes | Total size of segments in bytes. | long | +| elasticsearch.node.stats.indices.segments.norms.memory.bytes | | long | +| elasticsearch.node.stats.indices.segments.points.memory.bytes | | long | +| elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes | | long | +| elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes | | long | +| elasticsearch.node.stats.indices.segments.terms.memory.bytes | | long | +| elasticsearch.node.stats.indices.segments.version_map.memory.bytes | | long | +| elasticsearch.node.stats.indices.store.size.bytes | Total size of the store in bytes. | long | +| elasticsearch.node.stats.jvm.gc.collectors.old.collection.count | | long | +| elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms | | long | +| elasticsearch.node.stats.jvm.gc.collectors.young.collection.count | | long | +| elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms | | long | +| elasticsearch.node.stats.jvm.mem.heap.max.bytes | | long | +| elasticsearch.node.stats.jvm.mem.heap.used.bytes | | long | +| elasticsearch.node.stats.jvm.mem.heap.used.pct | | double | +| elasticsearch.node.stats.jvm.mem.pools.old.max.bytes | Max bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.old.peak.bytes | Peak bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.old.peak_max.bytes | Peak max bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.old.used.bytes | Used bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.survivor.max.bytes | Max bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.survivor.peak.bytes | Peak bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.survivor.peak_max.bytes | Peak max bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.survivor.used.bytes | Used bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.young.max.bytes | Max bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.young.peak.bytes | Peak bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.young.peak_max.bytes | Peak max bytes. | long | +| elasticsearch.node.stats.jvm.mem.pools.young.used.bytes | Used bytes. | long | +| elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us | | long | +| elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count | | long | +| elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns | | long | +| elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count | | long | +| elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns | | long | +| elasticsearch.node.stats.os.cgroup.memory.control_group | | keyword | +| elasticsearch.node.stats.os.cgroup.memory.limit.bytes | | long | +| elasticsearch.node.stats.os.cgroup.memory.usage.bytes | | long | +| elasticsearch.node.stats.os.cpu.load_avg.1m | | half_float | +| elasticsearch.node.stats.process.cpu.pct | | double | +| elasticsearch.node.stats.thread_pool.bulk.queue.count | | long | +| elasticsearch.node.stats.thread_pool.bulk.rejected.count | | long | +| elasticsearch.node.stats.thread_pool.get.queue.count | | long | +| elasticsearch.node.stats.thread_pool.get.rejected.count | | long | +| elasticsearch.node.stats.thread_pool.index.queue.count | | long | +| elasticsearch.node.stats.thread_pool.index.rejected.count | | long | +| elasticsearch.node.stats.thread_pool.search.queue.count | | long | +| elasticsearch.node.stats.thread_pool.search.rejected.count | | long | +| elasticsearch.node.stats.thread_pool.write.queue.count | | long | +| elasticsearch.node.stats.thread_pool.write.rejected.count | | long | + +### Pending tasks + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.pending_task.insert_order | Insert order | long | +| elasticsearch.cluster.pending_task.priority | Priority | keyword | +| elasticsearch.cluster.pending_task.source | Source. For example: put-mapping | keyword | +| elasticsearch.cluster.pending_task.time_in_queue.ms | Time in queue | long | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | + +# Shard + +`shard` interrogates the +https://www.elastic.co/guide/en/elasticsearch/reference/6.2/cluster-state.html[Cluster State API endpoint] to fetch +information about all shards. + +An example event for `shard` looks as following: + +```json +{ + "agent": { + "hostname": "docker-fleet-agent", + "name": "docker-fleet-agent", + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "type": "metricbeat", + "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", + "version": "7.14.0" + }, + "elastic_agent": { + "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "version": "7.14.0", + "snapshot": true + }, + "@timestamp": "2021-07-30T14:41:17.832Z", + "ecs": { + "version": "1.10.0" + }, + "elasticsearch": { + "node": { + "name": "6XuAxHXaRbeX6LUrxIfAxg" + }, + "cluster": { + "name": "docker-cluster", + "id": "bvF4SoDLQU-sdM3YY8JI8Q", + "state": { + "id": "mOYQ8E-ORnGSnnp9sB4BCw" + } + }, + "index": { + "name": ".async-search" + }, + "shard": { + "number": 0, + "relocating_node": {}, + "state": "STARTED", + "primary": true + } + }, + "service": { + "address": "http://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.shard" + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.11.10-arch1-1", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "type": "linux", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.18.0.7" + ], + "name": "docker-fleet-agent", + "id": "8979eb4aa312c3dccea3823dd92f92f5", + "mac": [ + "02:42:ac:12:00:07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "shard" + }, + "event": { + "duration": 4139652, + "agent_id_status": "verified", + "ingested": "2021-07-30T14:41:18.844042490Z", + "module": "elasticsearch", + "dataset": "elasticsearch.shard" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | +| elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | +| elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.index.name | | keyword | +| elasticsearch.node.id | Node ID | keyword | +| elasticsearch.node.master | Is the node the master node? | boolean | +| elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | +| elasticsearch.node.name | Node name. | keyword | +| elasticsearch.shard.number | The number of this shard. | long | +| elasticsearch.shard.primary | True if this is the primary shard. | boolean | +| elasticsearch.shard.relocating_node.id | The node the shard was relocated from. It has the exact same value than relocating_node.name for compatibility purposes. | keyword | +| elasticsearch.shard.relocating_node.name | The node the shard was relocated from. | keyword | +| elasticsearch.shard.source_node.name | | keyword | +| elasticsearch.shard.source_node.uuid | | keyword | +| elasticsearch.shard.state | The state of this shard. | keyword | diff --git a/packages/elasticsearch/1.1.0-preview1/img/logo_elasticsearch.svg b/packages/elasticsearch/1.1.0-preview1/img/logo_elasticsearch.svg new file mode 100755 index 0000000000..20a620d162 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/img/logo_elasticsearch.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/packages/elasticsearch/1.1.0-preview1/manifest.yml b/packages/elasticsearch/1.1.0-preview1/manifest.yml new file mode 100755 index 0000000000..9f6ed9a508 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/manifest.yml @@ -0,0 +1,61 @@ +name: elasticsearch +title: Elasticsearch +version: 1.0.0 +release: experimental +description: Elasticsearch Integration +type: integration +icons: + - src: /img/logo_elasticsearch.svg + title: logo elasticsearch + size: 32x32 + type: image/svg+xml +format_version: 1.0.0 +license: basic +categories: ["elastic_stack", "datastore"] +conditions: + kibana.version: ^8.5.0 +policy_templates: + - name: elasticsearch + title: Elasticsearch logs and metrics + description: Collect logs and metrics from Elasticsearch instances + inputs: + - type: logfile + title: Collect Elasticsearch logs + description: "Collecting audit, deprecation, gc, server and slowlog logs from Elasticsearch instances (input: logfile)" + - type: elasticsearch/metrics + title: Collect Elasticsearch metrics + description: Collect Elasticsearch metrics about indices, CCR, cluster stats, machine learning or node statistics + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - http://localhost:9200 + - name: username + type: text + title: Username + description: Use when connecting to elasticsearch + multi: false + required: false + show_user: false + - name: password + type: password + title: Password + description: Use when connecting to elasticsearch + multi: false + required: false + show_user: false + - name: scope + type: text + title: Scope + description: >- + Options are `node` or `cluster`. By default, scope is set to node and each entry in the hosts list indicates a distinct node in an Elasticsearch cluster. If the scope is set to cluster then each entry in the hosts list indicates a single endpoint for a distinct Elasticsearch cluster (for example, a load-balancing proxy fronting the cluster). Cluster should be used if the cluster has dedicated master nodes, and configure the endpoint in the hosts list not to direct requests to the dedicated master nodes. + multi: false + required: true + show_user: false + default: node +owner: + github: elastic/infra-monitoring-ui From 5bc402465e4ad8ec359555bfd78605ecdd9bc2fc Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Wed, 19 Oct 2022 01:43:07 +0000 Subject: [PATCH 2/2] Copy over local package sources --- .../1.1.0-preview1/changelog.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 8 +- .../data_stream/audit/fields/ecs.yml | 53 + .../data_stream/audit/fields/fields.yml | 4 + .../audit/fields/package-fields.yml | 6 + .../data_stream/audit/sample_event.json | 113 ++ .../data_stream/ccr/fields/ecs.yml | 3 + .../data_stream/ccr/sample_event.json | 155 ++ .../data_stream/cluster_stats/fields/ecs.yml | 3 + .../elasticsearch/ingest_pipeline/default.yml | 8 +- .../ingest_pipeline/pipeline-json.yml | 4 + .../data_stream/deprecation/fields/ecs.yml | 45 + .../deprecation/fields/package-fields.yml | 10 + .../data_stream/deprecation/sample_event.json | 98 ++ .../data_stream/enrich/fields/ecs.yml | 3 + .../elasticsearch/ingest_pipeline/default.yml | 10 +- .../data_stream/gc/fields/ecs.yml | 25 + .../data_stream/gc/fields/package-fields.yml | 4 + .../data_stream/gc/sample_event.json | 77 + .../data_stream/index/fields/ecs.yml | 3 + .../data_stream/index/fields/fields.yml | 15 + .../data_stream/index/sample_event.json | 161 +- .../data_stream/index_recovery/fields/ecs.yml | 3 + .../data_stream/index_summary/fields/ecs.yml | 3 + .../index_summary/sample_event.json | 174 +- .../data_stream/ml_job/fields/ecs.yml | 3 + .../data_stream/node/fields/ecs.yml | 3 + .../data_stream/node/sample_event.json | 96 +- .../data_stream/node_stats/fields/ecs.yml | 3 + .../data_stream/node_stats/fields/fields.yml | 13 + .../data_stream/node_stats/sample_event.json | 375 +++-- .../data_stream/pending_tasks/fields/ecs.yml | 3 + .../pending_tasks/fields/fields.yml | 2 +- .../pending_tasks/sample_event.json | 71 +- .../elasticsearch/ingest_pipeline/default.yml | 21 +- .../ingest_pipeline/pipeline-json.yml | 112 +- .../data_stream/server/fields/ecs.yml | 45 + .../data_stream/server/fields/fields.yml | 2 + .../server/fields/package-fields.yml | 8 + .../data_stream/server/sample_event.json | 82 + .../data_stream/shard/fields/ecs.yml | 3 + .../data_stream/shard/sample_event.json | 105 +- .../elasticsearch/ingest_pipeline/default.yml | 6 +- .../ingest_pipeline/pipeline-json.yml | 3 + .../data_stream/slowlog/fields/ecs.yml | 45 + .../data_stream/slowlog/fields/fields.yml | 2 +- .../slowlog/fields/package-fields.yml | 4 + .../data_stream/slowlog/sample_event.json | 98 ++ .../1.1.0-preview1/docs/README.md | 1415 +++++++++++++---- .../1.1.0-preview1/img/logo_elasticsearch.svg | 10 +- .../elasticsearch/1.1.0-preview1/manifest.yml | 3 +- 51 files changed, 2686 insertions(+), 837 deletions(-) create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/audit/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/ccr/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/gc/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/server/sample_event.json create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/ecs.yml create mode 100755 packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/sample_event.json diff --git a/packages/elasticsearch/1.1.0-preview1/changelog.yml b/packages/elasticsearch/1.1.0-preview1/changelog.yml index f6d2302dff..cb99a05482 100755 --- a/packages/elasticsearch/1.1.0-preview1/changelog.yml +++ b/packages/elasticsearch/1.1.0-preview1/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "1.0.0" +- version: "1.1.0-preview1" changes: - description: Suffix `stack_monitoring` to the datasets type: enhancement diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 0cca384c70..ba87b176b1 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -4,9 +4,6 @@ processors: - set: field: event.ingested value: "{{_ingest.timestamp}}" - - set: - copy_from: "@timestamp" - field: event.created - grok: field: message patterns: @@ -17,7 +14,10 @@ processors: if: ctx.first_char != '{' - pipeline: if: ctx.first_char == '{' - name: '{< IngestPipeline "pipeline-json" >}' + name: '{{ IngestPipeline "pipeline-json" }}' + - set: + copy_from: "@timestamp" + field: event.created - set: field: event.kind value: event diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/ecs.yml index 7b7a839904..cb309753a2 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/ecs.yml @@ -1,3 +1,8 @@ +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword - description: The full HTTP request body. multi_fields: - name: text @@ -22,3 +27,51 @@ type: match_only_text name: user.name type: keyword +- description: |- + A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. + The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. + name: http.request.id + type: keyword +- description: |- + HTTP request method. + Prior to ECS 1.6.0 the following guidance was provided: + "The field value must be normalized to lowercase for querying." + As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 + name: http.request.method + type: keyword +- description: |- + Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. + If the event wasn't read from a log file, do not populate this field. + name: log.file.path + type: keyword +- description: |- + Original log level of the log event. + If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). + Some examples are `warn`, `err`, `i`, `informational`. + name: log.level + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. + Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. + name: source.address + type: keyword +- description: Port of the source. + name: source.port + type: long +- description: |- + Unique identifier of the trace. + A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. + name: trace.id + type: keyword +- description: |- + For log events the message field contains the log message, optimized for viewing in a log viewer. + For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. + If multiple messages exist, they can be combined into one message. + name: message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/fields.yml index b50e865553..7d2bbef22f 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/fields.yml @@ -1,6 +1,10 @@ - name: elasticsearch.audit type: group fields: + - name: authentication.type + type: keyword + - name: opaque_id + type: keyword - name: layer type: keyword description: 'The layer from which this event originated: rest, transport or ip_filter' diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/package-fields.yml index 7ef974b1ab..2202a7b4af 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/package-fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/fields/package-fields.yml @@ -1,3 +1,9 @@ +- name: input.type + type: keyword +- name: log.offset + type: long +- name: related.user + type: keyword - name: elasticsearch type: group fields: diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/audit/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/sample_event.json new file mode 100755 index 0000000000..ff55f9e775 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/audit/sample_event.json @@ -0,0 +1,113 @@ +{ + "@timestamp": "2022-09-01T19:20:17.967Z", + "agent": { + "ephemeral_id": "ec83bfa3-8e61-430e-91ca-dc784bfa56c0", + "id": "97025ce1-28a3-4aeb-926b-ed68301fc4d2", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.5.0" + }, + "data_stream": { + "dataset": "elasticsearch.audit", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.10.0" + }, + "elastic_agent": { + "id": "97025ce1-28a3-4aeb-926b-ed68301fc4d2", + "snapshot": true, + "version": "8.5.0" + }, + "elasticsearch": { + "audit": { + "action": "cluster:monitor/main", + "authentication.type": "REALM", + "cluster": {}, + "event": {}, + "layer": "transport", + "origin": {}, + "origin.type": "rest", + "request": { + "id": "YCHBXylbRnSC3Vc8-f3sIA" + }, + "request.name": "MainRequest", + "user": {}, + "user.realm": "reserved", + "user.roles": [ + "superuser" + ] + }, + "cluster": { + "uuid": "wkVNYOctQ8mbbp1EkrFjKw" + }, + "node": { + "id": "VdwTr-luTomz8dDpOp2OJQ" + } + }, + "event": { + "action": "access_granted", + "agent_id_status": "verified", + "category": "database", + "created": "2022-09-01T19:20:39.899Z", + "dataset": "elasticsearch.audit", + "ingested": "2022-09-01T19:20:43Z", + "kind": "event", + "outcome": "success" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "VdwTr-luTomz8dDpOp2OJQ", + "ip": [ + "172.21.0.7" + ], + "mac": [ + "02:42:ac:15:00:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "http": { + "request": { + "id": "YCHBXylbRnSC3Vc8-f3sIA" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/elasticsearch_audit.json" + }, + "level": "info", + "offset": 0 + }, + "message": "{\"type\":\"audit\", \"timestamp\":\"2022-09-01T19:20:17,967+0000\", \"cluster.uuid\":\"wkVNYOctQ8mbbp1EkrFjKw\", \"node.id\":\"VdwTr-luTomz8dDpOp2OJQ\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53716\", \"request.id\":\"YCHBXylbRnSC3Vc8-f3sIA\", \"action\":\"cluster:monitor/main\", \"request.name\":\"MainRequest\"}", + "related": { + "user": [ + "elastic" + ] + }, + "service": { + "type": "elasticsearch" + }, + "source": { + "address": "127.0.0.1:53716", + "ip": "127.0.0.1", + "port": 53716 + }, + "user": { + "name": "elastic" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/sample_event.json new file mode 100755 index 0000000000..9826cded86 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ccr/sample_event.json @@ -0,0 +1,155 @@ +{ + "agent": { + "name": "docker-fleet-agent", + "id": "071bc382-9090-4d81-9181-98472331a5c2", + "type": "metricbeat", + "ephemeral_id": "3d4a3c4d-ceab-4935-a8fc-368770f6172f", + "version": "8.5.0" + }, + "@timestamp": "2022-09-05T00:30:28.871Z", + "ecs": { + "version": "8.0.0" + }, + "elasticsearch": { + "cluster": { + "name": "elasticsearch", + "id": "_WZA8YdZQr2tet-oqyQPGA" + }, + "ccr": { + "leader": { + "index": "foo-bar", + "max_seq_no": 47, + "global_checkpoint": 47 + }, + "follower": { + "operations": { + "read": { + "count": 48 + } + }, + "time_since_last_read": { + "ms": 29356 + }, + "index": "foo-bar-follower", + "settings_version": 1, + "shard": { + "number": 0 + }, + "aliases_version": 1, + "max_seq_no": 47, + "global_checkpoint": 47, + "operations_written": 48 + }, + "read_exceptions": [], + "bytes_read": 3600, + "requests": { + "outstanding": { + "read": { + "count": 1 + }, + "write": { + "count": 0 + } + }, + "failed": { + "read": { + "count": 0 + }, + "write": { + "count": 0 + } + }, + "successful": { + "read": { + "count": 48 + }, + "write": { + "count": 48 + } + } + }, + "auto_follow": { + "success": { + "follow_indices": { + "count": 0 + } + }, + "failed": { + "follow_indices": { + "count": 0 + }, + "remote_cluster_state_requests": { + "count": 0 + } + } + }, + "total_time": { + "read": { + "ms": 57001, + "remote_exec": { + "ms": 56908 + } + }, + "write": { + "ms": 189 + } + }, + "write_buffer": { + "size": { + "bytes": 0 + }, + "operation": { + "count": 0 + } + } + } + }, + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.ccr" + }, + "elastic_agent": { + "id": "071bc382-9090-4d81-9181-98472331a5c2", + "version": "8.5.0", + "snapshot": true + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.10.47-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.5 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": true, + "ip": [ + "172.28.0.7" + ], + "name": "docker-fleet-agent", + "id": "1fcafd903db54c7f9f085ed0c657a92a", + "mac": [ + "02-42-AC-1C-00-07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "ccr" + }, + "event": { + "duration": 41217684, + "agent_id_status": "verified", + "ingested": "2022-09-05T00:30:29Z", + "module": "elasticsearch", + "dataset": "elasticsearch.ccr" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/cluster_stats/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/default.yml index 721368a023..e3ba8a3e36 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/default.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/default.yml @@ -4,9 +4,6 @@ processors: - set: field: event.ingested value: "{{_ingest.timestamp}}" - - set: - copy_from: "@timestamp" - field: event.created - grok: field: message patterns: @@ -17,7 +14,10 @@ processors: if: ctx.first_char != '{' - pipeline: if: ctx.first_char == '{' - name: '{< IngestPipeline "pipeline-json" >}' + name: '{{ IngestPipeline "pipeline-json" }}' + - set: + copy_from: "@timestamp" + field: event.created - set: field: event.kind value: event diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/pipeline-json.yml index 84ae736955..49260eb94d 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/pipeline-json.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -18,6 +18,10 @@ processors: - set: field: event.dataset value: elasticsearch.deprecation + # datastream as well for consistency + - set: + field: data_stream.dataset + value: elasticsearch.deprecation - set: field: service.type value: 'elasticsearch' diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/ecs.yml new file mode 100755 index 0000000000..d8ec6f1dce --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/ecs.yml @@ -0,0 +1,45 @@ +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. + If the event wasn't read from a log file, do not populate this field. + name: log.file.path + type: keyword +- description: The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. + name: log.logger + type: keyword +- description: |- + Original log level of the log event. + If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). + Some examples are `warn`, `err`, `i`, `informational`. + name: log.level + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Unique identifier of the trace. + A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. + name: trace.id + type: keyword +- description: Thread name. + name: process.thread.name + type: keyword +- description: |- + For log events the message field contains the log message, optimized for viewing in a log viewer. + For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. + If multiple messages exist, they can be combined into one message. + name: message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/package-fields.yml index 7ef974b1ab..82a7116868 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/package-fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/fields/package-fields.yml @@ -1,6 +1,16 @@ +- name: input.type + type: keyword +- name: log.offset + type: long - name: elasticsearch type: group fields: + - name: http.request.x_opaque_id + type: keyword + - name: elastic_product_origin + type: keyword + - name: event.category + type: keyword - name: component type: keyword description: Elasticsearch component from where the log event originated diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/sample_event.json new file mode 100755 index 0000000000..4ea5d73dd1 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/deprecation/sample_event.json @@ -0,0 +1,98 @@ +{ + "@timestamp": "2022-09-01T22:45:51.704Z", + "agent": { + "ephemeral_id": "db6ae9b2-aa56-456d-abd2-81f4b1c2504a", + "id": "f71a13d2-b5f8-4fdc-b6d8-df43765317ee", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.deprecation", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.2.0" + }, + "elastic_agent": { + "id": "f71a13d2-b5f8-4fdc-b6d8-df43765317ee", + "snapshot": false, + "version": "8.3.2" + }, + "elasticsearch": { + "cluster": { + "name": "elasticsearch", + "uuid": "GwNzFcCUQr-hpbWpMUHtgA" + }, + "elastic_product_origin": "", + "event": { + "category": "settings" + }, + "http": { + "request": { + "x_opaque_id": "myAppId" + } + }, + "node": { + "id": "qKcx_J71RvaI5YdYZspM9A", + "name": "5cd111df3d45" + } + }, + "event": { + "agent_id_status": "verified", + "category": "database", + "code": "index.data_path", + "created": "2022-09-01T22:46:14.090Z", + "dataset": "elasticsearch.deprecation", + "ingested": "2022-09-01T22:46:17Z", + "kind": "event", + "type": "info" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "qKcx_J71RvaI5YdYZspM9A", + "ip": [ + "172.31.0.7" + ], + "mac": [ + "02:42:ac:1f:00:07" + ], + "name": "5cd111df3d45", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/elasticsearch_deprecation.json" + }, + "level": "WARN", + "logger": "org.elasticsearch.deprecation.common.settings.Settings", + "offset": 0 + }, + "message": "[index.data_path] setting was deprecated in Elasticsearch and will be removed in a future release.", + "process": { + "thread": { + "name": "elasticsearch[5cd111df3d45][transport_worker][T#2]" + } + }, + "service": { + "name": "ES_ECS", + "type": "elasticsearch" + }, + "trace": { + "id": "0af7651916cd43dd8448eb211c80319c" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/enrich/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/elasticsearch/ingest_pipeline/default.yml index 98d32286a5..e12fca047b 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/elasticsearch/ingest_pipeline/default.yml @@ -38,14 +38,18 @@ processors: '\[Times: user=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.user_sec} sys=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.sys_sec}, real=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.real_sec} secs\]' - - set: - copy_from: "@timestamp" - field: event.created + - convert: + field: process.pid + type: long + ignore_missing: true - date: field: timestamp target_field: "@timestamp" formats: - ISO8601 + - set: + copy_from: "@timestamp" + field: event.created - remove: field: timestamp - set: diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/ecs.yml new file mode 100755 index 0000000000..d2adb2d2c5 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/ecs.yml @@ -0,0 +1,25 @@ +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. + If the event wasn't read from a log file, do not populate this field. + name: log.file.path + type: keyword +- description: |- + For log events the message field contains the log message, optimized for viewing in a log viewer. + For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. + If multiple messages exist, they can be combined into one message. + name: message + type: match_only_text +- description: Process id. + name: process.pid + type: long +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/package-fields.yml index 7ef974b1ab..da4033d9ba 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/package-fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/fields/package-fields.yml @@ -1,3 +1,7 @@ +- name: input.type + type: keyword +- name: log.offset + type: long - name: elasticsearch type: group fields: diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/gc/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/sample_event.json new file mode 100755 index 0000000000..6f0da90427 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/gc/sample_event.json @@ -0,0 +1,77 @@ +{ + "@timestamp": "2022-09-01T21:25:31.186Z", + "agent": { + "ephemeral_id": "ede3fb40-453f-4b6a-8e5a-529a4f11cebb", + "id": "4cabf8b7-fdae-4fc8-bd56-cbdad183bdd9", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.5.0" + }, + "data_stream": { + "dataset": "elasticsearch.gc", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.10.0" + }, + "elastic_agent": { + "id": "4cabf8b7-fdae-4fc8-bd56-cbdad183bdd9", + "snapshot": true, + "version": "8.5.0" + }, + "elasticsearch": { + "gc": { + "tags": [ + "gc", + "init" + ] + } + }, + "event": { + "agent_id_status": "verified", + "category": "database", + "created": "2022-09-01T21:26:15.164Z", + "dataset": "elasticsearch.gc", + "ingested": "2022-09-01T21:26:16Z", + "kind": "metric", + "type": "info" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "ip": [ + "192.168.144.7" + ], + "mac": [ + "02:42:c0:a8:90:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/gc.log" + }, + "offset": 0 + }, + "message": "CardTable entry size: 512", + "process": { + "pid": 215 + }, + "service": { + "type": "elasticsearch" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/fields.yml index 39ecdae58e..7d8a007097 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/fields/fields.yml @@ -11,6 +11,8 @@ fields: - name: total type: long + - name: primaries + type: long - name: uuid type: keyword - name: status @@ -98,6 +100,19 @@ - name: total type: group fields: + - name: bulk + type: group + fields: + - name: avg_size_in_bytes + type: long + - name: total_size_in_bytes + type: long + - name: avg_time_in_millis + type: long + - name: total_time_in_millis + type: long + - name: total_operations + type: long - name: docs.count type: long description: | diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/index/sample_event.json index 5702f5392d..ab257a2a78 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/index/sample_event.json +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index/sample_event.json @@ -1,88 +1,143 @@ { + "@timestamp": "2022-09-02T14:06:12.353Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "5c8415cd-4402-4ddf-b627-b13790bc3197", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", "type": "metricbeat", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", - "version": "7.14.0" + "version": "8.3.2" }, - "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true + "data_stream": { + "dataset": "elasticsearch.index", + "namespace": "ep", + "type": "metrics" }, - "@timestamp": "2021-07-30T14:46:47.831Z", "ecs": { - "version": "1.10.0" + "version": "8.0.0" + }, + "elastic_agent": { + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, "elasticsearch": { "cluster": { - "name": "docker-cluster", - "id": "bvF4SoDLQU-sdM3YY8JI8Q" + "id": "zv3a1lJUQoK10VDNC6J0qA", + "name": "elasticsearch" }, "index": { - "total": { + "hidden": false, + "name": "testindex2", + "primaries": { "docs": { - "deleted": 0, - "count": 13267 + "count": 0 + }, + "indexing": { + "index_time_in_millis": 0, + "index_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "total_size_in_bytes": 0 + }, + "refresh": { + "total_time_in_millis": 0 + }, + "segments": { + "count": 0 }, "store": { - "size": { - "bytes": 1490775 - } + "size_in_bytes": 675 + } + }, + "shards": { + "primaries": 3, + "total": 6 + }, + "status": "yellow", + "total": { + "bulk": { + "avg_size_in_bytes": 0, + "avg_time_in_millis": 0, + "total_operations": 0, + "total_size_in_bytes": 0, + "total_time_in_millis": 0 + }, + "docs": { + "count": 0 + }, + "fielddata": { + "memory_size_in_bytes": 0 + }, + "indexing": { + "index_time_in_millis": 0, + "index_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "total_size_in_bytes": 0 + }, + "refresh": { + "total_time_in_millis": 0 + }, + "search": { + "query_time_in_millis": 0, + "query_total": 0 }, "segments": { - "memory": { - "bytes": 50388 - }, - "count": 5 + "count": 0, + "doc_values_memory_in_bytes": 0, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 0, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 0, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 675 } }, - "name": ".ds-metrics-elasticsearch.shard-default-2021.07.30-000001" + "uuid": "lH2NeM70TlKGEB11uUxiuA" } }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.index" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.index", + "duration": 34210900, + "ingested": "2022-09-02T14:06:13Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", "mac": [ "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "index" + "name": "index", + "period": 10000 }, - "event": { - "duration": 14394992, - "agent_id_status": "verified", - "ingested": "2021-07-30T14:46:48.854674866Z", - "module": "elasticsearch", - "dataset": "elasticsearch.index" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "type": "elasticsearch" } } \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_recovery/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/sample_event.json index cd1c357bb8..2b30fbb958 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/sample_event.json +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/index_summary/sample_event.json @@ -1,106 +1,162 @@ { + "@timestamp": "2022-09-02T14:23:38.078Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "5dcbe5f9-d61d-4931-b4f3-a334e8e999b2", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", "type": "metricbeat", - "version": "7.14.0" + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.index_summary", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, - "@timestamp": "2021-07-30T14:47:15.391Z", "elasticsearch": { "cluster": { - "name": "docker-cluster", - "id": "bvF4SoDLQU-sdM3YY8JI8Q" + "id": "zZUl__19TuWgxPiewmnJ3Q", + "name": "elasticsearch" }, "index": { "summary": { "primaries": { + "bulk": { + "operations": { + "count": 3 + }, + "size": { + "bytes": 45 + }, + "time": { + "avg": { + "bytes": 4 + } + } + }, "docs": { - "deleted": 7226, - "count": 50723 + "count": 3, + "deleted": 0 }, - "store": { - "size": { - "bytes": 36769186 + "indexing": { + "index": { + "count": 3, + "time": { + "ms": 14 + } + } + }, + "search": { + "query": { + "count": 9, + "time": { + "ms": 20 + } } }, "segments": { + "count": 3, "memory": { - "bytes": 1790592 - }, - "count": 222 + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 8466 + } } }, "total": { + "bulk": { + "operations": { + "count": 3 + }, + "size": { + "bytes": 45 + }, + "time": { + "avg": { + "bytes": 4 + } + } + }, "docs": { - "deleted": 7226, - "count": 50723 + "count": 3, + "deleted": 0 }, - "store": { - "size": { - "bytes": 36769186 + "indexing": { + "index": { + "count": 3, + "time": { + "ms": 14 + } + } + }, + "search": { + "query": { + "count": 9, + "time": { + "ms": 20 + } } }, "segments": { + "count": 3, "memory": { - "bytes": 1790592 - }, - "count": 222 + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 8466 + } } } } } }, - "ecs": { - "version": "1.10.0" - }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.index_summary" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.index_summary", + "duration": 32732300, + "ingested": "2022-09-02T14:23:39Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", "mac": [ "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "index_summary" + "name": "index_summary", + "period": 10000 }, - "event": { - "duration": 12151260, - "agent_id_status": "verified", - "ingested": "2021-07-30T14:47:16.373343461Z", - "module": "elasticsearch", - "dataset": "elasticsearch.index_summary" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "name": "elasticsearch", + "type": "elasticsearch" } } \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/ml_job/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/node/sample_event.json index 116e96c809..bf2f4f7e9f 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/node/sample_event.json +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node/sample_event.json @@ -1,24 +1,32 @@ { + "@timestamp": "2022-09-02T14:13:34.927Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "47946444-4c3a-4915-91dd-bf515aba9740", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "27d29977-878e-4309-81ed-8788662503ad", - "ephemeral_id": "f8f510e7-9503-4e3d-af7f-da2992648d31", "type": "metricbeat", - "version": "7.15.0" + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.node", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "27d29977-878e-4309-81ed-8788662503ad", - "version": "7.15.0", - "snapshot": true + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, - "@timestamp": "2021-08-03T12:27:26.083Z", "elasticsearch": { "cluster": { - "name": "docker-cluster", - "id": "icut8oAwR--oCfUTlFaPMg" + "id": "ziL93dUTRmGy5hsfhhq3Ww", + "name": "elasticsearch" }, "node": { + "id": "3nCEJ8F6SCuBH_c_YJNQSA", "jvm": { "memory": { "heap": { @@ -38,60 +46,50 @@ } } }, - "version": "16.0.1" + "version": "18.0.2" }, + "name": "1a6b5d803000", "process": { "mlockall": false }, - "name": "2b8824139b92", - "id": "saWHxJSZQF6VqGZvEb45Uw", - "version": "7.15.0" + "version": "8.5.0" } }, - "ecs": { - "version": "1.10.0" - }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.node" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.node", + "duration": 18259400, + "ingested": "2022-09-02T14:13:35Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ - "172.24.0.7" + "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "1292624d19b2cee1a317ad634c9a8358", "mac": [ - "02:42:ac:18:00:07" + "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "node" + "name": "node", + "period": 10000 }, - "event": { - "duration": 9853150, - "agent_id_status": "verified", - "ingested": "2021-08-03T12:27:27.080460943Z", - "module": "elasticsearch", - "dataset": "elasticsearch.node" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "name": "elasticsearch", + "type": "elasticsearch" } } \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/fields.yml index 4f31817122..f1c30b622e 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/fields/fields.yml @@ -5,6 +5,19 @@ - name: indices type: group fields: + - name: bulk + type: group + fields: + - name: avg_size.bytes + type: long + - name: avg_time.ms + type: long + - name: operations.total.count + type: long + - name: total_size.bytes + type: long + - name: total_time.ms + type: long - name: docs.count type: long description: | diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/sample_event.json index fc1542deec..509fdf09d7 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/sample_event.json +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/node_stats/sample_event.json @@ -1,168 +1,305 @@ { + "@timestamp": "2022-09-02T14:32:24.121Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "5d429743-0cf8-44a9-afb4-7523cf960d76", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", "type": "metricbeat", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", - "version": "7.14.0" + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.node_stats", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, - "@timestamp": "2021-07-30T14:47:15.407Z", "elasticsearch": { + "cluster": { + "id": "PdQA6zKZQaK1LAvajgbnug", + "name": "elasticsearch" + }, "node": { + "id": "vnPGsgkoQ5-kwzmE6DOjOQ", + "master": true, + "mlockall": false, + "name": "be467614bdb0", "stats": { - "jvm": { - "mem": { - "pools": { - "young": { - "max": { - "bytes": 0 - }, - "used": { - "bytes": 33554432 - }, - "peak": { - "bytes": 633339904 - }, - "peak_max": { - "bytes": 0 - } - }, - "old": { - "max": { - "bytes": 1073741824 - }, - "used": { - "bytes": 248498176 - }, - "peak": { - "bytes": 371192832 - }, - "peak_max": { - "bytes": 1073741824 - } - }, - "survivor": { - "max": { - "bytes": 0 - }, - "peak": { - "bytes": 67829936 - }, - "peak_max": { - "bytes": 0 - }, - "used": { - "bytes": 3283184 - } + "fs": { + "io_stats": {}, + "summary": { + "available": { + "bytes": 36166852608 + }, + "free": { + "bytes": 46061998080 + }, + "total": { + "bytes": 193393164288 + } + }, + "total": { + "available_in_bytes": 36166852608, + "total_in_bytes": 193393164288 + } + }, + "indices": { + "bulk": { + "avg_size": { + "bytes": 139 + }, + "avg_time": { + "ms": 4 + }, + "operations": { + "total": { + "count": 6 } + }, + "total_size": { + "bytes": 5303 + }, + "total_time": { + "ms": 175 + } + }, + "docs": { + "count": 11, + "deleted": 0 + }, + "fielddata": { + "memory": { + "bytes": 0 + } + }, + "indexing": { + "index_time": { + "ms": 31 + }, + "index_total": { + "count": 11 + }, + "throttle_time": { + "ms": 0 } }, + "query_cache": { + "memory": { + "bytes": 0 + } + }, + "request_cache": { + "memory": { + "bytes": 0 + } + }, + "search": { + "query_time": { + "ms": 19 + }, + "query_total": { + "count": 9 + } + }, + "segments": { + "count": 6, + "doc_values": { + "memory": { + "bytes": 0 + } + }, + "fixed_bit_set": { + "memory": { + "bytes": 0 + } + }, + "index_writer": { + "memory": { + "bytes": 0 + } + }, + "memory": { + "bytes": 0 + }, + "norms": { + "memory": { + "bytes": 0 + } + }, + "points": { + "memory": { + "bytes": 0 + } + }, + "stored_fields": { + "memory": { + "bytes": 0 + } + }, + "term_vectors": { + "memory": { + "bytes": 0 + } + }, + "terms": { + "memory": { + "bytes": 0 + } + }, + "version_map": { + "memory": { + "bytes": 0 + } + } + }, + "store": { + "size": { + "bytes": 40643 + } + } + }, + "jvm": { "gc": { "collectors": { - "young": { + "old": { "collection": { - "ms": 6100, - "count": 425 + "count": 0, + "ms": 0 } }, - "old": { + "young": { "collection": { - "ms": 0, - "count": 0 + "count": 9, + "ms": 217 } } } + }, + "mem": { + "heap": { + "max": { + "bytes": 1073741824 + }, + "used": { + "bytes": 400155760, + "pct": 37 + } + } } }, - "indices": { - "docs": { - "deleted": 7226, - "count": 50805 - }, - "store": { - "size": { - "bytes": 37101213 + "os": { + "cgroup": { + "cpu": { + "cfs": { + "quota": { + "us": -1 + } + }, + "stat": { + "elapsed_periods": { + "count": 0 + }, + "times_throttled": { + "count": 0 + } + } + }, + "cpuacct": { + "usage": { + "ns": 56233628308 + } + }, + "memory": { + "control_group": "/", + "limit": { + "bytes": "9223372036854771712" + }, + "usage": { + "bytes": "1536434176" + } } }, - "segments": { - "memory": { - "bytes": 1805548 - }, - "count": 227 + "cpu": { + "load_avg": { + "1m": 1.53 + } } }, - "fs": { - "summary": { - "total": { - "bytes": 958613114880 + "process": { + "cpu": { + "pct": 1 + } + }, + "thread_pool": { + "get": { + "queue": { + "count": 0 }, - "available": { - "bytes": 261931741184 + "rejected": { + "count": 0 + } + }, + "search": { + "queue": { + "count": 0 }, - "free": { - "bytes": 310698074112 + "rejected": { + "count": 0 + } + }, + "write": { + "queue": { + "count": 0 + }, + "rejected": { + "count": 0 } } } - }, - "name": "e7e895f7c41e", - "id": "6XuAxHXaRbeX6LUrxIfAxg" - }, - "cluster": { - "name": "docker-cluster", - "id": "bvF4SoDLQU-sdM3YY8JI8Q" + } } }, - "ecs": { - "version": "1.10.0" - }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.node_stats" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.node_stats", + "duration": 34932600, + "ingested": "2022-09-02T14:32:25Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", "mac": [ "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "node_stats" + "name": "node_stats", + "period": 10000 }, - "event": { - "duration": 32401229, - "agent_id_status": "verified", - "ingested": "2021-07-30T14:47:16.373437564Z", - "module": "elasticsearch", - "dataset": "elasticsearch.node_stats" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "name": "elasticsearch", + "type": "elasticsearch" } } \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/fields.yml index 324d4327d9..d21b1fabc2 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/fields/fields.yml @@ -1,4 +1,4 @@ -- name: elasticsearch.cluster.pending_task +- name: elasticsearch.pending_tasks type: group release: ga fields: diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/sample_event.json index 7bf7e262ad..05054fec45 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/sample_event.json +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/pending_tasks/sample_event.json @@ -1,74 +1,73 @@ { "agent": { - "hostname": "docker-fleet-agent", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", + "id": "f11de143-c31c-49a2-8756-83697dbabe0f", + "ephemeral_id": "3469da57-3138-4702-abc6-8b95e081fc12", "type": "metricbeat", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", - "version": "7.14.0" - }, - "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true - }, - "@timestamp": "2021-07-30T14:41:17.832Z", - "ecs": { - "version": "1.10.0" + "version": "8.5.0" }, + "@timestamp": "2022-09-21T16:00:34.116Z", "elasticsearch": { "cluster": { - "id": "3LbUkLkURz--FR-YO0wLNA", - "name": "es1", - "pending_task": { - "insert_order": 47, - "priority": "HIGH", - "source": "put-mapping", - "time_in_queue.ms": 34 - } + "name": "elasticsearch", + "id": "N9ZLPL5RQHS67eZBrujPYg" + }, + "pending_tasks": { + "time_in_queue.ms": 50, + "source": "create-index [foo-bar-1663776034], cause [api]", + "priority": "URGENT", + "insert_order": 3272 } }, + "ecs": { + "version": "8.0.0" + }, "service": { - "address": "http://elasticsearch:9200", + "address": "https://elasticsearch:9200", "name": "elasticsearch", "type": "elasticsearch" }, "data_stream": { "namespace": "default", "type": "metrics", - "dataset": "elasticsearch.pending_task" + "dataset": "elasticsearch.stack_monitoring.pending_tasks" + }, + "elastic_agent": { + "id": "f11de143-c31c-49a2-8756-83697dbabe0f", + "version": "8.5.0", + "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", + "kernel": "5.10.47-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "family": "debian", "type": "linux", - "version": "7 (Core)", - "platform": "centos" + "version": "20.04.5 LTS (Focal Fossa)", + "platform": "ubuntu" }, "containerized": true, "ip": [ - "172.18.0.7" + "172.28.0.7" ], "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", + "id": "f1eefc91053740c399ff6f1cd52c37bb", "mac": [ - "02:42:ac:12:00:07" + "02-42-AC-1C-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 10000, - "name": "pending_task" + "name": "pending_tasks" }, "event": { - "duration": 4139652, + "duration": 4546300, "agent_id_status": "verified", - "ingested": "2021-07-30T14:41:18.844042490Z", + "ingested": "2022-09-21T16:00:35Z", "module": "elasticsearch", - "dataset": "elasticsearch.pending_task" + "dataset": "elasticsearch.stack_monitoring.pending_tasks" } } \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/default.yml index 67dc3d2435..1af1baeb2f 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/default.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/default.yml @@ -4,9 +4,6 @@ processors: - set: field: event.ingested value: "{{_ingest.timestamp}}" - - set: - copy_from: "@timestamp" - field: event.created - grok: field: message patterns: @@ -17,11 +14,14 @@ processors: if: ctx.first_char != '{' - pipeline: if: ctx.first_char == '{' - name: '{< IngestPipeline "pipeline-json" >}' + name: '{{ IngestPipeline "pipeline-json" }}' + - set: + copy_from: "@timestamp" + field: event.created - script: lang: painless source: >- - if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.observation_duration != null) { + if (ctx.elasticsearch.server != null && ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.observation_duration != null) { if (ctx.elasticsearch.server.gc.observation_duration.unit == params.seconds_unit) { ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_s; } @@ -32,7 +32,7 @@ processors: ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_m; } } - if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.collection_duration != null) { + if (ctx.elasticsearch.server != null && ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.collection_duration != null) { if (ctx.elasticsearch.server.gc.collection_duration.unit == params.seconds_unit) { ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_s; } @@ -75,6 +75,10 @@ processors: field: host.id value: "{{elasticsearch.node.id}}" ignore_empty_value: true + - rename: + field: tags + target_field: elasticsearch.server.tags + ignore_missing: true - remove: field: - elasticsearch.server.gc.collection_duration.time @@ -82,11 +86,6 @@ processors: - elasticsearch.server.gc.observation_duration.time - elasticsearch.server.gc.observation_duration.unit ignore_missing: true - - remove: - field: - - elasticsearch.server.timestamp - - elasticsearch.server.@timestamp - ignore_missing: true - remove: field: - first_char diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/pipeline-json.yml index 5d9b063ef2..544164809f 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/pipeline-json.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -5,88 +5,42 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: - - json: + - rename: field: message - target_field: elasticsearch.server - - dot_expander: - field: event.dataset - path: elasticsearch.server - - drop: - if: ctx.elasticsearch.server.event.dataset != 'elasticsearch.server' - - set: - value: '{{ elasticsearch.server.event.dataset }}' - field: event.dataset - ignore_empty_value: true - - remove: - field: elasticsearch.server.event.dataset - - dot_expander: - field: ecs.version - path: elasticsearch.server - - set: - value: '{{ elasticsearch.server.ecs.version }}' - field: ecs.version - ignore_empty_value: true + target_field: _ecs_json_message + ignore_missing: true + - json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false - remove: - field: elasticsearch.server.ecs.version - - dot_expander: - field: service.name - path: elasticsearch.server - - rename: - field: elasticsearch.server.service.name - target_field: service.name + field: _ecs_json_message ignore_missing: true + - dot_expander: + field: "*" + override: true + - join: + field: error.stack_trace + separator: "\n" + if: ctx.error?.stack_trace instanceof Collection + - drop: + if: ctx.event.dataset != 'elasticsearch.server' - set: field: service.type value: 'elasticsearch' - - dot_expander: - field: elasticsearch.cluster.name - path: elasticsearch.server - - rename: - field: elasticsearch.server.elasticsearch.cluster.name - target_field: elasticsearch.cluster.name - - dot_expander: - field: elasticsearch.node.name - path: elasticsearch.server - - rename: - field: elasticsearch.server.elasticsearch.node.name - target_field: elasticsearch.node.name - - dot_expander: - field: elasticsearch.cluster.uuid - path: elasticsearch.server - - rename: - field: elasticsearch.server.elasticsearch.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true - - dot_expander: - field: elasticsearch.node.id - path: elasticsearch.server - - rename: - field: elasticsearch.server.elasticsearch.node.id - target_field: elasticsearch.node.id - ignore_missing: true - - dot_expander: - field: log.level - path: elasticsearch.server - - rename: - field: elasticsearch.server.log.level - target_field: log.level - ignore_missing: true - - dot_expander: - field: log.logger - path: elasticsearch.server - - rename: - field: elasticsearch.server.log.logger - target_field: log.logger - ignore_missing: true - - dot_expander: - field: process.thread.name - path: elasticsearch.server - - rename: - field: elasticsearch.server.process.thread.name - target_field: process.thread.name - ignore_missing: true - grok: - field: elasticsearch.server.message + field: message pattern_definitions: GREEDYMULTILINE: |- (.| @@ -101,14 +55,6 @@ processors: - '%{GC_ALL}' - '%{GC_YOUNG}' - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} - - remove: - field: elasticsearch.server.message - - set: - field: '@timestamp' - value: '{{ elasticsearch.server.@timestamp }}' - ignore_empty_value: true - - remove: - field: elasticsearch.server.@timestamp - date: field: '@timestamp' target_field: '@timestamp' diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/ecs.yml new file mode 100755 index 0000000000..5dc3e5ac98 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/ecs.yml @@ -0,0 +1,45 @@ +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. + If the event wasn't read from a log file, do not populate this field. + name: log.file.path + type: keyword +- description: |- + Original log level of the log event. + If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). + Some examples are `warn`, `err`, `i`, `informational`. + name: log.level + type: keyword +- description: The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. + name: log.logger + type: keyword +- description: |- + For log events the message field contains the log message, optimized for viewing in a log viewer. + For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. + If multiple messages exist, they can be combined into one message. + name: message + type: match_only_text +- description: Thread name. + name: process.thread.name + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Unique identifier of the trace. + A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. + name: trace.id + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/fields.yml index 2bee399d0c..f3d2cc8b4e 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/fields.yml @@ -1,6 +1,8 @@ - name: elasticsearch.server type: group fields: + - name: tags + type: nested - name: stacktrace type: keyword - name: gc diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/package-fields.yml index 7ef974b1ab..0484263f80 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/package-fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/fields/package-fields.yml @@ -1,3 +1,11 @@ +- name: server.name + type: keyword +- name: server.type + type: keyword +- name: input.type + type: keyword +- name: log.offset + type: long - name: elasticsearch type: group fields: diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/server/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/server/sample_event.json new file mode 100755 index 0000000000..cc81eec13a --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/server/sample_event.json @@ -0,0 +1,82 @@ +{ + "@timestamp": "2022-09-02T11:20:38.272Z", + "agent": { + "ephemeral_id": "90585bcf-75a6-4833-8bda-60d7dbfb0bcc", + "id": "06b7abfc-6020-4cad-a582-3edfcc4d8ca1", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.server", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.2.0" + }, + "elastic_agent": { + "id": "06b7abfc-6020-4cad-a582-3edfcc4d8ca1", + "snapshot": false, + "version": "8.3.2" + }, + "elasticsearch": { + "cluster": { + "name": "elasticsearch" + }, + "node": { + "name": "0022789c5047" + } + }, + "event": { + "agent_id_status": "verified", + "category": "database", + "created": "2022-09-02T11:21:13.177Z", + "dataset": "elasticsearch.server", + "ingested": "2022-09-02T11:21:14Z", + "kind": "event", + "type": "info" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "ip": [ + "172.18.0.7" + ], + "mac": [ + "02:42:ac:12:00:07" + ], + "name": "0022789c5047", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/elasticsearch_server.json" + }, + "level": "INFO", + "logger": "org.elasticsearch.node.Node", + "offset": 0 + }, + "message": "version[8.5.0-SNAPSHOT], pid[199], build[docker/e94b4befc5f59db2c56eb6b042a735e77c77cd87/2022-08-02T00:20:46.767618562Z], OS[Linux/5.10.47-linuxkit/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/18.0.2/18.0.2+9-61]", + "process": { + "thread": { + "name": "main" + } + }, + "service": { + "name": "ES_ECS", + "type": "elasticsearch" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/ecs.yml index 26fee338b7..184ac7ed84 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/ecs.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/fields/ecs.yml @@ -46,3 +46,6 @@ In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. name: service.name type: keyword +- description: Error message. + name: error.message + type: match_only_text diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/sample_event.json index bf76eef883..e6c9dbad8a 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/shard/sample_event.json +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/shard/sample_event.json @@ -1,83 +1,90 @@ { + "@timestamp": "2022-09-02T14:19:48.613Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "7533d718-43c3-4106-aa29-37168d6a2769", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", "type": "metricbeat", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", - "version": "7.14.0" + "version": "8.3.2" }, - "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true + "data_stream": { + "dataset": "elasticsearch.shard", + "namespace": "ep", + "type": "metrics" }, - "@timestamp": "2021-07-30T14:41:17.832Z", "ecs": { - "version": "1.10.0" + "version": "8.0.0" + }, + "elastic_agent": { + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, "elasticsearch": { - "node": { - "name": "6XuAxHXaRbeX6LUrxIfAxg" - }, "cluster": { - "name": "docker-cluster", - "id": "bvF4SoDLQU-sdM3YY8JI8Q", + "id": "hBVXsE1NTkqWp6cdjr-yWw", + "name": "elasticsearch", "state": { - "id": "mOYQ8E-ORnGSnnp9sB4BCw" + "id": "V7ASeCFmSXWm7W-tuSl_bA" + }, + "stats": { + "state": { + "state_uuid": "V7ASeCFmSXWm7W-tuSl_bA" + } } }, "index": { - "name": ".async-search" + "name": ".ds-.logs-deprecation.elasticsearch-default-2022.09.02-000001" + }, + "node": { + "id": "JGcyPUWaTiOW2Ri0hDUC-A", + "name": "32a9c755b09e" }, "shard": { "number": 0, + "primary": true, "relocating_node": {}, - "state": "STARTED", - "primary": true + "source_node": { + "name": "32a9c755b09e", + "uuid": "JGcyPUWaTiOW2Ri0hDUC-A" + }, + "state": "STARTED" } }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.shard" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.shard", + "duration": 17200300, + "ingested": "2022-09-02T14:19:49Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "type": "linux", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", "mac": [ "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "shard" + "name": "shard", + "period": 10000 }, - "event": { - "duration": 4139652, - "agent_id_status": "verified", - "ingested": "2021-07-30T14:41:18.844042490Z", - "module": "elasticsearch", - "dataset": "elasticsearch.shard" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "type": "elasticsearch" } } \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml index 76ba77ebd1..86e0624b08 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml @@ -4,9 +4,6 @@ processors: - set: field: event.ingested value: "{{_ingest.timestamp}}" - - set: - copy_from: "@timestamp" - field: event.created - grok: field: message patterns: @@ -18,6 +15,9 @@ processors: - pipeline: if: ctx.first_char == '{' name: '{{ IngestPipeline "pipeline-json" }}' + - set: + copy_from: "@timestamp" + field: event.created - remove: field: - elasticsearch.slowlog.timestamp diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/pipeline-json.yml index 31d1b22fda..7f6a2bdbfa 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/pipeline-json.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -22,6 +22,9 @@ processors: field: elasticsearch.slowlog.took_millis target_field: elasticsearch.slowlog.duration ignore_missing: true + - remove: + field: elasticsearch.slowlog.id + if: "ctx.elasticsearch?.slowlog?.id == null" - grok: field: elasticsearch.slowlog.message pattern_definitions: diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/ecs.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/ecs.yml new file mode 100755 index 0000000000..5dc3e5ac98 --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/ecs.yml @@ -0,0 +1,45 @@ +- description: |- + ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + name: ecs.version + type: keyword +- description: |- + Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. + If the event wasn't read from a log file, do not populate this field. + name: log.file.path + type: keyword +- description: |- + Original log level of the log event. + If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). + Some examples are `warn`, `err`, `i`, `informational`. + name: log.level + type: keyword +- description: The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. + name: log.logger + type: keyword +- description: |- + For log events the message field contains the log message, optimized for viewing in a log viewer. + For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. + If multiple messages exist, they can be combined into one message. + name: message + type: match_only_text +- description: Thread name. + name: process.thread.name + type: keyword +- description: |- + Name of the service data is collected from. + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. + name: service.name + type: keyword +- description: |- + The type of the service data is collected from. + The type can be used to group and correlate logs and metrics from one service type. + Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + name: service.type + type: keyword +- description: |- + Unique identifier of the trace. + A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. + name: trace.id + type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/fields.yml index 61f22cc5d0..fce1db1f64 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/fields.yml @@ -26,7 +26,7 @@ type: keyword description: Total hits - name: total_shards - type: keyword + type: long description: Total queried shards - name: routing type: keyword diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/package-fields.yml b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/package-fields.yml index 7ef974b1ab..da4033d9ba 100755 --- a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/package-fields.yml +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/fields/package-fields.yml @@ -1,3 +1,7 @@ +- name: input.type + type: keyword +- name: log.offset + type: long - name: elasticsearch type: group fields: diff --git a/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/sample_event.json b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/sample_event.json new file mode 100755 index 0000000000..cb667cdb1e --- /dev/null +++ b/packages/elasticsearch/1.1.0-preview1/data_stream/slowlog/sample_event.json @@ -0,0 +1,98 @@ +{ + "@timestamp": "2022-09-02T12:33:09.756Z", + "agent": { + "ephemeral_id": "331f5b1f-6690-49cb-a5a6-3760a091f620", + "id": "06b7abfc-6020-4cad-a582-3edfcc4d8ca1", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.slowlog", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.2.0" + }, + "elastic_agent": { + "id": "06b7abfc-6020-4cad-a582-3edfcc4d8ca1", + "snapshot": false, + "version": "8.3.2" + }, + "elasticsearch": { + "cluster": { + "name": "elasticsearch", + "uuid": "Im49t1FlRCWO3bNmVn2Cng" + }, + "index": { + "id": "nTm8TtT-TCOlqkM_URP0zg", + "name": "test_1" + }, + "node": { + "id": "ZLlIEp7LR9WrG9PPP_XP-A", + "name": "d0a05a04075e" + }, + "slowlog": { + "id": "j6sw_oIBZJ8FoknY1lQP", + "source": "{\\\"a\\\":\\\"b\\\"}", + "took": "17.8ms" + } + }, + "event": { + "agent_id_status": "verified", + "category": "database", + "created": "2022-09-02T12:33:32.393Z", + "dataset": "elasticsearch.index_indexing_slowlog", + "duration": 17000000, + "ingested": "2022-09-02T12:33:33Z", + "kind": "event", + "type": "info" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "ZLlIEp7LR9WrG9PPP_XP-A", + "ip": [ + "172.18.0.7" + ], + "mac": [ + "02:42:ac:12:00:07" + ], + "name": "d0a05a04075e", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/elasticsearch_index_indexing_slowlog.json" + }, + "level": "TRACE", + "logger": "index.indexing.slowlog.index", + "offset": 0 + }, + "message": "[test_1/nTm8TtT-TCOlqkM_URP0zg]", + "process": { + "thread": { + "name": "elasticsearch[d0a05a04075e][write][T#3]" + } + }, + "service": { + "name": "ES_ECS", + "type": "elasticsearch" + }, + "trace": { + "id": "0af7651916cd43dd8448eb211c80319c" + } +} \ No newline at end of file diff --git a/packages/elasticsearch/1.1.0-preview1/docs/README.md b/packages/elasticsearch/1.1.0-preview1/docs/README.md index 456dc33182..3f3f1bb3ab 100755 --- a/packages/elasticsearch/1.1.0-preview1/docs/README.md +++ b/packages/elasticsearch/1.1.0-preview1/docs/README.md @@ -4,17 +4,11 @@ The `elasticsearch` package collects metrics and logs of Elasticsearch. ## Compatibility -The `elasticsearch` package can monitor Elasticsearch 6.7.0 and later. +The `elasticsearch` package can monitor Elasticsearch 8.5.0 and later. ## Logs -NOTE: If you're running against Elasticsearch >= 7.0.0, configure the -`var.paths` setting to point to JSON logs. Otherwise, configure it -to point to plain text logs. - -### Compatibility - -The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and newer. +NOTE: Configure the `var.paths` setting to point to JSON logs. ### Audit @@ -26,13 +20,16 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.audit.action | The name of the action that was executed | keyword | +| elasticsearch.audit.authentication.type | | keyword | | elasticsearch.audit.component | | keyword | | elasticsearch.audit.event_type | The type of event that occurred: anonymous_access_denied, authentication_failed, access_denied, access_granted, connection_granted, connection_denied, tampered_request, run_as_granted, run_as_denied | keyword | | elasticsearch.audit.indices | Indices accessed by action | keyword | | elasticsearch.audit.invalidate.apikeys.owned_by_authenticated_user | | boolean | | elasticsearch.audit.layer | The layer from which this event originated: rest, transport or ip_filter | keyword | | elasticsearch.audit.message | | text | +| elasticsearch.audit.opaque_id | | keyword | | elasticsearch.audit.origin.type | Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request) | keyword | | elasticsearch.audit.realm | The authentication realm the authentication was validated against | keyword | | elasticsearch.audit.request.id | Unique ID of request | keyword | @@ -53,8 +50,20 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | http | Fields related to HTTP activity. Use the `url` field set to store the url of the request. | group | | http.request.body.content | The full HTTP request body. | wildcard | | http.request.body.content.text | Multi-field of `http.request.body.content`. | match_only_text | +| http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | +| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| input.type | | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.offset | | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| related.user | | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | source | Source fields capture details about the sender of a network exchange/packet. These fields are populated from a network event, packet, or other event containing details of a network transaction. Source fields are usually populated in conjunction with destination fields. The source and destination fields are considered the baseline and should always be filled if an event contains source and destination details from a network transaction. If the event also contains identification of the client and server roles, then the client and server fields should also be populated. | group | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.port | Port of the source. | long | +| trace.id | Unique identifier of the trace. A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. | keyword | | url | URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. | group | | url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | | url.original.text | Multi-field of `url.original`. | match_only_text | @@ -73,14 +82,28 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.name | Name of the cluster | keyword | | elasticsearch.cluster.uuid | UUID of the cluster | keyword | | elasticsearch.component | Elasticsearch component from where the log event originated | keyword | +| elasticsearch.elastic_product_origin | | keyword | +| elasticsearch.event.category | | keyword | +| elasticsearch.http.request.x_opaque_id | | keyword | | elasticsearch.index.id | Index id | keyword | | elasticsearch.index.name | Index name | keyword | | elasticsearch.node.id | ID of the node | keyword | | elasticsearch.node.name | Name of the node | keyword | | elasticsearch.shard.id | Id of the shard | keyword | +| input.type | | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | +| log.offset | | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.thread.name | Thread name. | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| trace.id | Unique identifier of the trace. A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. | keyword | ### Garbage collection @@ -93,6 +116,7 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.name | Name of the cluster | keyword | | elasticsearch.cluster.uuid | UUID of the cluster | keyword | | elasticsearch.component | Elasticsearch component from where the log event originated | keyword | @@ -121,6 +145,12 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | elasticsearch.node.id | ID of the node | keyword | | elasticsearch.node.name | Name of the node | keyword | | elasticsearch.shard.id | Id of the shard | keyword | +| input.type | | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.offset | | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.pid | Process id. | long | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | ### Server @@ -133,6 +163,7 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.name | Name of the cluster | keyword | | elasticsearch.cluster.uuid | UUID of the cluster | keyword | | elasticsearch.component | Elasticsearch component from where the log event originated | keyword | @@ -146,7 +177,20 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | elasticsearch.server.gc.young.one | | long | | elasticsearch.server.gc.young.two | | long | | elasticsearch.server.stacktrace | | keyword | +| elasticsearch.server.tags | | nested | | elasticsearch.shard.id | Id of the shard | keyword | +| input.type | | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | +| log.offset | | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.thread.name | Thread name. | keyword | +| server.name | | keyword | +| server.type | | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| trace.id | Unique identifier of the trace. A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. | keyword | ### Slowlog @@ -159,6 +203,7 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.name | Name of the cluster | keyword | | elasticsearch.cluster.uuid | UUID of the cluster | keyword | | elasticsearch.component | Elasticsearch component from where the log event originated | keyword | @@ -177,9 +222,19 @@ The Elasticsearch package is compatible with logs from Elasticsearch 6.2 and new | elasticsearch.slowlog.stats | Stats groups | keyword | | elasticsearch.slowlog.took | Time it took to execute the query | keyword | | elasticsearch.slowlog.total_hits | Total hits | keyword | -| elasticsearch.slowlog.total_shards | Total queried shards | keyword | +| elasticsearch.slowlog.total_shards | Total queried shards | long | | elasticsearch.slowlog.type | Type | keyword | | elasticsearch.slowlog.types | Types | keyword | +| input.type | | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | +| log.offset | | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.thread.name | Thread name. | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| trace.id | Unique identifier of the trace. A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. | keyword | ## Metrics @@ -214,9 +269,41 @@ will not collect metrics. A DEBUG log message about this will be emitted in the | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| ccr_auto_follow_stats.follower.failed_read_requests | | alias | +| ccr_auto_follow_stats.number_of_failed_follow_indices | | alias | +| ccr_auto_follow_stats.number_of_failed_remote_cluster_state_requests | | alias | +| ccr_auto_follow_stats.number_of_successful_follow_indices | | alias | +| ccr_stats.bytes_read | | alias | +| ccr_stats.failed_read_requests | | alias | +| ccr_stats.failed_write_requests | | alias | +| ccr_stats.follower_aliases_version | | alias | +| ccr_stats.follower_global_checkpoint | | alias | +| ccr_stats.follower_index | | alias | +| ccr_stats.follower_mapping_version | | alias | +| ccr_stats.follower_max_seq_no | | alias | +| ccr_stats.follower_settings_version | | alias | +| ccr_stats.last_requested_seq_no | | alias | +| ccr_stats.leader_global_checkpoint | | alias | +| ccr_stats.leader_index | | alias | +| ccr_stats.leader_max_seq_no | | alias | +| ccr_stats.operations_read | | alias | +| ccr_stats.operations_written | | alias | +| ccr_stats.outstanding_read_requests | | alias | +| ccr_stats.outstanding_write_requests | | alias | +| ccr_stats.remote_cluster | | alias | +| ccr_stats.shard_id | | alias | +| ccr_stats.successful_read_requests | | alias | +| ccr_stats.successful_write_requests | | alias | +| ccr_stats.total_read_remote_exec_time_millis | | alias | +| ccr_stats.total_read_time_millis | | alias | +| ccr_stats.total_write_time_millis | | alias | +| ccr_stats.write_buffer_operation_count | | alias | +| ccr_stats.write_buffer_size_in_bytes | | alias | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.ccr.auto_follow.failed.follow_indices.count | | long | | elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count | | long | | elasticsearch.ccr.auto_follow.success.follow_indices.count | | long | @@ -256,6 +343,18 @@ will not collect metrics. A DEBUG log message about this will be emitted in the | elasticsearch.node.master | Is the node the master node? | boolean | | elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | | elasticsearch.node.name | Node name. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + ### Cluster Stats @@ -361,9 +460,22 @@ An example event for `cluster_stats` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_state.master_node | | alias | +| cluster_state.nodes_hash | | alias | +| cluster_state.state_uuid | | alias | +| cluster_state.status | | alias | +| cluster_state.version | | alias | +| cluster_stats.indices.count | | alias | +| cluster_stats.indices.shards.total | | alias | +| cluster_stats.nodes.count.total | | alias | +| cluster_stats.nodes.jvm.max_uptime_in_millis | | alias | +| cluster_stats.nodes.jvm.mem.heap_max_in_bytes | | alias | +| cluster_stats.nodes.jvm.mem.heap_used_in_bytes | | alias | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | @@ -399,10 +511,27 @@ An example event for `cluster_stats` looks as following: | elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | | elasticsearch.node.name | Node name. | keyword | | elasticsearch.version | | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| license.status | | alias | +| license.type | | alias | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| stack_stats.apm.found | | alias | +| stack_stats.xpack.ccr.available | | alias | +| stack_stats.xpack.ccr.enabled | | alias | +| timestamp | | alias | + ### Enrich -Enrch interrogates the [Enrich Stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-apis.html) +Enrch interrogates the [Enrich Stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-apis.html) endpoint to fetch information about Enrich coordinator nodesin the Elasticsearch cluster that are participating in ingest-time enrichment. @@ -499,9 +628,11 @@ An example event for `enrich` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | @@ -521,6 +652,18 @@ An example event for `enrich` looks as following: | elasticsearch.node.master | Is the node the master node? | boolean | | elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | | elasticsearch.node.name | Node name. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + ### Index @@ -528,91 +671,146 @@ An example event for `index` looks as following: ```json { + "@timestamp": "2022-09-02T14:06:12.353Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "5c8415cd-4402-4ddf-b627-b13790bc3197", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", "type": "metricbeat", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", - "version": "7.14.0" + "version": "8.3.2" }, - "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true + "data_stream": { + "dataset": "elasticsearch.index", + "namespace": "ep", + "type": "metrics" }, - "@timestamp": "2021-07-30T14:46:47.831Z", "ecs": { - "version": "1.10.0" + "version": "8.0.0" + }, + "elastic_agent": { + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, "elasticsearch": { "cluster": { - "name": "docker-cluster", - "id": "bvF4SoDLQU-sdM3YY8JI8Q" + "id": "zv3a1lJUQoK10VDNC6J0qA", + "name": "elasticsearch" }, "index": { - "total": { + "hidden": false, + "name": "testindex2", + "primaries": { "docs": { - "deleted": 0, - "count": 13267 + "count": 0 + }, + "indexing": { + "index_time_in_millis": 0, + "index_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "total_size_in_bytes": 0 + }, + "refresh": { + "total_time_in_millis": 0 + }, + "segments": { + "count": 0 }, "store": { - "size": { - "bytes": 1490775 - } + "size_in_bytes": 675 + } + }, + "shards": { + "primaries": 3, + "total": 6 + }, + "status": "yellow", + "total": { + "bulk": { + "avg_size_in_bytes": 0, + "avg_time_in_millis": 0, + "total_operations": 0, + "total_size_in_bytes": 0, + "total_time_in_millis": 0 + }, + "docs": { + "count": 0 + }, + "fielddata": { + "memory_size_in_bytes": 0 + }, + "indexing": { + "index_time_in_millis": 0, + "index_total": 0, + "throttle_time_in_millis": 0 + }, + "merges": { + "total_size_in_bytes": 0 + }, + "refresh": { + "total_time_in_millis": 0 + }, + "search": { + "query_time_in_millis": 0, + "query_total": 0 }, "segments": { - "memory": { - "bytes": 50388 - }, - "count": 5 + "count": 0, + "doc_values_memory_in_bytes": 0, + "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 0, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 0, + "version_map_memory_in_bytes": 0 + }, + "store": { + "size_in_bytes": 675 } }, - "name": ".ds-metrics-elasticsearch.shard-default-2021.07.30-000001" + "uuid": "lH2NeM70TlKGEB11uUxiuA" } }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.index" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.index", + "duration": 34210900, + "ingested": "2022-09-02T14:06:13Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", "mac": [ "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "index" + "name": "index", + "period": 10000 }, - "event": { - "duration": 14394992, - "agent_id_status": "verified", - "ingested": "2021-07-30T14:46:48.854674866Z", - "module": "elasticsearch", - "dataset": "elasticsearch.index" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "type": "elasticsearch" } } ``` @@ -621,10 +819,12 @@ An example event for `index` looks as following: | Field | Description | Type | |---|---|---| -| @timestamp | Event timestamp. | date | +| @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | @@ -660,8 +860,14 @@ An example event for `index` looks as following: | elasticsearch.index.primaries.segments.terms_memory_in_bytes | | long | | elasticsearch.index.primaries.segments.version_map_memory_in_bytes | | long | | elasticsearch.index.primaries.store.size_in_bytes | | long | +| elasticsearch.index.shards.primaries | | long | | elasticsearch.index.shards.total | | long | | elasticsearch.index.status | | keyword | +| elasticsearch.index.total.bulk.avg_size_in_bytes | | long | +| elasticsearch.index.total.bulk.avg_time_in_millis | | long | +| elasticsearch.index.total.bulk.total_operations | | long | +| elasticsearch.index.total.bulk.total_size_in_bytes | | long | +| elasticsearch.index.total.bulk.total_time_in_millis | | long | | elasticsearch.index.total.docs.count | Total number of documents in the index. | long | | elasticsearch.index.total.docs.deleted | Total number of deleted documents in the index. | long | | elasticsearch.index.total.fielddata.evictions | | long | @@ -686,6 +892,7 @@ An example event for `index` looks as following: | elasticsearch.index.total.segments.doc_values_memory_in_bytes | | long | | elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes | | long | | elasticsearch.index.total.segments.index_writer_memory_in_bytes | | long | +| elasticsearch.index.total.segments.memory.bytes | Total number of memory used by the segments in bytes. | long | | elasticsearch.index.total.segments.memory_in_bytes | Total number of memory used by the segments in bytes. | long | | elasticsearch.index.total.segments.norms_memory_in_bytes | | long | | elasticsearch.index.total.segments.points_memory_in_bytes | | long | @@ -693,12 +900,63 @@ An example event for `index` looks as following: | elasticsearch.index.total.segments.term_vectors_memory_in_bytes | | long | | elasticsearch.index.total.segments.terms_memory_in_bytes | | long | | elasticsearch.index.total.segments.version_map_memory_in_bytes | | long | +| elasticsearch.index.total.store.size.bytes | | long | | elasticsearch.index.total.store.size_in_bytes | Total size of the index in bytes. | long | | elasticsearch.index.uuid | | keyword | | elasticsearch.node.id | Node ID | keyword | | elasticsearch.node.master | Is the node the master node? | boolean | | elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | | elasticsearch.node.name | Node name. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| index_recovery.shards.start_time_in_millis | | alias | +| index_recovery.shards.stop_time_in_millis | | alias | +| index_stats.index | | alias | +| index_stats.primaries.docs.count | | alias | +| index_stats.primaries.indexing.index_time_in_millis | | alias | +| index_stats.primaries.indexing.index_total | | alias | +| index_stats.primaries.indexing.throttle_time_in_millis | | alias | +| index_stats.primaries.merges.total_size_in_bytes | | alias | +| index_stats.primaries.refresh.total_time_in_millis | | alias | +| index_stats.primaries.segments.count | | alias | +| index_stats.primaries.store.size_in_bytes | | alias | +| index_stats.total.fielddata.memory_size_in_bytes | | alias | +| index_stats.total.indexing.index_time_in_millis | | alias | +| index_stats.total.indexing.index_total | | alias | +| index_stats.total.indexing.throttle_time_in_millis | | alias | +| index_stats.total.merges.total_size_in_bytes | | alias | +| index_stats.total.query_cache.memory_size_in_bytes | | alias | +| index_stats.total.refresh.total_time_in_millis | | alias | +| index_stats.total.request_cache.memory_size_in_bytes | | alias | +| index_stats.total.search.query_time_in_millis | | alias | +| index_stats.total.search.query_total | | alias | +| index_stats.total.segments.count | | alias | +| index_stats.total.segments.doc_values_memory_in_bytes | | alias | +| index_stats.total.segments.fixed_bit_set_memory_in_bytes | | alias | +| index_stats.total.segments.index_writer_memory_in_bytes | | alias | +| index_stats.total.segments.memory_in_bytes | | alias | +| index_stats.total.segments.norms_memory_in_bytes | | alias | +| index_stats.total.segments.points_memory_in_bytes | | alias | +| index_stats.total.segments.stored_fields_memory_in_bytes | | alias | +| index_stats.total.segments.term_vectors_memory_in_bytes | | alias | +| index_stats.total.segments.terms_memory_in_bytes | | alias | +| index_stats.total.segments.version_map_memory_in_bytes | | alias | +| index_stats.total.store.size_in_bytes | | alias | +| indices_stats._all.primaries.indexing.index_time_in_millis | | alias | +| indices_stats._all.primaries.indexing.index_total | | alias | +| indices_stats._all.total.indexing.index_total | | alias | +| indices_stats._all.total.search.query_time_in_millis | | alias | +| indices_stats._all.total.search.query_total | | alias | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + ### Index recovery @@ -823,9 +1081,11 @@ An example event for `index_recovery` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | @@ -862,7 +1122,21 @@ An example event for `index_recovery` looks as following: | elasticsearch.node.master | Is the node the master node? | boolean | | elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | | elasticsearch.node.name | Node name. | keyword | -| version | | long | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| index_recovery.shards.start_time_in_millis | | alias | +| index_recovery.shards.stop_time_in_millis | | alias | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + + ### Index summary @@ -870,109 +1144,165 @@ An example event for `index_summary` looks as following: ```json { + "@timestamp": "2022-09-02T14:23:38.078Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "5dcbe5f9-d61d-4931-b4f3-a334e8e999b2", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", "type": "metricbeat", - "version": "7.14.0" + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.index_summary", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, - "@timestamp": "2021-07-30T14:47:15.391Z", "elasticsearch": { "cluster": { - "name": "docker-cluster", - "id": "bvF4SoDLQU-sdM3YY8JI8Q" + "id": "zZUl__19TuWgxPiewmnJ3Q", + "name": "elasticsearch" }, "index": { "summary": { "primaries": { + "bulk": { + "operations": { + "count": 3 + }, + "size": { + "bytes": 45 + }, + "time": { + "avg": { + "bytes": 4 + } + } + }, "docs": { - "deleted": 7226, - "count": 50723 + "count": 3, + "deleted": 0 }, - "store": { - "size": { - "bytes": 36769186 + "indexing": { + "index": { + "count": 3, + "time": { + "ms": 14 + } + } + }, + "search": { + "query": { + "count": 9, + "time": { + "ms": 20 + } } }, "segments": { + "count": 3, "memory": { - "bytes": 1790592 - }, - "count": 222 + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 8466 + } } }, "total": { + "bulk": { + "operations": { + "count": 3 + }, + "size": { + "bytes": 45 + }, + "time": { + "avg": { + "bytes": 4 + } + } + }, "docs": { - "deleted": 7226, - "count": 50723 + "count": 3, + "deleted": 0 }, - "store": { - "size": { - "bytes": 36769186 + "indexing": { + "index": { + "count": 3, + "time": { + "ms": 14 + } + } + }, + "search": { + "query": { + "count": 9, + "time": { + "ms": 20 + } } }, "segments": { + "count": 3, "memory": { - "bytes": 1790592 - }, - "count": 222 + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 8466 + } } } } } }, - "ecs": { - "version": "1.10.0" - }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.index_summary" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.index_summary", + "duration": 32732300, + "ingested": "2022-09-02T14:23:39Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", "mac": [ "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "index_summary" + "name": "index_summary", + "period": 10000 }, - "event": { - "duration": 12151260, - "agent_id_status": "verified", - "ingested": "2021-07-30T14:47:16.373343461Z", - "module": "elasticsearch", - "dataset": "elasticsearch.index_summary" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "name": "elasticsearch", + "type": "elasticsearch" } } ``` @@ -982,9 +1312,11 @@ An example event for `index_summary` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | @@ -1021,6 +1353,23 @@ An example event for `index_summary` looks as following: | elasticsearch.node.master | Is the node the master node? | boolean | | elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | | elasticsearch.node.name | Node name. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| indices_stats._all.primaries.indexing.index_time_in_millis | | alias | +| indices_stats._all.primaries.indexing.index_total | | alias | +| indices_stats._all.total.indexing.index_total | | alias | +| indices_stats._all.total.search.query_time_in_millis | | alias | +| indices_stats._all.total.search.query_total | | alias | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + ### Machine Learning Jobs @@ -1075,9 +1424,11 @@ An example event for `ml_job` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | @@ -1092,11 +1443,25 @@ An example event for `ml_job` looks as following: | elasticsearch.node.master | Is the node the master node? | boolean | | elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | | elasticsearch.node.name | Node name. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| job_stats.forecasts_stats.total | | alias | +| job_stats.job_id | | alias | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + ### Node `node` interrogates the -https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-info.html[Cluster API endpoint] of +[Cluster API endpoint](https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-info.html) of Elasticsearch to get cluster nodes information. It only fetches the data from the `_local` node so it must run on each Elasticsearch node. @@ -1104,26 +1469,34 @@ An example event for `node` looks as following: ```json { + "@timestamp": "2022-09-02T14:13:34.927Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "47946444-4c3a-4915-91dd-bf515aba9740", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "27d29977-878e-4309-81ed-8788662503ad", - "ephemeral_id": "f8f510e7-9503-4e3d-af7f-da2992648d31", "type": "metricbeat", - "version": "7.15.0" + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.node", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "27d29977-878e-4309-81ed-8788662503ad", - "version": "7.15.0", - "snapshot": true + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, - "@timestamp": "2021-08-03T12:27:26.083Z", "elasticsearch": { "cluster": { - "name": "docker-cluster", - "id": "icut8oAwR--oCfUTlFaPMg" + "id": "ziL93dUTRmGy5hsfhhq3Ww", + "name": "elasticsearch" }, "node": { + "id": "3nCEJ8F6SCuBH_c_YJNQSA", "jvm": { "memory": { "heap": { @@ -1143,61 +1516,51 @@ An example event for `node` looks as following: } } }, - "version": "16.0.1" + "version": "18.0.2" }, + "name": "1a6b5d803000", "process": { "mlockall": false }, - "name": "2b8824139b92", - "id": "saWHxJSZQF6VqGZvEb45Uw", - "version": "7.15.0" + "version": "8.5.0" } }, - "ecs": { - "version": "1.10.0" - }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.node" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.node", + "duration": 18259400, + "ingested": "2022-09-02T14:13:35Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ - "172.24.0.7" + "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "1292624d19b2cee1a317ad634c9a8358", "mac": [ - "02:42:ac:18:00:07" + "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "node" + "name": "node", + "period": 10000 }, - "event": { - "duration": 9853150, - "agent_id_status": "verified", - "ingested": "2021-08-03T12:27:27.080460943Z", - "module": "elasticsearch", - "dataset": "elasticsearch.node" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "name": "elasticsearch", + "type": "elasticsearch" } } ``` @@ -1207,9 +1570,11 @@ An example event for `node` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | @@ -1224,11 +1589,23 @@ An example event for `node` looks as following: | elasticsearch.node.name | Node name. | keyword | | elasticsearch.node.process.mlockall | If process locked in memory. | boolean | | elasticsearch.node.version | Node version. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + ### Node stats `node_stats` interrogates the -https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-stats.html[Cluster API endpoint] of +[Cluster API endpoint](https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-stats.html) of Elasticsearch to get the cluster nodes statistics. The data received is only for the local node so the Agent has to be run on each Elasticsearch node. @@ -1238,171 +1615,308 @@ An example event for `node_stats` looks as following: ```json { + "@timestamp": "2022-09-02T14:32:24.121Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "5d429743-0cf8-44a9-afb4-7523cf960d76", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", "type": "metricbeat", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", - "version": "7.14.0" + "version": "8.3.2" + }, + "data_stream": { + "dataset": "elasticsearch.node_stats", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, - "@timestamp": "2021-07-30T14:47:15.407Z", "elasticsearch": { + "cluster": { + "id": "PdQA6zKZQaK1LAvajgbnug", + "name": "elasticsearch" + }, "node": { + "id": "vnPGsgkoQ5-kwzmE6DOjOQ", + "master": true, + "mlockall": false, + "name": "be467614bdb0", "stats": { - "jvm": { - "mem": { - "pools": { - "young": { - "max": { - "bytes": 0 - }, - "used": { - "bytes": 33554432 - }, - "peak": { - "bytes": 633339904 - }, - "peak_max": { - "bytes": 0 - } - }, - "old": { - "max": { - "bytes": 1073741824 - }, - "used": { - "bytes": 248498176 - }, - "peak": { - "bytes": 371192832 - }, - "peak_max": { - "bytes": 1073741824 - } - }, - "survivor": { - "max": { - "bytes": 0 - }, - "peak": { - "bytes": 67829936 - }, - "peak_max": { - "bytes": 0 - }, - "used": { - "bytes": 3283184 - } - } + "fs": { + "io_stats": {}, + "summary": { + "available": { + "bytes": 36166852608 + }, + "free": { + "bytes": 46061998080 + }, + "total": { + "bytes": 193393164288 + } + }, + "total": { + "available_in_bytes": 36166852608, + "total_in_bytes": 193393164288 + } + }, + "indices": { + "bulk": { + "avg_size": { + "bytes": 139 + }, + "avg_time": { + "ms": 4 + }, + "operations": { + "total": { + "count": 6 + } + }, + "total_size": { + "bytes": 5303 + }, + "total_time": { + "ms": 175 + } + }, + "docs": { + "count": 11, + "deleted": 0 + }, + "fielddata": { + "memory": { + "bytes": 0 + } + }, + "indexing": { + "index_time": { + "ms": 31 + }, + "index_total": { + "count": 11 + }, + "throttle_time": { + "ms": 0 + } + }, + "query_cache": { + "memory": { + "bytes": 0 + } + }, + "request_cache": { + "memory": { + "bytes": 0 + } + }, + "search": { + "query_time": { + "ms": 19 + }, + "query_total": { + "count": 9 + } + }, + "segments": { + "count": 6, + "doc_values": { + "memory": { + "bytes": 0 + } + }, + "fixed_bit_set": { + "memory": { + "bytes": 0 + } + }, + "index_writer": { + "memory": { + "bytes": 0 + } + }, + "memory": { + "bytes": 0 + }, + "norms": { + "memory": { + "bytes": 0 + } + }, + "points": { + "memory": { + "bytes": 0 + } + }, + "stored_fields": { + "memory": { + "bytes": 0 + } + }, + "term_vectors": { + "memory": { + "bytes": 0 + } + }, + "terms": { + "memory": { + "bytes": 0 + } + }, + "version_map": { + "memory": { + "bytes": 0 + } } }, + "store": { + "size": { + "bytes": 40643 + } + } + }, + "jvm": { "gc": { "collectors": { - "young": { + "old": { "collection": { - "ms": 6100, - "count": 425 + "count": 0, + "ms": 0 } }, - "old": { + "young": { "collection": { - "ms": 0, - "count": 0 + "count": 9, + "ms": 217 } } } + }, + "mem": { + "heap": { + "max": { + "bytes": 1073741824 + }, + "used": { + "bytes": 400155760, + "pct": 37 + } + } } }, - "indices": { - "docs": { - "deleted": 7226, - "count": 50805 - }, - "store": { - "size": { - "bytes": 37101213 + "os": { + "cgroup": { + "cpu": { + "cfs": { + "quota": { + "us": -1 + } + }, + "stat": { + "elapsed_periods": { + "count": 0 + }, + "times_throttled": { + "count": 0 + } + } + }, + "cpuacct": { + "usage": { + "ns": 56233628308 + } + }, + "memory": { + "control_group": "/", + "limit": { + "bytes": "9223372036854771712" + }, + "usage": { + "bytes": "1536434176" + } } }, - "segments": { - "memory": { - "bytes": 1805548 - }, - "count": 227 + "cpu": { + "load_avg": { + "1m": 1.53 + } } }, - "fs": { - "summary": { - "total": { - "bytes": 958613114880 + "process": { + "cpu": { + "pct": 1 + } + }, + "thread_pool": { + "get": { + "queue": { + "count": 0 }, - "available": { - "bytes": 261931741184 + "rejected": { + "count": 0 + } + }, + "search": { + "queue": { + "count": 0 }, - "free": { - "bytes": 310698074112 + "rejected": { + "count": 0 + } + }, + "write": { + "queue": { + "count": 0 + }, + "rejected": { + "count": 0 } } } - }, - "name": "e7e895f7c41e", - "id": "6XuAxHXaRbeX6LUrxIfAxg" - }, - "cluster": { - "name": "docker-cluster", - "id": "bvF4SoDLQU-sdM3YY8JI8Q" + } } }, - "ecs": { - "version": "1.10.0" - }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.node_stats" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.node_stats", + "duration": 34932600, + "ingested": "2022-09-02T14:32:25Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "type": "linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", "mac": [ "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "node_stats" + "name": "node_stats", + "period": 10000 }, - "event": { - "duration": 32401229, - "agent_id_status": "verified", - "ingested": "2021-07-30T14:47:16.373437564Z", - "module": "elasticsearch", - "dataset": "elasticsearch.node_stats" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "name": "elasticsearch", + "type": "elasticsearch" } } ``` @@ -1412,9 +1926,11 @@ An example event for `node_stats` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | @@ -1430,6 +1946,11 @@ An example event for `node_stats` looks as following: | elasticsearch.node.stats.fs.summary.total.bytes | | long | | elasticsearch.node.stats.fs.total.available_in_bytes | | long | | elasticsearch.node.stats.fs.total.total_in_bytes | | long | +| elasticsearch.node.stats.indices.bulk.avg_size.bytes | | long | +| elasticsearch.node.stats.indices.bulk.avg_time.ms | | long | +| elasticsearch.node.stats.indices.bulk.operations.total.count | | long | +| elasticsearch.node.stats.indices.bulk.total_size.bytes | | long | +| elasticsearch.node.stats.indices.bulk.total_time.ms | | long | | elasticsearch.node.stats.indices.docs.count | Total number of existing documents. | long | | elasticsearch.node.stats.indices.docs.deleted | Total number of deleted documents. | long | | elasticsearch.node.stats.indices.fielddata.memory.bytes | | long | @@ -1477,8 +1998,8 @@ An example event for `node_stats` looks as following: | elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count | | long | | elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns | | long | | elasticsearch.node.stats.os.cgroup.memory.control_group | | keyword | -| elasticsearch.node.stats.os.cgroup.memory.limit.bytes | | long | -| elasticsearch.node.stats.os.cgroup.memory.usage.bytes | | long | +| elasticsearch.node.stats.os.cgroup.memory.limit.bytes | | keyword | +| elasticsearch.node.stats.os.cgroup.memory.usage.bytes | | keyword | | elasticsearch.node.stats.os.cpu.load_avg.1m | | half_float | | elasticsearch.node.stats.process.cpu.pct | | double | | elasticsearch.node.stats.thread_pool.bulk.queue.count | | long | @@ -1491,119 +2012,287 @@ An example event for `node_stats` looks as following: | elasticsearch.node.stats.thread_pool.search.rejected.count | | long | | elasticsearch.node.stats.thread_pool.write.queue.count | | long | | elasticsearch.node.stats.thread_pool.write.rejected.count | | long | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| node_stats.fs.io_stats.total.operations | | alias | +| node_stats.fs.io_stats.total.read_operations | | alias | +| node_stats.fs.io_stats.total.write_operations | | alias | +| node_stats.fs.summary.available.bytes | | alias | +| node_stats.fs.summary.total.bytes | | alias | +| node_stats.fs.total.available_in_bytes | | alias | +| node_stats.fs.total.total_in_bytes | | alias | +| node_stats.indices.docs.count | | alias | +| node_stats.indices.fielddata.memory_size_in_bytes | | alias | +| node_stats.indices.indexing.index_time_in_millis | | alias | +| node_stats.indices.indexing.index_total | | alias | +| node_stats.indices.indexing.throttle_time_in_millis | | alias | +| node_stats.indices.query_cache.memory_size_in_bytes | | alias | +| node_stats.indices.request_cache.memory_size_in_bytes | | alias | +| node_stats.indices.search.query_time_in_millis | | alias | +| node_stats.indices.search.query_total | | alias | +| node_stats.indices.segments.count | | alias | +| node_stats.indices.segments.doc_values_memory_in_bytes | | alias | +| node_stats.indices.segments.fixed_bit_set_memory_in_bytes | | alias | +| node_stats.indices.segments.index_writer_memory_in_bytes | | alias | +| node_stats.indices.segments.memory_in_bytes | | alias | +| node_stats.indices.segments.norms_memory_in_bytes | | alias | +| node_stats.indices.segments.points_memory_in_bytes | | alias | +| node_stats.indices.segments.stored_fields_memory_in_bytes | | alias | +| node_stats.indices.segments.term_vectors_memory_in_bytes | | alias | +| node_stats.indices.segments.terms_memory_in_bytes | | alias | +| node_stats.indices.segments.version_map_memory_in_bytes | | alias | +| node_stats.indices.store.size.bytes | | alias | +| node_stats.indices.store.size_in_bytes | | alias | +| node_stats.jvm.gc.collectors.old.collection_count | | alias | +| node_stats.jvm.gc.collectors.old.collection_time_in_millis | | alias | +| node_stats.jvm.gc.collectors.young.collection_count | | alias | +| node_stats.jvm.gc.collectors.young.collection_time_in_millis | | alias | +| node_stats.jvm.mem.heap_max_in_bytes | | alias | +| node_stats.jvm.mem.heap_used_in_bytes | | alias | +| node_stats.jvm.mem.heap_used_percent | | alias | +| node_stats.node_id | | alias | +| node_stats.os.cgroup.cpu.cfs_quota_micros | | alias | +| node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods | | alias | +| node_stats.os.cgroup.cpu.stat.number_of_times_throttled | | alias | +| node_stats.os.cgroup.cpu.stat.time_throttled_nanos | | alias | +| node_stats.os.cgroup.cpuacct.usage_nanos | | alias | +| node_stats.os.cgroup.memory.control_group | | alias | +| node_stats.os.cgroup.memory.limit_in_bytes | | alias | +| node_stats.os.cgroup.memory.usage_in_bytes | | alias | +| node_stats.os.cpu.load_average.1m | | alias | +| node_stats.process.cpu.percent | | alias | +| node_stats.thread_pool.bulk.queue | | alias | +| node_stats.thread_pool.bulk.rejected | | alias | +| node_stats.thread_pool.get.queue | | alias | +| node_stats.thread_pool.get.rejected | | alias | +| node_stats.thread_pool.index.queue | | alias | +| node_stats.thread_pool.index.rejected | | alias | +| node_stats.thread_pool.search.queue | | alias | +| node_stats.thread_pool.search.rejected | | alias | +| node_stats.thread_pool.write.queue | | alias | +| node_stats.thread_pool.write.rejected | | alias | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + ### Pending tasks +An example event for `pending_tasks` looks as following: + +```json +{ + "agent": { + "name": "docker-fleet-agent", + "id": "f11de143-c31c-49a2-8756-83697dbabe0f", + "ephemeral_id": "3469da57-3138-4702-abc6-8b95e081fc12", + "type": "metricbeat", + "version": "8.5.0" + }, + "@timestamp": "2022-09-21T16:00:34.116Z", + "elasticsearch": { + "cluster": { + "name": "elasticsearch", + "id": "N9ZLPL5RQHS67eZBrujPYg" + }, + "pending_tasks": { + "time_in_queue.ms": 50, + "source": "create-index [foo-bar-1663776034], cause [api]", + "priority": "URGENT", + "insert_order": 3272 + } + }, + "ecs": { + "version": "8.0.0" + }, + "service": { + "address": "https://elasticsearch:9200", + "name": "elasticsearch", + "type": "elasticsearch" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "elasticsearch.stack_monitoring.pending_tasks" + }, + "elastic_agent": { + "id": "f11de143-c31c-49a2-8756-83697dbabe0f", + "version": "8.5.0", + "snapshot": true + }, + "host": { + "hostname": "docker-fleet-agent", + "os": { + "kernel": "5.10.47-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "family": "debian", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "containerized": true, + "ip": [ + "172.28.0.7" + ], + "name": "docker-fleet-agent", + "id": "f1eefc91053740c399ff6f1cd52c37bb", + "mac": [ + "02-42-AC-1C-00-07" + ], + "architecture": "x86_64" + }, + "metricset": { + "period": 10000, + "name": "pending_tasks" + }, + "event": { + "duration": 4546300, + "agent_id_status": "verified", + "ingested": "2022-09-21T16:00:35Z", + "module": "elasticsearch", + "dataset": "elasticsearch.stack_monitoring.pending_tasks" + } +} +``` + **Exported fields** | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | -| elasticsearch.cluster.pending_task.insert_order | Insert order | long | -| elasticsearch.cluster.pending_task.priority | Priority | keyword | -| elasticsearch.cluster.pending_task.source | Source. For example: put-mapping | keyword | -| elasticsearch.cluster.pending_task.time_in_queue.ms | Time in queue | long | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | | elasticsearch.node.id | Node ID | keyword | | elasticsearch.node.master | Is the node the master node? | boolean | | elasticsearch.node.mlockall | Is mlockall enabled on the node? | boolean | | elasticsearch.node.name | Node name. | keyword | - -# Shard +| elasticsearch.pending_tasks.insert_order | Insert order | long | +| elasticsearch.pending_tasks.priority | Priority | keyword | +| elasticsearch.pending_tasks.source | Source. For example: put-mapping | keyword | +| elasticsearch.pending_tasks.time_in_queue.ms | Time in queue | long | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + + +### Shard `shard` interrogates the -https://www.elastic.co/guide/en/elasticsearch/reference/6.2/cluster-state.html[Cluster State API endpoint] to fetch +[Cluster State API endpoint](https://www.elastic.co/guide/en/elasticsearch/reference/6.2/cluster-state.html) to fetch information about all shards. An example event for `shard` looks as following: ```json { + "@timestamp": "2022-09-02T14:19:48.613Z", "agent": { - "hostname": "docker-fleet-agent", + "ephemeral_id": "7533d718-43c3-4106-aa29-37168d6a2769", + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", "name": "docker-fleet-agent", - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", "type": "metricbeat", - "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1", - "version": "7.14.0" + "version": "8.3.2" }, - "elastic_agent": { - "id": "60e15e27-7080-4c28-9900-5a087c2ff74c", - "version": "7.14.0", - "snapshot": true + "data_stream": { + "dataset": "elasticsearch.shard", + "namespace": "ep", + "type": "metrics" }, - "@timestamp": "2021-07-30T14:41:17.832Z", "ecs": { - "version": "1.10.0" + "version": "8.0.0" + }, + "elastic_agent": { + "id": "1ead23a7-d3be-410c-b5c7-c48d297f4939", + "snapshot": false, + "version": "8.3.2" }, "elasticsearch": { - "node": { - "name": "6XuAxHXaRbeX6LUrxIfAxg" - }, "cluster": { - "name": "docker-cluster", - "id": "bvF4SoDLQU-sdM3YY8JI8Q", + "id": "hBVXsE1NTkqWp6cdjr-yWw", + "name": "elasticsearch", "state": { - "id": "mOYQ8E-ORnGSnnp9sB4BCw" + "id": "V7ASeCFmSXWm7W-tuSl_bA" + }, + "stats": { + "state": { + "state_uuid": "V7ASeCFmSXWm7W-tuSl_bA" + } } }, "index": { - "name": ".async-search" + "name": ".ds-.logs-deprecation.elasticsearch-default-2022.09.02-000001" + }, + "node": { + "id": "JGcyPUWaTiOW2Ri0hDUC-A", + "name": "32a9c755b09e" }, "shard": { "number": 0, + "primary": true, "relocating_node": {}, - "state": "STARTED", - "primary": true + "source_node": { + "name": "32a9c755b09e", + "uuid": "JGcyPUWaTiOW2Ri0hDUC-A" + }, + "state": "STARTED" } }, - "service": { - "address": "http://elasticsearch:9200", - "name": "elasticsearch", - "type": "elasticsearch" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "elasticsearch.shard" + "event": { + "agent_id_status": "verified", + "dataset": "elasticsearch.shard", + "duration": 17200300, + "ingested": "2022-09-02T14:19:49Z", + "module": "elasticsearch" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.11.10-arch1-1", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "type": "linux", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "172.18.0.7" ], - "name": "docker-fleet-agent", - "id": "8979eb4aa312c3dccea3823dd92f92f5", "mac": [ "02:42:ac:12:00:07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 10000, - "name": "shard" + "name": "shard", + "period": 10000 }, - "event": { - "duration": 4139652, - "agent_id_status": "verified", - "ingested": "2021-07-30T14:41:18.844042490Z", - "module": "elasticsearch", - "dataset": "elasticsearch.shard" + "service": { + "address": "http://elastic-package-service_elasticsearch_1:9200", + "type": "elasticsearch" } } ``` @@ -1613,12 +2302,15 @@ An example event for `shard` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | +| cluster_uuid | | alias | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | +| elasticsearch.cluster.stats.state.state_uuid | | keyword | | elasticsearch.index.name | | keyword | | elasticsearch.node.id | Node ID | keyword | | elasticsearch.node.master | Is the node the master node? | boolean | @@ -1631,3 +2323,20 @@ An example event for `shard` looks as following: | elasticsearch.shard.source_node.name | | keyword | | elasticsearch.shard.source_node.uuid | | keyword | | elasticsearch.shard.state | The state of this shard. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| service.address | Service address | keyword | +| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| shard.index | | alias | +| shard.node | | alias | +| shard.primary | | alias | +| shard.shard | | alias | +| shard.state | | alias | +| source_node.name | | alias | +| source_node.uuid | | alias | +| timestamp | | alias | + diff --git a/packages/elasticsearch/1.1.0-preview1/img/logo_elasticsearch.svg b/packages/elasticsearch/1.1.0-preview1/img/logo_elasticsearch.svg index 20a620d162..2d9238bc1c 100755 --- a/packages/elasticsearch/1.1.0-preview1/img/logo_elasticsearch.svg +++ b/packages/elasticsearch/1.1.0-preview1/img/logo_elasticsearch.svg @@ -1,7 +1,5 @@ - - - - - - + + + + diff --git a/packages/elasticsearch/1.1.0-preview1/manifest.yml b/packages/elasticsearch/1.1.0-preview1/manifest.yml index 9f6ed9a508..2b3fc79740 100755 --- a/packages/elasticsearch/1.1.0-preview1/manifest.yml +++ b/packages/elasticsearch/1.1.0-preview1/manifest.yml @@ -1,7 +1,6 @@ name: elasticsearch title: Elasticsearch -version: 1.0.0 -release: experimental +version: 1.1.0-preview1 description: Elasticsearch Integration type: integration icons: