diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 05d336127f5..236b476e1ce 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -15772,6 +15772,536 @@ Elasticsearch module +*`timestamp`*:: ++ +-- +type: alias + +alias to: @timestamp + +-- + +*`cluster_uuid`*:: ++ +-- +type: alias + +alias to: elasticsearch.cluster.id + +-- + + +*`source_node.uuid`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.id + +-- + +*`source_node.name`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.name + +-- + + + + +*`node_stats.fs.total.available_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.fs.summary.available.bytes + +-- + + + +*`node_stats.fs.io_stats.total.operations`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.fs.io_stats.total.operations.count + +-- + +*`node_stats.fs.io_stats.total.read_operations`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.fs.io_stats.total.read.operations.count + +-- + +*`node_stats.fs.io_stats.total.write_operations`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.fs.io_stats.total.write.operations.count + +-- + + + +*`node_stats.indices.indexing.index_time_in_millis`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.indexing.index_time.ms + +-- + +*`node_stats.indices.indexing.index_total`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.indexing.index_total.count + +-- + +*`node_stats.indices.indexing.throttle_time_in_millis`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.indexing.throttle_time.ms + +-- + + +*`node_stats.indices.fielddata.memory_size_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.fielddata.memory.bytes + +-- + + +*`node_stats.indices.query_cache.memory_size_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.query_cache.memory.bytes + +-- + + +*`node_stats.indices.request_cache.memory_size_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.request_cache.memory.bytes + +-- + + +*`node_stats.indices.search.query_time_in_millis`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.search.query_time.ms + +-- + +*`node_stats.indices.search.query_total`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.search.query_total.count + +-- + + +*`node_stats.indices.segments.count`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.count + +-- + +*`node_stats.indices.segments.doc_values_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.doc_values.memory.bytes + +-- + +*`node_stats.indices.segments.fixed_bit_set_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes + +-- + +*`node_stats.indices.segments.index_writer_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.index_writer.memory.bytes + +-- + +*`node_stats.indices.segments.memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.memory.bytes + +-- + +*`node_stats.indices.segments.norms_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.norms.memory.bytes + +-- + +*`node_stats.indices.segments.points_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.points.memory.bytes + +-- + +*`node_stats.indices.segments.stored_fields_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes + +-- + +*`node_stats.indices.segments.term_vectors_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes + +-- + +*`node_stats.indices.segments.terms_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.terms.memory.bytes + +-- + +*`node_stats.indices.segments.version_map_memory_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.indices.segments.version_map.memory.bytes + +-- + + + + + +*`node_stats.jvm.gc.collectors.old.collection_count`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.jvm.gc.collectors.old.collection.count + +-- + +*`node_stats.jvm.gc.collectors.old.collection_time_in_millis`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms + +-- + + +*`node_stats.jvm.gc.collectors.young.collection_count`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.jvm.gc.collectors.young.collection.count + +-- + +*`node_stats.jvm.gc.collectors.young.collection_time_in_millis`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms + +-- + + +*`node_stats.jvm.mem.heap_max_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.jvm.mem.heap.max.bytes + +-- + +*`node_stats.jvm.mem.heap_used_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.jvm.mem.heap.used.bytes + +-- + +*`node_stats.jvm.mem.heap_used_percent`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.jvm.mem.heap.used.pct + +-- + +*`node_stats.node_id`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.id + +-- + + + + +*`node_stats.os.cpu.load_average.1m`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cpu.load_avg.1m + +-- + + + +*`node_stats.os.cgroup.cpuacct.usage_nanos`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns + +-- + + +*`node_stats.os.cgroup.cpu.cfs_quota_micros`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us + +-- + + +*`node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count + +-- + +*`node_stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count + +-- + +*`node_stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns + +-- + + +*`node_stats.os.cgroup.memory.control_group`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cgroup.memory.control_group + +-- + +*`node_stats.os.cgroup.memory.limit_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cgroup.memory.limit.bytes + +-- + +*`node_stats.os.cgroup.memory.usage_in_bytes`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.os.cgroup.memory.usage.bytes + +-- + + + +*`node_stats.process.cpu.percent`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.process.cpu.pct + +-- + + + +*`node_stats.thread_pool.bulk.queue`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.bulk.queue.count + +-- + +*`node_stats.thread_pool.bulk.rejected`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.bulk.rejected.count + +-- + + +*`node_stats.thread_pool.get.queue`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.get.queue.count + +-- + +*`node_stats.thread_pool.get.rejected`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.get.rejected.count + +-- + + +*`node_stats.thread_pool.index.queue`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.index.queue.count + +-- + +*`node_stats.thread_pool.index.rejected`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.index.rejected.count + +-- + + +*`node_stats.thread_pool.search.queue`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.search.queue.count + +-- + +*`node_stats.thread_pool.search.rejected`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.search.rejected.count + +-- + + +*`node_stats.thread_pool.write.queue`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.write.queue.count + +-- + +*`node_stats.thread_pool.write.rejected`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.stats.thread_pool.write.rejected.count + +-- + [float] === elasticsearch @@ -15808,6 +16338,7 @@ type: keyword -- + *`elasticsearch.node.id`*:: + -- @@ -15828,6 +16359,26 @@ type: keyword -- +*`elasticsearch.node.master`*:: ++ +-- +Is the node the master node? + + +type: boolean + +-- + +*`elasticsearch.node.mlockall`*:: ++ +-- +Is mlockall enabled on the node? + + +type: boolean + +-- + [float] === ccr @@ -16413,155 +16964,333 @@ node -*`elasticsearch.node.version`*:: +*`elasticsearch.node.version`*:: ++ +-- +Node version. + + +type: keyword + +-- + +[float] +=== jvm + +JVM Info. + + + +*`elasticsearch.node.jvm.version`*:: ++ +-- +JVM version. + + +type: keyword + +-- + +*`elasticsearch.node.jvm.memory.heap.init.bytes`*:: ++ +-- +Heap init used by the JVM in bytes. + + +type: long + +format: bytes + +-- + +*`elasticsearch.node.jvm.memory.heap.max.bytes`*:: ++ +-- +Heap max used by the JVM in bytes. + + +type: long + +format: bytes + +-- + +*`elasticsearch.node.jvm.memory.nonheap.init.bytes`*:: ++ +-- +Non-Heap init used by the JVM in bytes. + + +type: long + +format: bytes + +-- + +*`elasticsearch.node.jvm.memory.nonheap.max.bytes`*:: ++ +-- +Non-Heap max used by the JVM in bytes. + + +type: long + +format: bytes + +-- + +*`elasticsearch.node.process.mlockall`*:: ++ +-- +If process locked in memory. + + +type: boolean + +-- + +[float] +=== node.stats + +node_stats + + + +[float] +=== indices + +Node indices stats + + + +*`elasticsearch.node.stats.indices.docs.count`*:: ++ +-- +Total number of existing documents. + + +type: long + +-- + +*`elasticsearch.node.stats.indices.docs.deleted`*:: ++ +-- +Total number of deleted documents. + + +type: long + +-- + +*`elasticsearch.node.stats.indices.segments.count`*:: ++ +-- +Total number of segments. + + +type: long + +-- + +*`elasticsearch.node.stats.indices.segments.memory.bytes`*:: ++ +-- +Total size of segments in bytes. + + +type: long + +format: bytes + +-- + +*`elasticsearch.node.stats.indices.store.size.bytes`*:: ++ +-- +Total size of the store in bytes. + + +type: long + +-- + + +*`elasticsearch.node.stats.indices.fielddata.memory.bytes`*:: ++ +-- +type: long + +format: bytes + +-- + + +*`elasticsearch.node.stats.indices.indexing.index_time.ms`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.indices.indexing.index_total.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.indices.indexing.throttle_time.ms`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.indices.query_cache.memory.bytes`*:: + -- -Node version. - +type: long -type: keyword +format: bytes -- -[float] -=== jvm - -JVM Info. +*`elasticsearch.node.stats.indices.request_cache.memory.bytes`*:: ++ +-- +type: long +format: bytes -*`elasticsearch.node.jvm.version`*:: -+ -- -JVM version. -type: keyword +*`elasticsearch.node.stats.indices.search.query_time.ms`*:: ++ +-- +type: long -- -*`elasticsearch.node.jvm.memory.heap.init.bytes`*:: +*`elasticsearch.node.stats.indices.search.query_total.count`*:: + -- -Heap init used by the JVM in bytes. +type: long + +-- +*`elasticsearch.node.stats.indices.segments.doc_values.memory.bytes`*:: ++ +-- type: long format: bytes -- -*`elasticsearch.node.jvm.memory.heap.max.bytes`*:: +*`elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes`*:: + -- -Heap max used by the JVM in bytes. - - type: long format: bytes -- -*`elasticsearch.node.jvm.memory.nonheap.init.bytes`*:: +*`elasticsearch.node.stats.indices.segments.index_writer.memory.bytes`*:: + -- -Non-Heap init used by the JVM in bytes. - - type: long format: bytes -- -*`elasticsearch.node.jvm.memory.nonheap.max.bytes`*:: +*`elasticsearch.node.stats.indices.segments.norms.memory.bytes`*:: + -- -Non-Heap max used by the JVM in bytes. - - type: long format: bytes -- -*`elasticsearch.node.process.mlockall`*:: +*`elasticsearch.node.stats.indices.segments.points.memory.bytes`*:: + -- -If process locked in memory. - +type: long -type: boolean +format: bytes -- -[float] -=== node.stats - -node_stats - +*`elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes`*:: ++ +-- +type: long +format: bytes -[float] -=== indices +-- -Node indices stats +*`elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes`*:: ++ +-- +type: long +format: bytes +-- -*`elasticsearch.node.stats.indices.docs.count`*:: +*`elasticsearch.node.stats.indices.segments.terms.memory.bytes`*:: + -- -Total number of existing documents. - - type: long +format: bytes + -- -*`elasticsearch.node.stats.indices.docs.deleted`*:: +*`elasticsearch.node.stats.indices.segments.version_map.memory.bytes`*:: + -- -Total number of deleted documents. - - type: long --- +format: bytes -*`elasticsearch.node.stats.indices.segments.count`*:: -+ -- -Total number of segments. +*`elasticsearch.node.stats.jvm.mem.heap.max.bytes`*:: ++ +-- type: long --- +format: bytes -*`elasticsearch.node.stats.indices.segments.memory.bytes`*:: -+ -- -Total size of segments in bytes. +*`elasticsearch.node.stats.jvm.mem.heap.used.bytes`*:: ++ +-- type: long format: bytes -- -*`elasticsearch.node.stats.indices.store.size.bytes`*:: +*`elasticsearch.node.stats.jvm.mem.heap.used.pct`*:: + -- -Total size of the store in bytes. - - type: long +format: percent + -- [float] @@ -16781,8 +17510,9 @@ type: long -- + [float] -=== fs.summary +=== summary File system summary @@ -16821,6 +17551,187 @@ format: bytes -- + + +*`elasticsearch.node.stats.fs.io_stats.total.operations.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.fs.io_stats.total.read.operations.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.fs.io_stats.total.write.operations.count`*:: ++ +-- +type: long + +-- + + + +*`elasticsearch.node.stats.os.cpu.load_avg.1m`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.os.cgroup.memory.control_group`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.node.stats.os.cgroup.memory.limit.bytes`*:: ++ +-- +type: long + +format: bytes + +-- + +*`elasticsearch.node.stats.os.cgroup.memory.usage.bytes`*:: ++ +-- +type: long + +format: bytes + +-- + +*`elasticsearch.node.stats.process.cpu.pct`*:: ++ +-- +type: long + +format: percent + +-- + + + +*`elasticsearch.node.stats.thread_pool.bulk.queue.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.thread_pool.bulk.rejected.count`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.thread_pool.get.queue.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.thread_pool.get.rejected.count`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.thread_pool.index.queue.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.thread_pool.index.rejected.count`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.thread_pool.search.queue.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.thread_pool.search.rejected.count`*:: ++ +-- +type: long + +-- + + +*`elasticsearch.node.stats.thread_pool.write.queue.count`*:: ++ +-- +type: long + +-- + +*`elasticsearch.node.stats.thread_pool.write.rejected.count`*:: ++ +-- +type: long + +-- + [float] === cluster.pending_task diff --git a/metricbeat/module/elasticsearch/_meta/fields.yml b/metricbeat/module/elasticsearch/_meta/fields.yml index 3614c9e6319..1ae2a10ed7a 100644 --- a/metricbeat/module/elasticsearch/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/_meta/fields.yml @@ -6,6 +6,276 @@ settings: ["ssl", "http"] short_config: false fields: + - name: timestamp + type: alias + path: "@timestamp" + - name: cluster_uuid + type: alias + path: elasticsearch.cluster.id + - name: source_node + type: group + fields: + - name: uuid + type: alias + path: elasticsearch.node.id + - name: name + type: alias + path: elasticsearch.node.name + - name: node_stats + type: group + fields: + - name: fs + type: group + fields: + - name: total + type: group + fields: + - name: available_in_bytes + path: elasticsearch.node.stats.fs.summary.available.bytes + type: alias + - name: io_stats + type: group + fields: + - name: total + type: group + fields: + - name: operations + path: elasticsearch.node.stats.fs.io_stats.total.operations.count + type: alias + - name: read_operations + path: elasticsearch.node.stats.fs.io_stats.total.read.operations.count + type: alias + - name: write_operations + path: elasticsearch.node.stats.fs.io_stats.total.write.operations.count + type: alias + - name: indices + type: group + fields: + - name: indexing + type: group + fields: + - name: index_time_in_millis + path: elasticsearch.node.stats.indices.indexing.index_time.ms + type: alias + - name: index_total + path: elasticsearch.node.stats.indices.indexing.index_total.count + type: alias + - name: throttle_time_in_millis + path: elasticsearch.node.stats.indices.indexing.throttle_time.ms + type: alias + - name: fielddata + type: group + fields: + - name: memory_size_in_bytes + path: elasticsearch.node.stats.indices.fielddata.memory.bytes + type: alias + - name: query_cache + type: group + fields: + - name: memory_size_in_bytes + path: elasticsearch.node.stats.indices.query_cache.memory.bytes + type: alias + - name: request_cache + type: group + fields: + - name: memory_size_in_bytes + path: elasticsearch.node.stats.indices.request_cache.memory.bytes + type: alias + - name: search + type: group + fields: + - name: query_time_in_millis + path: elasticsearch.node.stats.indices.search.query_time.ms + type: alias + - name: query_total + path: elasticsearch.node.stats.indices.search.query_total.count + type: alias + - name: segments + type: group + fields: + - name: count + path: elasticsearch.node.stats.indices.segments.count + type: alias + - name: doc_values_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.doc_values.memory.bytes + type: alias + - name: fixed_bit_set_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes + type: alias + - name: index_writer_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.index_writer.memory.bytes + type: alias + - name: memory_in_bytes + path: elasticsearch.node.stats.indices.segments.memory.bytes + type: alias + - name: norms_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.norms.memory.bytes + type: alias + - name: points_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.points.memory.bytes + type: alias + - name: stored_fields_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes + type: alias + - name: term_vectors_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes + type: alias + - name: terms_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.terms.memory.bytes + type: alias + - name: version_map_memory_in_bytes + path: elasticsearch.node.stats.indices.segments.version_map.memory.bytes + type: alias + - name: jvm + type: group + fields: + - name: gc + type: group + fields: + - name: collectors + type: group + fields: + - name: old + type: group + fields: + - name: collection_count + path: elasticsearch.node.stats.jvm.gc.collectors.old.collection.count + type: alias + - name: collection_time_in_millis + path: elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms + type: alias + - name: young + type: group + fields: + - name: collection_count + path: elasticsearch.node.stats.jvm.gc.collectors.young.collection.count + type: alias + - name: collection_time_in_millis + path: elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms + type: alias + - name: mem + type: group + fields: + - name: heap_max_in_bytes + path: elasticsearch.node.stats.jvm.mem.heap.max.bytes + type: alias + - name: heap_used_in_bytes + path: elasticsearch.node.stats.jvm.mem.heap.used.bytes + type: alias + - name: heap_used_percent + path: elasticsearch.node.stats.jvm.mem.heap.used.pct + type: alias + - name: node_id + path: elasticsearch.node.id + type: alias + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: load_average + type: group + fields: + - name: 1m + path: elasticsearch.node.stats.os.cpu.load_avg.1m + type: alias + - name: cgroup + type: group + fields: + - name: cpuacct + type: group + fields: + - name: usage_nanos + path: elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns + type: alias + - name: cpu + type: group + fields: + - name: cfs_quota_micros + path: elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us + type: alias + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + path: elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count + type: alias + - name: number_of_times_throttled + path: elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count + type: alias + - name: time_throttled_nanos + path: elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns + type: alias + - name: memory + type: group + fields: + - name: control_group + path: elasticsearch.node.stats.os.cgroup.memory.control_group + type: alias + - name: limit_in_bytes + path: elasticsearch.node.stats.os.cgroup.memory.limit.bytes + type: alias + - name: usage_in_bytes + path: elasticsearch.node.stats.os.cgroup.memory.usage.bytes + type: alias + - name: process + type: group + fields: + - name: cpu + type: group + fields: + - name: percent + path: elasticsearch.node.stats.process.cpu.pct + type: alias + - name: thread_pool + type: group + fields: + - name: bulk + type: group + fields: + - name: queue + path: elasticsearch.node.stats.thread_pool.bulk.queue.count + type: alias + - name: rejected + path: elasticsearch.node.stats.thread_pool.bulk.rejected.count + type: alias + - name: get + type: group + fields: + - name: queue + path: elasticsearch.node.stats.thread_pool.get.queue.count + type: alias + - name: rejected + path: elasticsearch.node.stats.thread_pool.get.rejected.count + type: alias + - name: index + type: group + fields: + - name: queue + path: elasticsearch.node.stats.thread_pool.index.queue.count + type: alias + - name: rejected + path: elasticsearch.node.stats.thread_pool.index.rejected.count + type: alias + - name: search + type: group + fields: + - name: queue + path: elasticsearch.node.stats.thread_pool.search.queue.count + type: alias + - name: rejected + path: elasticsearch.node.stats.thread_pool.search.rejected.count + type: alias + - name: write + type: group + fields: + - name: queue + path: elasticsearch.node.stats.thread_pool.write.queue.count + type: alias + - name: rejected + path: elasticsearch.node.stats.thread_pool.write.rejected.count + type: alias - name: elasticsearch type: group description: > @@ -14,23 +284,30 @@ type: keyword description: > Elasticsearch cluster name. - - name: cluster.id type: keyword description: > Elasticsearch cluster id. - - name: cluster.state.id type: keyword description: > Elasticsearch state id. - - - name: node.id - type: keyword - description: > - Node ID - - - name: node.name - type: keyword - description: > - Node name. + - name: node + type: group + fields: + - name: id + type: keyword + description: > + Node ID + - name: name + type: keyword + description: > + Node name. + - name: master + type: boolean + description: > + Is the node the master node? + - name: mlockall + type: boolean + description: > + Is mlockall enabled on the node? diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 52ee233d88c..e7a083e5ff6 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -54,7 +54,6 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { "index_recovery", "index_summary", "ml_job", - "node_stats", "shard", } return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName)) @@ -137,14 +136,14 @@ func GetClusterID(http *helper.HTTP, uri string, nodeID string) (string, error) return info.ClusterID, nil } -// IsMaster checks if the given node host is a master node. +// isMaster checks if the given node host is a master node. // // The detection of the master is done in two steps: // * Fetch node name from /_nodes/_local/name // * Fetch current master name from cluster state /_cluster/state/master_node // // The two names are compared -func IsMaster(http *helper.HTTP, uri string) (bool, error) { +func isMaster(http *helper.HTTP, uri string) (bool, error) { node, err := getNodeName(http, uri) if err != nil { @@ -415,49 +414,6 @@ func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSet return ret, nil } -// IsMLockAllEnabled returns if the given Elasticsearch node has mlockall enabled -func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) { - content, err := fetchPath(http, resetURI, "_nodes/"+nodeID, "filter_path=nodes.*.process.mlockall") - if err != nil { - return false, err - } - - var response map[string]map[string]map[string]map[string]bool - err = json.Unmarshal(content, &response) - if err != nil { - return false, err - } - - for _, nodeInfo := range response["nodes"] { - mlockall := nodeInfo["process"]["mlockall"] - return mlockall, nil - } - - return false, fmt.Errorf("could not determine if mlockall is enabled on node ID = %v", nodeID) -} - -// GetMasterNodeID returns the ID of the Elasticsearch cluster's master node -func GetMasterNodeID(http *helper.HTTP, resetURI string) (string, error) { - content, err := fetchPath(http, resetURI, "_nodes/_master", "filter_path=nodes.*.name") - if err != nil { - return "", err - } - - var response struct { - Nodes map[string]interface{} `json:"nodes"` - } - - if err := json.Unmarshal(content, &response); err != nil { - return "", err - } - - for nodeID, _ := range response.Nodes { - return nodeID, nil - } - - return "", errors.New("could not determine master node ID") -} - // PassThruField copies the field at the given path from the given source data object into // the same path in the given target data object. func PassThruField(fieldPath string, sourceData, targetData common.MapStr) error { diff --git a/metricbeat/module/elasticsearch/elasticsearch_test.go b/metricbeat/module/elasticsearch/elasticsearch_test.go index 9b9682aef3b..f5b19f9b22d 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_test.go @@ -45,12 +45,12 @@ func TestXPackEnabledMetricsets(t *testing.T) { } metricSets := mbtest.NewReportingMetricSetV2Errors(t, config) - require.Len(t, metricSets, 9) + require.Len(t, metricSets, 8) for _, ms := range metricSets { name := ms.Name() switch name { case "ccr", "enrich", "cluster_stats", "index", "index_recovery", - "index_summary", "ml_job", "node_stats", "shard": + "index_summary", "ml_job", "shard": default: t.Errorf("unexpected metricset name = %v", name) } diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go index aadc37a05c3..b289382e5a7 100644 --- a/metricbeat/module/elasticsearch/fields.go +++ b/metricbeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded gzipped contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzsXN1u47oRvvdTDHK1B0j0AL7ozfacbQpsuuhmCxRFoaWlsc2EPwpJOXafviAp2bJMWbIkx14c+y6W9c03v5yhqDzAK26mgIxoQxONRCXLCYChhuEU7n6vfn83AUhRJ4pmhkoxhb9MAAD2fgNcpjnDCYBChkTjFBZkAqDRGCoWegr/udOa3d3D3dKY7O6/9tpSKhMnUszpYgpzwrS9f06RpXrqRDyAIBwPadqP2WRWiJJ5VnwT4LgPV4VMWK4Nqsj+tb1Yor7i5l2qtPJ9ENt/9u1Q4Dop0aRRLE3PIZSmR0RqQwyOLNhhhsUKmQ6X9iRThMe/NqAP953D9646MFuiDrCr0daC/FlJrR9KxyjMGE2I/aGzma78dj9hyk89bqvUGJIU1d6lJoZNUFU4KlJcH1xttmcH7cvPE+EIcl4wbpBU8uBkHWt8i4VsJMOkWPRj8pWsKc85aHzLUSQIIuczVJaczFB510gBZoklW70kNb1LpnPJmHz/tVxQcm5xglM68sY5gxuetla3hnbC4J2aJfWWP85t6ygdvytqDIqzMtyJ87wwhU9lImP6G1BhpGO9Na3XZ64kPx5HVaUM5RhrKhKMbWmNFZI04voMmj1TjvdABXB9D07iPnsrHuZokiUeKNFIf8HkjLA4WWLymkkqzBmIf3EyYCcDVoTlaNN13/RH1z49qJwXhXx48bYIed29x/K8xUBVZrmGTwuFKO5hg9Yw96Aw/S0KErFLaJhHqIK1sLALqXYcqGsPok4FcOsmmQcC50jYtAbNszSEVWq8U9bGfhkRDUw4sVdHpbIrKB78ARld0BnDzqRSYsiZKFnoFh6VNYom40XMo4e74pgpFO7gIFd8Dkt2s206sPnu1pJm67Q3FWELjbCO1OzktT9iph2lTFFOFD2IonPQ8rI27fS2XZ21p02IiCOXahPNNibAdEiIfXXAkGu7wkpVEXmwdKFQdDvsQp8163eHMMKS9ZZjjpGm/8Ng8gdM0bZa7GLHz5HKduXaOC/ZpsNLDJJRyKXBuLxjtBY8yZXCs+TLZ49cnThyow0RKRWLQp+tBZpzx9j4Pm/D20wLcI1JbjAtGke7ZNjIUibPwn4qb4i9i1FHIQXGjyCzJKZIHsiUTFBrqTQsyQq7KdE0k52cf3WA0xOvtsEBwzrFR8unttsBLQHWP59Smejoo5agVCY5R7ErIM72zbnkyKXI0GBorB6dnpd0Mk1tpPKFt2EtaqE6l4oTM4WmmzurYimUQ7vjbBVwqEfI48Lp+mFB4IltxbYTO7rMf5Rpd/x5pTmYbfwGSUE1YO5wvYoUJnKFanPZwkXrWdW3zvsuuFTK7TSH5Fn48QplTahFCostWsyg5JmUDEl9h6pF8rPKEWitdw3L1oYsRtT5n6W2DjcQZlXZhqgFmqjBy73kPztIvyQ3etmLXUpdLycjCbbIQNJUodbwKZE5S2GG8Pht+6VU7keWT8O2SkFy3KW7SnJ/AQ/HhsxVgqP657uDPO6fQuy4/qkKHsM/Bclx/VMlGX6c5KuzznmtYFygODcN4bdebxR6t17v1uu18j+t14PblHbL3Fvm/oKZu32uxKIXORuy7nN2jomsV7fzQ9C3HIEzeJGz5m7QEDNii/V3OfOQYWkpMSR2MayjYtsP09jOcCqNQ7HddyL9VoL7Z2e4OgzjECcqVoTRNE6JwVH5PC+rZ1m8wtqdqQCkZokKCHCqNRULSwh9lNhGmfi/3Yap76WFNLafzojSmAZmjIOwtg3vkKCu3X96WK9QaSrrU/aAMHMnwgrUsFdfVrzz8tsW0//6Co9iLrs9+WzTuk3zDoRKUkEDVBkURXmJJIuooOZi5flvSDKwDPYqstWhfe2rKsHJ+rI6cLLur4KQ4vKueJLiYQR3lLpc0iNbVbp7ZTddu/Uh4kwmr4SFW/Vem4OP8xIcLDam7iyXN1qwMg8/9mRR4uEPkEc+OuJqNK2cH6kjX9XUg2uqjVt9y7nieieeK5oTfrUJoRy+WsZ4GHFw7DUYOqEtNexlxa1Ro0xKNlrW2tJZDE8Wt1fiStacFuFzVp2s9A+WHlI79FszP9g/xt7oTmhzKXQIybpSDT9xZ96bQrBKOEPyeiWMvyF57Uo5vh5DO9q8m7VtN3EltH/4xuZojdrIPEhkcM792wLfsu4aGN+y7tqyTudqRVey+Q2cAYn3vcC+5d41ML7l3qVzr7EDXiRRIhnDxEg1Whf85TNsQcNZ16EHLnkd2wMc2A7vJMAi6VsYmmZF6OL0DkS3Bah3ZLUI2euCzmx13xD9ue0ezMW5DpybgSFJ+AdlCHqjDXIIQ7cloXsKf5Edh61VFF7meXNJgKwIZWTGPpZF/c3ODN37CrEh+nVSF33CTufPEOBPSKQwhAoNBIoLYC9UkapJ2mdrVKMysVRNb/Sf/kTw0UHCIWTlAJhU1ITTqc8T0QDc/nG7oKQBJ+0i+EMqwDXhGbMK5eaBkyyjNep771hTEfvXig6qVu9Hr5S7nTQHexCh9de+Tw7J4m1yFzyDYuxsx5TNkmqg2u0rdjiyHPyvAuM893ZMjp+WHvMwwrPbSCUGu8hWyGRCjC0qoX9bMgIVd+608g8ViC6FFm/yR5P/BwAA//+sJJat" + return "eJzsXd2P2zYSf89fQexTCyQC7nUfrgf02t4WSK64pgUOh4NKS2ObWUr0ktRmt3/9gdSHKZlfkqjEya0fgqxt/eY3w+Hwazh+g+7h+RYBxUKSQgDmxfEVQpJICrfo5gfz/ZtXCJUgCk5OkrD6Fv31FUIIjb6DKlY2FF4hxIECFnCLDvgVQgKkJPVB3KL/3AhBb16jm6OUp5v/qs+OjMu8YPWeHG7RHlOhnt8ToKW41SLeoBpXcIskqUBIXJ30uwjJ5xPcIkwJFt07JyyPt+jmb8M3b0YABW2EBJ43DSkDGCOTZN2DWfdYjydYwwvIa1bCCO7AWdOTNBUxnzU42Hm4uChpmfFsD6j+XQ44PD3AsRJyIbEUs1Xbiwse5mO2R0fNzCSmo09cKC4kEw0/YkLxjkJO6nz3LEFcfNVjFm2AbC8y0VQV5s/ZAJe5sOyGPxMibGTW9Rra7OVH9KGayOwEHKvObtMzzm69tplmmZ0Rs4I1tXTgumw4JcgBl3l6lgo2NdWPnEjYgKvGXUN28Mu6JAWs67qkLuGJ1Idkvq0BcxXOVfetCKVkQf/tVMt6etkZNqvm9GELNUfnW8hIN6mrBWNYySNnUlJIb7MR8jyzDQODau0SS5zMPyqoGH/OBflzTXzvdR34ZS3u4hj/0AB/zgtcHOFKVTUYrlWWw0MDQl61uiOOaxUe5slpNG2bIlV/7T45gy6PcB3Gugg35rMovp3tfqigTjhzchGJVq3lsypgl6zIHzFtQOSdy6/39oHYGXypz6NR+H6CMt8RmQuQG7Ad4acg3A6qeobEN+Brwqegm55hAlI149UWrqlxUxA8MVLLLRi2wCkoCsk4lHkbjjZgOsJPQVgCr/JHKCTjW/A14VPR3YpnEoKPwAVhdV7h0wY0DfQlZHuSHx6rVYu/Q5FwYKa0dY9t9jVo6V0gu7BD+BYVVLv4VuQo3N4fHqvsUGRnm2SMltkZ37viRxFbFA7awRlpKv7WCWos+Z74M2su9h3GGNfeqlqDL7pdLzRY1LLGdChZQDmCirz4aUXYVepWUGUKKqvw06oBQfNpBJSpCCmsRIxOwAtYsi66JHQqYpdFo2OH0clIzBmIH5Ot29QsTk0yP6QMlzl+BI4P060SP7AP3BTwl2mf6V+BtmMiK05N1vE7ZE6cUK8tbOxXzARODS48XrTGVo3AB8hrXLOFu/HKaJpA1tHMNGTm3N2P6YiX7pZG22Iv8oeGSZxXpOBJVM6Kvcg0ZtYsURmN1kjYf2qRYvium2oHPGf7HCg+dcGOsHLFKDg2iHovm2AnHcfPGugz7rzflXdNaJdpMMFOqoGefgzQ3u63nLzB3dkZQ8TH2zLb9ElWS85o7vPtaAN0S78YzNhOSUlFpG+KsoSgBnXOVebQawN4YnptCJ9Lb9iN4qwAcT0TjsWTuU4R3a3mT+PkUR/Nnxijq0yxa+h9Mls8NNDYZl0BSxi6ZIpPpnFWbfpz+ACFtAbtuWR6qMWnKgeYPvVZLXwAeTUGVlxW21efElyThTWhq7Fxy2a1ldOfya418/ng8yrs3H262tD6uOua7NwmQF2LmVs2s63ck5hm4tqta0nDdWdj9rmr1vzQe3j+yLipsxW7fY1TfTtcLSVzSrVs0iSQSUq3RNUm1s2hFXI1plWqkftrbytb05gAZOpwLrIBwur1jpWA7v5ulTNp/hSSxi1vCquwagqruB1jFHA9T9ydQPII2tj6Py2+/vs7OwHKintM7WnESyn0oAhqvKNQIlYPtL679MeCBx3DI/N7zoR40zs8hxMlhc7wRNPs4XGqff/y+RwFXDqaZ3GSpjPI2f0roH3/eocrQGzfMXZIOnvdUy7gIa+Zkwxl1kOiCCZv8ROpmgoJeGigLqDbiVHkhuTb3iE6tuKIJ3oPmTSMUvbxy2qCnnOgEbTSWWucDZrh3WB1ZWgtDH0k8khay/u5nbOkddaOhGkMSMvwLK7lBSX6pu/IUH6LSC2ZZj2YttVnz1nl9yM03U8TpC4gV2NWrpPXPTl/yzV7Typ4jUiNKvEaaYlj9ko82oMsjnChhJP+gbIdpnlxhOJeZ9xsQPwnLQOdZSCdFKe669j03klFeFPHF867QL4+eCuEi932FUO5yawR6JsDB6hfo2dQhnmNOJTf2gd5NezF52JGTCiE5kD0vCuLCoD+RE6P2wSd5j2TmBoxXiurfL/3CAcT68xnHZVzQGnB3wAlB7KjEE3KknKfipKCDvBwXzBBazzmroW7Yp/pFI5oIB18LkO274AhyOZXPZa4rbM8PXr1ODKxU6u9x0zm9jqpMCeeLId0tFpZz2F6F9db/Fl4a1zsrQZGjVAjLOOGyIuhC2pORjtis8esHzRCgiGr3ZwR5E/7CtRiitBocfaddoHe3e/QraQmHa1EKxkOFZOQ908km4IXDef2k5a1jvl9i2yuOBopJK5LUh86fQYLuPuO/55okgmvmxaCJygaCWU3cdRLeSExl83J3k79A3nbxNBdeNzcg+QRy67z9Kd5jAt0xI8Qp4RrTTa7/00B5ne8tJs+d4qPZ9cn7cXtkhWhmzXpYn3JikYnUyNzIenuS5pcCRTsG8bJ6bWSZtPUFwN04A3kBjqo7hmvsLxFroejVVEU+kW75qwU0Kge8nHXq9JZuSU2iA0Ti0q239q0Z/6VMTnYPbcbJB1Vi7nt8SrjULBHGOW7fIbA5dgVnx/n21lwr9RoC9+Up+DTBcqJUIVkF9tNMa2SF+1Sv+cNIDKZu9plC3mZg7pC53/12mpci5uZsiXmB5BZyrOP9xqyHZKdrdyKPTJhT3lYLVghI1yWHIRA3xSsoSXaAbr7ZXiTcf0lxcexrdKRTDt0myTHA7jdN3Rhm6Tt86uG9LdPJzZt+5iCU7RPRzJt+5gk7QeqbXTuytB83uDsWoS/zPWS0HuZ673M9YL858310Msq7aXnvvTcL7DnnhNasg9st2bcr+gWK7JFs53favLQAKoo+sB27tmgxI5Mw0VCf2a7FtIurcQSt3d6h9x3KHO1huOl9a7v0hXpLz14e3YGj5dubONE6kdMSZmXWEJSPu+PZi5Lq7DQORUIiDwCRxhVRAhSHxQhaL1ETZRx+7feMG3n0jWTaj59wlxAaVljXLh1VPqcR4HJ8/PduqudkM7NdGZch2pv1XGdBbfaEcJ+/v0tuqv3LO7kM6R1SPMIQj0pqwHQxaWq9qowqT33gbYOz/8AfEKKwSgiKx3CY5+pRPhG+MY6VPhpuQo1qz9/U7xj9ZsEzdHr8jlbZFAlvlUmN8iyDVJY9z04UthQ6lyu1mjWyLw+7emiijBaNvlImzqiYzQx8kemyFe16oEnIqQefft1xfWueK5onfClrRD6xVdgGY8SLhwXLQy10AhyrmKvKMnN6UCTBVVHwaYbbbxe1jROo0aoEHGEHhMob4XNGXAR5X4DaEb2j6MYLvrCXMFX6fZL08V6YTKNEqHasxFaTKDWOPU02m+icnyh1Qjlw4OKGWVn1ExNLHtO/dPEoqOqhyaWGVcPNLHQWaU9E8ueU6ZzA9GfWuaMMporJRs7QUPBtOi1TbBw+jbL3h5eLWY3CaGfqhdba5nMEmKrqTJt0hNjNNl69eff3/bHBgp30ZLVXvPU32wR64N/0vKS2uWiIHL243FdlMYNxko5vqJve7rWNybhE+D7K2H8C+D7WMr59Rha067irO2tcflpaf/Wbul5F8CuirSr+9y/FfBLr7sGxi+97tp6nWj4I3lk7rvnKzrerx32S9+7BsYvfe8a+p458R1VA081+f3p+/PPAdg7W8TU1yhKvtUs+CwBHYql8WDFxm0U0SHuLHaogJDR5Gdjq7fzoP9vu1v74j5dZvZlwnkY00PXfP1IKCDxLCRUHjExrdfuD3/SOBlo5z2HmBqyn45Q+HdrPxWrYffY/ju4KMlmkuuucxg9JMGUEvmrqyjGsuYpU/yvz86EnvVjsR7swQAJr89vVPY+9hcNvGdH1h81SMNv1i8HBM/L7FYMMw2xHckIF/2PYIwmmb5BJB/1GPqmwLkV+iP1MUXMLaG/UERsofsI+PEZ9PZuFFGnHkVkpI5Rw8XlUayhY8Y6c6kWKhufQPA0R9BenN0pJOYAw165Ha2K7ZYq7n5AHygapwd4qv56bTEGChbsDWD1OJfF1L9SRf0FJ78qVbfNlLkuXW3Vvb8SVT3/GYakrr7XCXQNo1xiYcat2dnPf9gA/9AjHya1QBh1HyD1gYlk7mMsSZcWwGXOuKvK7/xbQncaEl1CGpfCGSfSXstiyS0pC9zQI/VteaukFbfvM/Qj4wiecHWiSqFGvqnw6TRN/RzNvkidt459sbGz+DoWqXR27bgy/qD5pBTsbJfsKsxq51nlY5uVLpFHIhBpC35HlDGxVhpOcxdOM/FXUEl5QfG9Tq7GEmJkc6CswFIFFX1To05bLsQstt4VWcaiF9pV981e/S8AAP//Cti0jQ==" } diff --git a/metricbeat/module/elasticsearch/metricset.go b/metricbeat/module/elasticsearch/metricset.go index 22b4b2c6c49..7e8afbf268a 100644 --- a/metricbeat/module/elasticsearch/metricset.go +++ b/metricbeat/module/elasticsearch/metricset.go @@ -18,6 +18,7 @@ package elasticsearch import ( + "encoding/json" "fmt" "github.com/pkg/errors" @@ -65,6 +66,12 @@ func (h *Scope) Unpack(str string) error { return nil } +type MetricSetAPI interface { + Module() mb.Module + GetMasterNodeID() (string, error) + IsMLockAllEnabled(string) (bool, error) +} + // MetricSet can be used to build other metric sets that query RabbitMQ // management plugin type MetricSet struct { @@ -122,7 +129,7 @@ func (m *MetricSet) ShouldSkipFetch() (bool, error) { // If we're talking to a set of ES nodes directly, only collect stats from the master node so // we don't collect the same stats from every node and end up duplicating them. if m.Scope == ScopeNode { - isMaster, err := IsMaster(m.HTTP, m.GetServiceURI()) + isMaster, err := isMaster(m.HTTP, m.GetServiceURI()) if err != nil { return false, errors.Wrap(err, "error determining if connected Elasticsearch node is master") } @@ -136,3 +143,52 @@ func (m *MetricSet) ShouldSkipFetch() (bool, error) { return false, nil } + +// GetMasterNodeID returns the ID of the Elasticsearch cluster's master node +func (m *MetricSet) GetMasterNodeID() (string, error) { + http := m.HTTP + resetURI := m.GetServiceURI() + + content, err := fetchPath(http, resetURI, "_nodes/_master", "filter_path=nodes.*.name") + if err != nil { + return "", err + } + + var response struct { + Nodes map[string]interface{} `json:"nodes"` + } + + if err := json.Unmarshal(content, &response); err != nil { + return "", err + } + + for nodeID, _ := range response.Nodes { + return nodeID, nil + } + + return "", errors.New("could not determine master node ID") +} + +// IsMLockAllEnabled returns if the given Elasticsearch node has mlockall enabled +func (m *MetricSet) IsMLockAllEnabled(nodeID string) (bool, error) { + http := m.HTTP + resetURI := m.GetServiceURI() + + content, err := fetchPath(http, resetURI, "_nodes/"+nodeID, "filter_path=nodes.*.process.mlockall") + if err != nil { + return false, err + } + + var response map[string]map[string]map[string]map[string]bool + err = json.Unmarshal(content, &response) + if err != nil { + return false, err + } + + for _, nodeInfo := range response["nodes"] { + mlockall := nodeInfo["process"]["mlockall"] + return mlockall, nil + } + + return false, fmt.Errorf("could not determine if mlockall is enabled on node ID = %v", nodeID) +} diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data-xpack.json b/metricbeat/module/elasticsearch/node_stats/_meta/data-xpack.json deleted file mode 100644 index 2dbc5c5801b..00000000000 --- a/metricbeat/module/elasticsearch/node_stats/_meta/data-xpack.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "@timestamp": "2018-04-05T12:17:50.378Z", - "@metadata": { - "beat": "metricbeat", - "type": "doc", - "version": "7.0.0-alpha1" - }, - "cluster_uuid": "elasticsearch", - "interval_ms": 10000, - "type": "node_stats", - "source_node": { - "transport_address": "127.0.0.1:9300", - "ip": "127.0.0.1:9300", - "name": "0F564AX", - "uuid": "0F564AXWTwme40EvgjAyPg", - "host": "127.0.0.1" - }, - "node_stats": { - "node_id": "0F564AXWTwme40EvgjAyPg", - "mlockall": false, - "node_master": true - }, - "beat": { - "hostname": "ruflin", - "version": "7.0.0-alpha1", - "name": "ruflin" - }, - "indices": { - "fs": { - "total": { - "free_in_bytes": 20373749760, - "available_in_bytes": 20111605760, - "total_in_bytes": 249779191808 - } - }, - "indices": { - "indexing": { - "throttle_time_in_millis": 0, - "index_total": 147, - "index_time_in_millis": 3635 - }, - "search": { - "query_total": 16, - "query_time_in_millis": 261 - }, - "query_cache": { - "hit_count": 0, - "miss_count": 0, - "evictions": 0, - "memory_size_in_bytes": 0 - }, - "fielddata": { - "evictions": 0, - "memory_size_in_bytes": 0 - }, - "segments": { - "index_writer_memory_in_bytes": 0, - "memory_in_bytes": 51216, - "terms_memory_in_bytes": 39654, - "term_vectors_memory_in_bytes": 0, - "version_map_memory_in_bytes": 0, - "stored_fields_memory_in_bytes": 4072, - "points_memory_in_bytes": 406, - "fixed_bit_set_memory_in_bytes": 0, - "count": 13, - "doc_values_memory_in_bytes": 1900, - "norms_memory_in_bytes": 5184 - }, - "request_cache": { - "memory_size_in_bytes": 0, - "evictions": 0, - "hit_count": 0, - "miss_count": 3 - }, - "docs": { - "count": 139 - }, - "store": { - "size_in_bytes": 333573 - } - }, - "os": { - "cpu": {} - }, - "process": { - "max_file_descriptors": 10240, - "cpu": {}, - "open_file_descriptors": 190 - }, - "jvm": { - "mem": { - "heap_used_in_bytes": 225025520, - "heap_used_percent": 21, - "heap_max_in_bytes": 1038876672 - }, - "gc": { - "collectors": { - "young": { - "collection_time_in_millis": 1444, - "collection_count": 8 - }, - "old": { - "collection_count": 8, - "collection_time_in_millis": 1444 - } - } - } - }, - "thread_pool": { - "bulk": { - "threads": 4, - "queue": 0, - "rejected": 0 - }, - "generic": { - "queue": 0, - "rejected": 0, - "threads": 4 - }, - "get": { - "threads": 4, - "queue": 0, - "rejected": 0 - }, - "index": { - "threads": 2, - "queue": 0, - "rejected": 0 - }, - "management": { - "queue": 0, - "rejected": 0, - "threads": 4 - }, - "search": { - "threads": 7, - "queue": 0, - "rejected": 0 - } - } - }, - "metricset": { - "name": "node_stats", - "module": "elasticsearch", - "host": "localhost:9200", - "rtt": 5047 - } -} diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data.json b/metricbeat/module/elasticsearch/node_stats/_meta/data.json index cb33996adf0..9de1ea7ec42 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/data.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/data.json @@ -1,45 +1,142 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "elasticsearch": { "cluster": { - "id": "3LbUkLkURz--FR-YO0wLNA", - "name": "es1" + "id": "w3oo88LcQ1i-7K4f-wrEgQ", + "name": "docker-cluster" }, "node": { - "id": "FMRmkE3HTU6xxxoFK-06Ww", - "name": "es1_1", + "id": "EjV2AqqvQNq5ZF5cVlaPDQ", + "master": true, + "mlockall": false, + "name": "foo", "stats": { "fs": { + "io_stats": {}, "summary": { "available": { - "bytes": 350828584960 + "bytes": 45897547776 }, "free": { - "bytes": 354770468864 + "bytes": 49114263552 }, "total": { - "bytes": 499963170816 + "bytes": 62725623808 } } }, "indices": { + "bulk": { + "avg_size": { + "bytes": 111 + }, + "avg_time": { + "ms": 0 + }, + "operations": { + "total": { + "count": 10 + } + }, + "total_size": { + "bytes": 2939 + }, + "total_time": { + "ms": 75 + } + }, "docs": { - "count": 30880, - "deleted": 124 + "count": 9207, + "deleted": 43 + }, + "fielddata": { + "memory": { + "bytes": 0 + } + }, + "indexing": { + "index_time": { + "ms": 21 + }, + "index_total": { + "count": 4 + }, + "throttle_time": { + "ms": 0 + } + }, + "query_cache": { + "memory": { + "bytes": 0 + } + }, + "request_cache": { + "memory": { + "bytes": 3736 + } + }, + "search": { + "query_time": { + "ms": 83 + }, + "query_total": { + "count": 18 + } }, "segments": { - "count": 39, + "count": 63, + "doc_values": { + "memory": { + "bytes": 117620 + } + }, + "fixed_bit_set": { + "memory": { + "bytes": 3912 + } + }, + "index_writer": { + "memory": { + "bytes": 0 + } + }, "memory": { - "bytes": 300797 + "bytes": 330956 + }, + "norms": { + "memory": { + "bytes": 2688 + } + }, + "points": { + "memory": { + "bytes": 0 + } + }, + "stored_fields": { + "memory": { + "bytes": 31000 + } + }, + "term_vectors": { + "memory": { + "bytes": 0 + } + }, + "terms": { + "memory": { + "bytes": 179648 + } + }, + "version_map": { + "memory": { + "bytes": 0 + } } }, "store": { "size": { - "bytes": 15205991 + "bytes": 18049725 } } }, @@ -48,76 +145,158 @@ "collectors": { "old": { "collection": { - "count": 3, - "ms": 219 + "count": 0, + "ms": 0 } }, "young": { "collection": { - "count": 505, - "ms": 2439 + "count": 10, + "ms": 290 } } } }, "mem": { + "heap": { + "max": { + "bytes": 1073741824 + }, + "used": { + "bytes": 177654272, + "pct": 16 + } + }, "pools": { "old": { "max": { - "bytes": 715849728 + "bytes": 1073741824 }, "peak": { - "bytes": 543519960 + "bytes": 40290816 }, "peak_max": { - "bytes": 715849728 + "bytes": 1073741824 }, "used": { - "bytes": 382281744 + "bytes": 40290816 } }, "survivor": { "max": { - "bytes": 35782656 + "bytes": 0 }, "peak": { - "bytes": 35782656 + "bytes": 53477376 }, "peak_max": { - "bytes": 35782656 + "bytes": 0 }, "used": { - "bytes": 6418816 + "bytes": 50331648 } }, "young": { "max": { - "bytes": 286326784 + "bytes": 0 }, "peak": { - "bytes": 286326784 + "bytes": 637534208 }, "peak_max": { - "bytes": 286326784 + "bytes": 0 }, "used": { - "bytes": 118870448 + "bytes": 87031808 } } } } + }, + "os": { + "cgroup": { + "cpu": { + "cfs": { + "quota": { + "us": -1 + } + }, + "stat": { + "elapsed_periods": { + "count": 0 + }, + "times_throttled": { + "count": 0 + } + } + }, + "cpuacct": { + "usage": { + "ns": 57724017512 + } + }, + "memory": { + "control_group": "/", + "limit": { + "bytes": "9223372036854771712" + }, + "usage": { + "bytes": "1508503552" + } + } + }, + "cpu": { + "load_avg": { + "1m": 2.06 + } + } + }, + "process": { + "cpu": { + "pct": 32 + } + }, + "thread_pool": { + "get": { + "queue": { + "count": 0 + }, + "rejected": { + "count": 0 + } + }, + "search": { + "queue": { + "count": 0 + }, + "rejected": { + "count": 0 + } + }, + "write": { + "queue": { + "count": 0 + }, + "rejected": { + "count": 0 + } + } } } } }, + "event": { + "dataset": "elasticsearch.node.stats", + "duration": 115000, + "module": "elasticsearch" + }, "metricset": { - "host": "127.0.0.1:9200", - "module": "elasticsearch", "name": "node_stats", - "namespace": "elasticsearch.node.stats", - "rtt": 115 + "period": 10000 }, "service": { - "name": "elasticsearch" + "address": "localhost:9200", + "name": "elasticsearch", + "type": "elasticsearch" } } \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml b/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml index 82c0d9feb70..b9db2dc2ebe 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml @@ -30,6 +30,85 @@ type: long description: > Total size of the store in bytes. + - name: fielddata + type: group + fields: + - name: memory.bytes + type: long + format: bytes + - name: indexing + type: group + fields: + - name: index_time.ms + type: long + - name: index_total.count + type: long + - name: throttle_time.ms + type: long + - name: query_cache + type: group + fields: + - name: memory.bytes + type: long + format: bytes + - name: request_cache + type: group + fields: + - name: memory.bytes + type: long + format: bytes + - name: search + type: group + fields: + - name: query_time.ms + type: long + - name: query_total.count + type: long + - name: segments + type: group + fields: + - name: doc_values.memory.bytes + type: long + format: bytes + - name: fixed_bit_set.memory.bytes + type: long + format: bytes + - name: index_writer.memory.bytes + type: long + format: bytes + - name: norms.memory.bytes + type: long + format: bytes + - name: points.memory.bytes + type: long + format: bytes + - name: stored_fields.memory.bytes + type: long + format: bytes + - name: term_vectors.memory.bytes + type: long + format: bytes + - name: terms.memory.bytes + type: long + format: bytes + - name: version_map.memory.bytes + type: long + format: bytes + - name: jvm.mem.heap + type: group + fields: + - name: max.bytes + type: long + format: bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + - name: pct + type: long + format: percent - name: jvm.mem.pools type: group description: > @@ -110,7 +189,6 @@ format: bytes description: Used bytes. - - name: jvm.gc.collectors type: group description: > @@ -139,20 +217,117 @@ type: long description: > - - name: fs.summary + - name: fs type: group - description: > - File system summary fields: - - name: total.bytes - type: long - format: bytes - description: > - - name: free.bytes - type: long - format: bytes - description: > - - name: available.bytes - type: long - format: bytes + - name: summary + type: group description: > + File system summary + fields: + - name: total.bytes + type: long + format: bytes + description: > + - name: free.bytes + type: long + format: bytes + description: > + - name: available.bytes + type: long + format: bytes + description: > + - name: io_stats + type: group + fields: + - name: total + type: group + fields: + - name: operations.count + type: long + - name: read.operations.count + type: long + - name: write.operations.count + type: long + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: load_avg.1m + type: long + - name: cgroup + type: group + fields: + - name: cpuacct.usage.ns + type: long + - name: cpu + type: group + fields: + - name: cfs.quota.us + type: long + - name: stat + type: group + fields: + - name: elapsed_periods.count + type: long + - name: times_throttled.count + type: long + - name: time_throttled.ns + type: long + - name: memory + type: group + fields: + - name: control_group + type: keyword + - name: limit.bytes + type: long + format: bytes + - name: usage.bytes + type: long + format: bytes + - name: process.cpu.pct + type: long + format: percent + - name: thread_pool + type: group + fields: + - name: bulk + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + - name: get + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + - name: index + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + - name: search + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + - name: write + type: group + fields: + - name: queue.count + type: long + - name: rejected.count + type: long + + + diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.243.json b/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.243.json deleted file mode 100644 index cf8e33b9811..00000000000 --- a/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.243.json +++ /dev/null @@ -1,426 +0,0 @@ -{ - "cluster_name" : "elasticsearch", - "nodes" : { - "Tf3ps4nBSruDoLz3jx2uqg" : { - "timestamp" : 1491553881539, - "name" : "Projector", - "transport_address" : "127.0.0.1:9300", - "host" : "127.0.0.1", - "ip" : [ "127.0.0.1:9300", "NONE" ], - "indices" : { - "docs" : { - "count" : 0, - "deleted" : 0 - }, - "store" : { - "size_in_bytes" : 0, - "throttle_time_in_millis" : 0 - }, - "indexing" : { - "index_total" : 0, - "index_time_in_millis" : 0, - "index_current" : 0, - "index_failed" : 0, - "delete_total" : 0, - "delete_time_in_millis" : 0, - "delete_current" : 0, - "noop_update_total" : 0, - "is_throttled" : false, - "throttle_time_in_millis" : 0 - }, - "get" : { - "total" : 0, - "time_in_millis" : 0, - "exists_total" : 0, - "exists_time_in_millis" : 0, - "missing_total" : 0, - "missing_time_in_millis" : 0, - "current" : 0 - }, - "search" : { - "open_contexts" : 0, - "query_total" : 0, - "query_time_in_millis" : 0, - "query_current" : 0, - "fetch_total" : 0, - "fetch_time_in_millis" : 0, - "fetch_current" : 0, - "scroll_total" : 0, - "scroll_time_in_millis" : 0, - "scroll_current" : 0 - }, - "merges" : { - "current" : 0, - "current_docs" : 0, - "current_size_in_bytes" : 0, - "total" : 0, - "total_time_in_millis" : 0, - "total_docs" : 0, - "total_size_in_bytes" : 0, - "total_stopped_time_in_millis" : 0, - "total_throttled_time_in_millis" : 0, - "total_auto_throttle_in_bytes" : 0 - }, - "refresh" : { - "total" : 0, - "total_time_in_millis" : 0 - }, - "flush" : { - "total" : 0, - "total_time_in_millis" : 0 - }, - "warmer" : { - "current" : 0, - "total" : 0, - "total_time_in_millis" : 0 - }, - "query_cache" : { - "memory_size_in_bytes" : 0, - "total_count" : 0, - "hit_count" : 0, - "miss_count" : 0, - "cache_size" : 0, - "cache_count" : 0, - "evictions" : 0 - }, - "fielddata" : { - "memory_size_in_bytes" : 0, - "evictions" : 0 - }, - "percolate" : { - "total" : 0, - "time_in_millis" : 0, - "current" : 0, - "memory_size_in_bytes" : -1, - "memory_size" : "-1b", - "queries" : 0 - }, - "completion" : { - "size_in_bytes" : 0 - }, - "segments" : { - "count" : 0, - "memory_in_bytes" : 0, - "terms_memory_in_bytes" : 0, - "stored_fields_memory_in_bytes" : 0, - "term_vectors_memory_in_bytes" : 0, - "norms_memory_in_bytes" : 0, - "doc_values_memory_in_bytes" : 0, - "index_writer_memory_in_bytes" : 0, - "index_writer_max_memory_in_bytes" : 0, - "version_map_memory_in_bytes" : 0, - "fixed_bit_set_memory_in_bytes" : 0 - }, - "translog" : { - "operations" : 0, - "size_in_bytes" : 0 - }, - "suggest" : { - "total" : 0, - "time_in_millis" : 0, - "current" : 0 - }, - "request_cache" : { - "memory_size_in_bytes" : 0, - "evictions" : 0, - "hit_count" : 0, - "miss_count" : 0 - }, - "recovery" : { - "current_as_source" : 0, - "current_as_target" : 0, - "throttle_time_in_millis" : 0 - } - }, - "os" : { - "timestamp" : 1491553881554, - "cpu_percent" : 46, - "load_average" : 2.7578125, - "mem" : { - "total_in_bytes" : 17179869184, - "free_in_bytes" : 91217920, - "used_in_bytes" : 17088651264, - "free_percent" : 1, - "used_percent" : 99 - }, - "swap" : { - "total_in_bytes" : 2147483648, - "free_in_bytes" : 1660157952, - "used_in_bytes" : 487325696 - } - }, - "process" : { - "timestamp" : 1491553881554, - "open_file_descriptors" : 152, - "max_file_descriptors" : 10240, - "cpu" : { - "percent" : 13, - "total_in_millis" : 11733 - }, - "mem" : { - "total_virtual_in_bytes" : 5196320768 - } - }, - "jvm" : { - "timestamp" : 1491553881554, - "uptime_in_millis" : 14105, - "mem" : { - "heap_used_in_bytes" : 70949408, - "heap_used_percent" : 6, - "heap_committed_in_bytes" : 259522560, - "heap_max_in_bytes" : 1038876672, - "non_heap_used_in_bytes" : 45168576, - "non_heap_committed_in_bytes" : 46948352, - "pools" : { - "young" : { - "used_in_bytes" : 49528368, - "max_in_bytes" : 279183360, - "peak_used_in_bytes" : 71630848, - "peak_max_in_bytes" : 279183360 - }, - "survivor" : { - "used_in_bytes" : 8912896, - "max_in_bytes" : 34865152, - "peak_used_in_bytes" : 8912896, - "peak_max_in_bytes" : 34865152 - }, - "old" : { - "used_in_bytes" : 12508144, - "max_in_bytes" : 724828160, - "peak_used_in_bytes" : 12508144, - "peak_max_in_bytes" : 724828160 - } - } - }, - "threads" : { - "count" : 52, - "peak_count" : 52 - }, - "gc" : { - "collectors" : { - "young" : { - "collection_count" : 4, - "collection_time_in_millis" : 81 - }, - "old" : { - "collection_count" : 1, - "collection_time_in_millis" : 14 - } - } - }, - "buffer_pools" : { - "direct" : { - "count" : 30, - "used_in_bytes" : 3154146, - "total_capacity_in_bytes" : 3154146 - }, - "mapped" : { - "count" : 0, - "used_in_bytes" : 0, - "total_capacity_in_bytes" : 0 - } - }, - "classes" : { - "current_loaded_count" : 6631, - "total_loaded_count" : 6631, - "total_unloaded_count" : 0 - } - }, - "thread_pool" : { - "bulk" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "fetch_shard_started" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "fetch_shard_store" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "flush" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "force_merge" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "generic" : { - "threads" : 5, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 5, - "completed" : 32 - }, - "get" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "index" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "listener" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "management" : { - "threads" : 1, - "queue" : 0, - "active" : 1, - "rejected" : 0, - "largest" : 1, - "completed" : 1 - }, - "percolate" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "refresh" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "search" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "snapshot" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "suggest" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - }, - "warmer" : { - "threads" : 0, - "queue" : 0, - "active" : 0, - "rejected" : 0, - "largest" : 0, - "completed" : 0 - } - }, - "fs" : { - "timestamp" : 1491553881555, - "total" : { - "total_in_bytes" : 249779191808, - "free_in_bytes" : 18257371136, - "available_in_bytes" : 17995227136 - }, - "data" : [ { - "path" : "/Users/ruflin/Downloads/elasticsearch-2.4.3/data/elasticsearch/nodes/0", - "mount" : "/ (/dev/disk1)", - "type" : "hfs", - "total_in_bytes" : 249779191808, - "free_in_bytes" : 18257371136, - "available_in_bytes" : 17995227136 - } ] - }, - "transport" : { - "server_open" : 0, - "rx_count" : 6, - "rx_size_in_bytes" : 2472, - "tx_count" : 6, - "tx_size_in_bytes" : 2472 - }, - "http" : { - "current_open" : 1, - "total_opened" : 1 - }, - "breakers" : { - "request" : { - "limit_size_in_bytes" : 415550668, - "limit_size" : "396.2mb", - "estimated_size_in_bytes" : 0, - "estimated_size" : "0b", - "overhead" : 1.0, - "tripped" : 0 - }, - "fielddata" : { - "limit_size_in_bytes" : 623326003, - "limit_size" : "594.4mb", - "estimated_size_in_bytes" : 0, - "estimated_size" : "0b", - "overhead" : 1.03, - "tripped" : 0 - }, - "in_flight_requests" : { - "limit_size_in_bytes" : 1038876672, - "limit_size" : "990.7mb", - "estimated_size_in_bytes" : 0, - "estimated_size" : "0b", - "overhead" : 1.0, - "tripped" : 0 - }, - "parent" : { - "limit_size_in_bytes" : 727213670, - "limit_size" : "693.5mb", - "estimated_size_in_bytes" : 0, - "estimated_size" : "0b", - "overhead" : 1.0, - "tripped" : 0 - } - }, - "script" : { - "compilations" : 0, - "cache_evictions" : 0 - } - } - } -} \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.522.json b/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.522.json index 31dcdd8a3b7..09ced1c2cfa 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.522.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.522.json @@ -6,7 +6,7 @@ }, "cluster_name" : "elasticsearch", "nodes" : { - "x6_Rm157RqilNEqtdgcNrA" : { + "test_node_id" : { "timestamp" : 1491553781643, "name" : "x6_Rm15", "transport_address" : "127.0.0.1:9300", @@ -437,4 +437,4 @@ } } } -} \ No newline at end of file +} diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.623.json b/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.623.json index 627b117321f..1767caedeb5 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.623.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.623.json @@ -6,7 +6,7 @@ }, "cluster_name": "842d2a3cdf39e9ae2e0b2c7ca7cea075", "nodes": { - "r4XD9O8eTrCHyN_GJswZ5A": { + "test_node_id": { "timestamp": 1524464610026, "name": "instance-0000000016", "transport_address": "172.25.133.112:19608", @@ -503,7 +503,7 @@ } }, "adaptive_selection": { - "r4XD9O8eTrCHyN_GJswZ5A": { + "test_node_id": { "outgoing_searches": 0, "avg_queue_size": 0, "avg_service_time_ns": 79338, diff --git a/metricbeat/module/elasticsearch/node_stats/data.go b/metricbeat/module/elasticsearch/node_stats/data.go index 4e860d0b52e..41fd381a67b 100644 --- a/metricbeat/module/elasticsearch/node_stats/data.go +++ b/metricbeat/module/elasticsearch/node_stats/data.go @@ -43,6 +43,15 @@ var ( "survivor": c.Dict("survivor", poolSchema), "old": c.Dict("old", poolSchema), }), + "heap": s.Object{ + "max": s.Object{ + "bytes": c.Int("heap_max_in_bytes"), + }, + "used": s.Object{ + "bytes": c.Int("heap_used_in_bytes"), + "pct": c.Int("heap_used_percent"), + }, + }, }), "gc": c.Dict("gc", s.Schema{ "collectors": c.Dict("collectors", s.Schema{ @@ -52,10 +61,63 @@ var ( }), }), "indices": c.Dict("indices", s.Schema{ + "bulk": c.Dict("bulk", s.Schema{ + "avg_size": s.Object{ + "bytes": c.Int("avg_size_in_bytes"), + }, + "avg_time": s.Object{ + "ms": c.Int("avg_time_in_millis"), + }, + "total_size": s.Object{ + "bytes": c.Int("total_size_in_bytes"), + }, + "total_time": s.Object{ + "ms": c.Int("total_time_in_millis"), + }, + "operations": s.Object{ + "total": s.Object{ + "count": c.Int("total_operations"), + }, + }, + }, c.DictOptional), "docs": c.Dict("docs", s.Schema{ "count": c.Int("count"), "deleted": c.Int("deleted"), }), + "fielddata": c.Dict("fielddata", s.Schema{ + "memory": s.Object{ + "bytes": c.Int("memory_size_in_bytes"), + }, + }), + "indexing": c.Dict("indexing", s.Schema{ + "index_time": s.Object{ + "ms": c.Int("index_time_in_millis"), + }, + "index_total": s.Object{ + "count": c.Int("index_total"), + }, + "throttle_time": s.Object{ + "ms": c.Int("throttle_time_in_millis"), + }, + }), + "query_cache": c.Dict("query_cache", s.Schema{ + "memory": s.Object{ + "bytes": c.Int("memory_size_in_bytes"), + }, + }), + "request_cache": c.Dict("request_cache", s.Schema{ + "memory": s.Object{ + "bytes": c.Int("memory_size_in_bytes"), + }, + }), + "search": c.Dict("search", s.Schema{ + "query_time": s.Object{ + "ms": c.Int("query_time_in_millis"), + }, + "query_total": s.Object{ + "count": c.Int("query_total"), + }, + }), "store": c.Dict("store", s.Schema{ "size": s.Object{ "bytes": c.Int("size_in_bytes"), @@ -66,6 +128,51 @@ var ( "memory": s.Object{ "bytes": c.Int("memory_in_bytes"), }, + "doc_values": s.Object{ + "memory": s.Object{ + "bytes": c.Int("doc_values_memory_in_bytes"), + }, + }, + "fixed_bit_set": s.Object{ + "memory": s.Object{ + "bytes": c.Int("fixed_bit_set_memory_in_bytes"), + }, + }, + "index_writer": s.Object{ + "memory": s.Object{ + "bytes": c.Int("index_writer_memory_in_bytes"), + }, + }, + "norms": s.Object{ + "memory": s.Object{ + "bytes": c.Int("norms_memory_in_bytes"), + }, + }, + "points": s.Object{ + "memory": s.Object{ + "bytes": c.Int("points_memory_in_bytes"), + }, + }, + "stored_fields": s.Object{ + "memory": s.Object{ + "bytes": c.Int("stored_fields_memory_in_bytes"), + }, + }, + "term_vectors": s.Object{ + "memory": s.Object{ + "bytes": c.Int("term_vectors_memory_in_bytes"), + }, + }, + "terms": s.Object{ + "memory": s.Object{ + "bytes": c.Int("terms_memory_in_bytes"), + }, + }, + "version_map": s.Object{ + "memory": s.Object{ + "bytes": c.Int("version_map_memory_in_bytes"), + }, + }, }), }), "fs": c.Dict("fs", s.Schema{ @@ -80,6 +187,76 @@ var ( "bytes": c.Int("available_in_bytes"), }, }), + "io_stats": c.Dict("io_stats", s.Schema{ + "total": c.Dict("total", s.Schema{ + "operations": s.Object{ + "count": c.Int("operations"), + }, + "read": s.Object{ + "kb": c.Int("read_kilobytes"), + "operations": s.Object{ + "count": c.Int("read_operations"), + }, + }, + "write": s.Object{ + "kb": c.Int("write_kilobytes"), + "operations": s.Object{ + "count": c.Int("write_operations"), + }, + }, + }, c.DictOptional), + }, c.DictOptional), + }), + "os": c.Dict("os", s.Schema{ + "cpu": c.Dict("cpu", s.Schema{ + "load_avg": c.Dict("load_average", s.Schema{ + "1m": c.Float("1m", s.Optional), + }, c.DictOptional), // No load average reported by ES on Windows + }), + "cgroup": c.Dict("cgroup", s.Schema{ + "cpuacct": c.Dict("cpuacct", s.Schema{ + "usage": s.Object{ + "ns": c.Int("usage_nanos"), + }, + }), + "cpu": c.Dict("cpu", s.Schema{ + "cfs": s.Object{ + "quota": s.Object{ + "us": c.Int("cfs_quota_micros"), + }, + }, + "stat": c.Dict("stat", s.Schema{ + "elapsed_periods": s.Object{ + "count": c.Int("number_of_elapsed_periods"), + }, + "times_throttled": s.Object{ + "count": c.Int("number_of_times_throttled"), + }, + }), + }), + "memory": c.Dict("memory", s.Schema{ + "control_group": c.Str("control_group"), + // The two following values are currently string. See https://github.com/elastic/elasticsearch/pull/26166 + "limit": s.Object{ + "bytes": c.Str("limit_in_bytes"), + }, + "usage": s.Object{ + "bytes": c.Str("usage_in_bytes"), + }, + }), + }, c.DictOptional), + }), + "process": c.Dict("process", s.Schema{ + "cpu": c.Dict("cpu", s.Schema{ + "pct": c.Int("percent"), + }), + }), + "thread_pool": c.Dict("thread_pool", s.Schema{ + "bulk": c.Dict("bulk", threadPoolStatsSchema, c.DictOptional), + "index": c.Dict("index", threadPoolStatsSchema, c.DictOptional), + "write": c.Dict("write", threadPoolStatsSchema, c.DictOptional), + "get": c.Dict("get", threadPoolStatsSchema), + "search": c.Dict("search", threadPoolStatsSchema), }), } @@ -104,22 +281,43 @@ var ( "ms": c.Int("collection_time_in_millis"), }, } + + threadPoolStatsSchema = s.Schema{ + "queue": s.Object{ + "count": c.Int("queue"), + }, + "rejected": s.Object{ + "count": c.Int("rejected"), + }, + } ) type nodesStruct struct { Nodes map[string]map[string]interface{} `json:"nodes"` } -func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { - +func eventsMapping(r mb.ReporterV2, m elasticsearch.MetricSetAPI, info elasticsearch.Info, content []byte) error { nodeData := &nodesStruct{} err := json.Unmarshal(content, nodeData) if err != nil { return errors.Wrap(err, "failure parsing Elasticsearch Node Stats API response") } + masterNodeID, err := m.GetMasterNodeID() + if err != nil { + return err + } + var errs multierror.Errors - for id, node := range nodeData.Nodes { + for nodeID, node := range nodeData.Nodes { + isMaster := nodeID == masterNodeID + + mlockall, err := m.IsMLockAllEnabled(nodeID) + if err != nil { + errs = append(errs, errors.Wrap(err, "error determining if mlockall is set on Elasticsearch node")) + continue + } + event := mb.Event{} event.RootFields = common.MapStr{} @@ -127,7 +325,9 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err event.ModuleFields = common.MapStr{ "node": common.MapStr{ - "id": id, + "id": nodeID, + "mlockall": mlockall, + "master": isMaster, }, "cluster": common.MapStr{ "name": info.ClusterName, diff --git a/metricbeat/module/elasticsearch/node_stats/data_test.go b/metricbeat/module/elasticsearch/node_stats/data_test.go index 043a1447f25..a48e3106889 100644 --- a/metricbeat/module/elasticsearch/node_stats/data_test.go +++ b/metricbeat/module/elasticsearch/node_stats/data_test.go @@ -22,9 +22,41 @@ package node_stats import ( "testing" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) func TestStats(t *testing.T) { - elasticsearch.TestMapperWithInfo(t, "./_meta/test/node_stats.*.json", eventsMapping) + ms := mockMetricSet{} + elasticsearch.TestMapperWithMetricSetAndInfo(t, "./_meta/test/node_stats.*.json", ms, eventsMapping) +} + +type mockMetricSet struct{} + +func (m mockMetricSet) GetMasterNodeID() (string, error) { + return "test_node_id", nil +} + +func (m mockMetricSet) IsMLockAllEnabled(_ string) (bool, error) { + return true, nil +} + +func (m mockMetricSet) Module() mb.Module { + return mockModule{} +} + +type mockModule struct{} + +func (m mockModule) Name() string { + return "mock_module" +} + +func (m mockModule) Config() mb.ModuleConfig { + return mb.ModuleConfig{ + Period: 10000, + } +} + +func (m mockModule) UnpackConfig(to interface{}) error { + return nil } diff --git a/metricbeat/module/elasticsearch/node_stats/data_xpack.go b/metricbeat/module/elasticsearch/node_stats/data_xpack.go deleted file mode 100644 index e4efeb3b8e8..00000000000 --- a/metricbeat/module/elasticsearch/node_stats/data_xpack.go +++ /dev/null @@ -1,239 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package node_stats - -import ( - "encoding/json" - - "time" - - "github.com/joeshaw/multierror" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - s "github.com/elastic/beats/v7/libbeat/common/schema" - c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" - "github.com/elastic/beats/v7/metricbeat/helper/elastic" - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" -) - -var ( - schemaXpack = s.Schema{ - "name": c.Str("name"), - "transport_address": c.Str("transport_address"), - "indices": c.Dict("indices", s.Schema{ - "docs": c.Dict("docs", s.Schema{ - "count": c.Int("count"), - }), - "store": c.Dict("store", s.Schema{ - "size_in_bytes": c.Int("size_in_bytes"), - }), - "indexing": c.Dict("indexing", s.Schema{ - "index_total": c.Int("index_total"), - "index_time_in_millis": c.Int("index_time_in_millis"), - "throttle_time_in_millis": c.Int("throttle_time_in_millis"), - }), - "bulk": elasticsearch.BulkStatsDict, - "search": c.Dict("search", s.Schema{ - "query_total": c.Int("query_total"), - "query_time_in_millis": c.Int("query_time_in_millis"), - }), - "query_cache": c.Dict("query_cache", s.Schema{ - "memory_size_in_bytes": c.Int("memory_size_in_bytes"), - "hit_count": c.Int("hit_count"), - "miss_count": c.Int("miss_count"), - "evictions": c.Int("evictions"), - }), - "fielddata": c.Dict("fielddata", s.Schema{ - "memory_size_in_bytes": c.Int("memory_size_in_bytes"), - "evictions": c.Int("evictions"), - }), - "segments": c.Dict("segments", s.Schema{ - "count": c.Int("count"), - "memory_in_bytes": c.Int("memory_in_bytes"), - "terms_memory_in_bytes": c.Int("terms_memory_in_bytes"), - "stored_fields_memory_in_bytes": c.Int("stored_fields_memory_in_bytes"), - "term_vectors_memory_in_bytes": c.Int("term_vectors_memory_in_bytes"), - "norms_memory_in_bytes": c.Int("norms_memory_in_bytes"), - "points_memory_in_bytes": c.Int("points_memory_in_bytes"), - "doc_values_memory_in_bytes": c.Int("doc_values_memory_in_bytes"), - "index_writer_memory_in_bytes": c.Int("index_writer_memory_in_bytes"), - "version_map_memory_in_bytes": c.Int("version_map_memory_in_bytes"), - "fixed_bit_set_memory_in_bytes": c.Int("fixed_bit_set_memory_in_bytes"), - }), - "request_cache": c.Dict("request_cache", s.Schema{ - "memory_size_in_bytes": c.Int("memory_size_in_bytes"), - "evictions": c.Int("evictions"), - "hit_count": c.Int("hit_count"), - "miss_count": c.Int("miss_count"), - }), - }), - "os": c.Dict("os", s.Schema{ - "cpu": c.Dict("cpu", s.Schema{ - "load_average": c.Dict("load_average", s.Schema{ - "1m": c.Float("1m", s.Optional), - "5m": c.Float("5m", s.Optional), - "15m": c.Float("15m", s.Optional), - }, c.DictOptional), // No load average reported by ES on Windows - }), - "cgroup": c.Dict("cgroup", s.Schema{ - "cpuacct": c.Dict("cpuacct", s.Schema{ - "control_group": c.Str("control_group"), - "usage_nanos": c.Int("usage_nanos"), - }), - "cpu": c.Dict("cpu", s.Schema{ - "control_group": c.Str("control_group"), - "cfs_period_micros": c.Int("cfs_period_micros"), - "cfs_quota_micros": c.Int("cfs_quota_micros"), - "stat": c.Dict("stat", s.Schema{ - "number_of_elapsed_periods": c.Int("number_of_elapsed_periods"), - "number_of_times_throttled": c.Int("number_of_times_throttled"), - "time_throttled_nanos": c.Int("time_throttled_nanos"), - }), - }), - "memory": c.Dict("memory", s.Schema{ - "control_group": c.Str("control_group"), - // The two following values are currently string. See https://github.com/elastic/elasticsearch/pull/26166 - "limit_in_bytes": c.Str("limit_in_bytes"), - "usage_in_bytes": c.Str("usage_in_bytes"), - }), - }, c.DictOptional), - }), - "process": c.Dict("process", s.Schema{ - "open_file_descriptors": c.Int("open_file_descriptors"), - "max_file_descriptors": c.Int("max_file_descriptors"), - "cpu": c.Dict("cpu", s.Schema{ - "percent": c.Int("percent"), - }), - }), - "jvm": c.Dict("jvm", s.Schema{ - "mem": c.Dict("mem", s.Schema{ - "heap_used_in_bytes": c.Int("heap_used_in_bytes"), - "heap_used_percent": c.Int("heap_used_percent"), - "heap_max_in_bytes": c.Int("heap_max_in_bytes"), - }), - "gc": c.Dict("gc", s.Schema{ - "collectors": c.Dict("collectors", s.Schema{ - "young": c.Dict("young", s.Schema{ - "collection_count": c.Int("collection_count"), - "collection_time_in_millis": c.Int("collection_time_in_millis"), - }), - "old": c.Dict("young", s.Schema{ - "collection_count": c.Int("collection_count"), - "collection_time_in_millis": c.Int("collection_time_in_millis"), - }), - }), - }), - }), - "thread_pool": c.Dict("thread_pool", s.Schema{ - "bulk": c.Dict("bulk", threadPoolStatsSchema, c.DictOptional), - "index": c.Dict("index", threadPoolStatsSchema, c.DictOptional), - "write": c.Dict("write", threadPoolStatsSchema), - "generic": c.Dict("generic", threadPoolStatsSchema), - "get": c.Dict("get", threadPoolStatsSchema), - "management": c.Dict("management", threadPoolStatsSchema), - "search": c.Dict("search", threadPoolStatsSchema), - "watcher": c.Dict("watcher", threadPoolStatsSchema, c.DictOptional), - }), - "fs": c.Dict("fs", s.Schema{ - "total": c.Dict("total", s.Schema{ - "total_in_bytes": c.Int("total_in_bytes"), - "free_in_bytes": c.Int("free_in_bytes"), - "available_in_bytes": c.Int("available_in_bytes"), - }), - "io_stats": c.Dict("io_stats", s.Schema{ - "total": c.Dict("total", s.Schema{ - "operations": c.Int("operations"), - "read_kilobytes": c.Int("read_kilobytes"), - "read_operations": c.Int("read_operations"), - "write_kilobytes": c.Int("write_kilobytes"), - "write_operations": c.Int("write_operations"), - }, c.DictOptional), - }, c.DictOptional), - }), - } - - threadPoolStatsSchema = s.Schema{ - "threads": c.Int("threads"), - "queue": c.Int("queue"), - "rejected": c.Int("rejected"), - } -) - -func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { - nodesStruct := struct { - ClusterName string `json:"cluster_name"` - Nodes map[string]map[string]interface{} `json:"nodes"` - }{} - - err := json.Unmarshal(content, &nodesStruct) - if err != nil { - return errors.Wrap(err, "failure parsing Elasticsearch Node Stats API response") - } - - masterNodeID, err := elasticsearch.GetMasterNodeID(m.HTTP, m.HTTP.GetURI()) - if err != nil { - return err - } - - var errs multierror.Errors - for nodeID, node := range nodesStruct.Nodes { - isMaster := nodeID == masterNodeID - - event := mb.Event{} - - nodeData, err := schemaXpack.Apply(node) - if err != nil { - errs = append(errs, errors.Wrap(err, "failure to apply node schema")) - continue - } - - nodeData["node_master"] = isMaster - nodeData["node_id"] = nodeID - - mlockall, err := elasticsearch.IsMLockAllEnabled(m.HTTP, m.HTTP.GetURI(), nodeID) - if err != nil { - errs = append(errs, err) - continue - } - nodeData["mlockall"] = mlockall - - // Build source_node object - sourceNode := common.MapStr{ - "uuid": nodeID, - "name": nodeData["name"], - "transport_address": nodeData["transport_address"], - } - nodeData.Delete("name") - nodeData.Delete("transport_address") - - event.RootFields = common.MapStr{ - "timestamp": time.Now(), - "cluster_uuid": info.ClusterID, - "interval_ms": m.Module().Config().Period.Nanoseconds() / 1000 / 1000, - "type": "node_stats", - "node_stats": nodeData, - "source_node": sourceNode, - } - - event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) - r.Event(event) - } - return errs.Err() -} diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats.go b/metricbeat/module/elasticsearch/node_stats/node_stats.go index 5f856e2eeac..2ab26ffe8ed 100644 --- a/metricbeat/module/elasticsearch/node_stats/node_stats.go +++ b/metricbeat/module/elasticsearch/node_stats/node_stats.go @@ -74,20 +74,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - if m.XPack { - err = eventsMappingXPack(r, m, *info, content) - if err != nil { - // Since this is an x-pack code path, we log the error but don't - // return it. Otherwise it would get reported into `metricbeat-*` - // indices. - m.Logger().Error(err) - return nil - } - } else { - return eventsMapping(r, *info, content) - } - - return nil + return eventsMapping(r, m.MetricSet, *info, content) } func (m *MetricSet) updateServiceURI() error { diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py index 2ccf3b0fddd..8a39857cf99 100644 --- a/metricbeat/module/elasticsearch/test_elasticsearch.py +++ b/metricbeat/module/elasticsearch/test_elasticsearch.py @@ -95,7 +95,6 @@ def test_xpack(self): "index_recovery", "index_summary", "ml_job", - "node_stats", "shard" ], "hosts": self.get_hosts(), @@ -128,7 +127,6 @@ def test_xpack_cluster_stats(self): "index_recovery", "index_summary", "ml_job", - "node_stats", "shard" ], "hosts": self.get_hosts(), diff --git a/metricbeat/module/elasticsearch/testing.go b/metricbeat/module/elasticsearch/testing.go index a68685dd62d..b5d72e74315 100644 --- a/metricbeat/module/elasticsearch/testing.go +++ b/metricbeat/module/elasticsearch/testing.go @@ -76,3 +76,29 @@ func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, In }) } } + +// TestMapperWithMetricSetAndInfo tests mapping methods with Info fields +func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, mapper func(mb.ReporterV2, MetricSetAPI, Info, []byte) error) { + files, err := filepath.Glob(glob) + require.NoError(t, err) + // Makes sure glob matches at least 1 file + require.True(t, len(files) > 0) + + info := Info{ + ClusterID: "1234", + ClusterName: "helloworld", + } + + for _, f := range files { + t.Run(f, func(t *testing.T) { + input, err := ioutil.ReadFile(f) + require.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = mapper(reporter, ms, info, input) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + }) + } +}