Skip to content

Commit

Permalink
Make elasticsearch/node_stats metricset work for Stack Monitoring wit…
Browse files Browse the repository at this point in the history 
…hout xpack.enabled flag (#19747)

* Make node_stats ignore xpack.enabled: true for the POC

* Index more fields

* WIP: add fields to fields.yml

* Fleshing out alias fields

* Adding more fields to mapping

* Marking optional fields

* Fixing datatypes

* Fixing formatting

* Fixing alias field definitions

* More field fixes

* Making cgroups metrics collection optional

* Marking os.load_avg as optional for Windows

* Aliasing type => metricset.name

* Aliasing source_node fields

* Aliasing timestamp => @timestamp

* Removing field alias mappings for unused fields

* Removing unnecessary fields from mapping

* Updating generated files

* Reducing visibility of isMaster function

* Adding methods to metricset

* Refactoring for testability

* Fixing formatting

* Making indices.bulk and thread_pool.write fields optional for BC

* Deleting sample file from unsupported ES version

* Remove xpack.enabled code path!!!

* Updating xpack unit test

* Updating python system tests for xpack

* Remove top-level type field

* Removing data-xpack.json

* Updating data.json

* Not collecting unmapped fields

* Fixing formatting
  • Loading branch information
@ycombinator
ycombinator authored Aug 14, 2020
1 parent 0248e97 commit 575c571
Show file tree
Hide file tree
Showing 18 changed files with 1,995 additions and 1,011 deletions.
1,029 changes: 970 additions & 59 deletions metricbeat/docs/fields.asciidoc
View file

Large diffs are not rendered by default.

301 changes: 289 additions & 12 deletions metricbeat/module/elasticsearch/_meta/fields.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,276 @@
settings: ["ssl", "http"]
short_config: false
fields:
- name: timestamp
type: alias
path: "@timestamp"
- name: cluster_uuid
type: alias
path: elasticsearch.cluster.id
- name: source_node
type: group
fields:
- name: uuid
type: alias
path: elasticsearch.node.id
- name: name
type: alias
path: elasticsearch.node.name
- name: node_stats
type: group
fields:
- name: fs
type: group
fields:
- name: total
type: group
fields:
- name: available_in_bytes
path: elasticsearch.node.stats.fs.summary.available.bytes
type: alias
- name: io_stats
type: group
fields:
- name: total
type: group
fields:
- name: operations
path: elasticsearch.node.stats.fs.io_stats.total.operations.count
type: alias
- name: read_operations
path: elasticsearch.node.stats.fs.io_stats.total.read.operations.count
type: alias
- name: write_operations
path: elasticsearch.node.stats.fs.io_stats.total.write.operations.count
type: alias
- name: indices
type: group
fields:
- name: indexing
type: group
fields:
- name: index_time_in_millis
path: elasticsearch.node.stats.indices.indexing.index_time.ms
type: alias
- name: index_total
path: elasticsearch.node.stats.indices.indexing.index_total.count
type: alias
- name: throttle_time_in_millis
path: elasticsearch.node.stats.indices.indexing.throttle_time.ms
type: alias
- name: fielddata
type: group
fields:
- name: memory_size_in_bytes
path: elasticsearch.node.stats.indices.fielddata.memory.bytes
type: alias
- name: query_cache
type: group
fields:
- name: memory_size_in_bytes
path: elasticsearch.node.stats.indices.query_cache.memory.bytes
type: alias
- name: request_cache
type: group
fields:
- name: memory_size_in_bytes
path: elasticsearch.node.stats.indices.request_cache.memory.bytes
type: alias
- name: search
type: group
fields:
- name: query_time_in_millis
path: elasticsearch.node.stats.indices.search.query_time.ms
type: alias
- name: query_total
path: elasticsearch.node.stats.indices.search.query_total.count
type: alias
- name: segments
type: group
fields:
- name: count
path: elasticsearch.node.stats.indices.segments.count
type: alias
- name: doc_values_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.doc_values.memory.bytes
type: alias
- name: fixed_bit_set_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes
type: alias
- name: index_writer_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.index_writer.memory.bytes
type: alias
- name: memory_in_bytes
path: elasticsearch.node.stats.indices.segments.memory.bytes
type: alias
- name: norms_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.norms.memory.bytes
type: alias
- name: points_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.points.memory.bytes
type: alias
- name: stored_fields_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes
type: alias
- name: term_vectors_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes
type: alias
- name: terms_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.terms.memory.bytes
type: alias
- name: version_map_memory_in_bytes
path: elasticsearch.node.stats.indices.segments.version_map.memory.bytes
type: alias
- name: jvm
type: group
fields:
- name: gc
type: group
fields:
- name: collectors
type: group
fields:
- name: old
type: group
fields:
- name: collection_count
path: elasticsearch.node.stats.jvm.gc.collectors.old.collection.count
type: alias
- name: collection_time_in_millis
path: elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms
type: alias
- name: young
type: group
fields:
- name: collection_count
path: elasticsearch.node.stats.jvm.gc.collectors.young.collection.count
type: alias
- name: collection_time_in_millis
path: elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms
type: alias
- name: mem
type: group
fields:
- name: heap_max_in_bytes
path: elasticsearch.node.stats.jvm.mem.heap.max.bytes
type: alias
- name: heap_used_in_bytes
path: elasticsearch.node.stats.jvm.mem.heap.used.bytes
type: alias
- name: heap_used_percent
path: elasticsearch.node.stats.jvm.mem.heap.used.pct
type: alias
- name: node_id
path: elasticsearch.node.id
type: alias
- name: os
type: group
fields:
- name: cpu
type: group
fields:
- name: load_average
type: group
fields:
- name: 1m
path: elasticsearch.node.stats.os.cpu.load_avg.1m
type: alias
- name: cgroup
type: group
fields:
- name: cpuacct
type: group
fields:
- name: usage_nanos
path: elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns
type: alias
- name: cpu
type: group
fields:
- name: cfs_quota_micros
path: elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us
type: alias
- name: stat
type: group
fields:
- name: number_of_elapsed_periods
path: elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count
type: alias
- name: number_of_times_throttled
path: elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count
type: alias
- name: time_throttled_nanos
path: elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns
type: alias
- name: memory
type: group
fields:
- name: control_group
path: elasticsearch.node.stats.os.cgroup.memory.control_group
type: alias
- name: limit_in_bytes
path: elasticsearch.node.stats.os.cgroup.memory.limit.bytes
type: alias
- name: usage_in_bytes
path: elasticsearch.node.stats.os.cgroup.memory.usage.bytes
type: alias
- name: process
type: group
fields:
- name: cpu
type: group
fields:
- name: percent
path: elasticsearch.node.stats.process.cpu.pct
type: alias
- name: thread_pool
type: group
fields:
- name: bulk
type: group
fields:
- name: queue
path: elasticsearch.node.stats.thread_pool.bulk.queue.count
type: alias
- name: rejected
path: elasticsearch.node.stats.thread_pool.bulk.rejected.count
type: alias
- name: get
type: group
fields:
- name: queue
path: elasticsearch.node.stats.thread_pool.get.queue.count
type: alias
- name: rejected
path: elasticsearch.node.stats.thread_pool.get.rejected.count
type: alias
- name: index
type: group
fields:
- name: queue
path: elasticsearch.node.stats.thread_pool.index.queue.count
type: alias
- name: rejected
path: elasticsearch.node.stats.thread_pool.index.rejected.count
type: alias
- name: search
type: group
fields:
- name: queue
path: elasticsearch.node.stats.thread_pool.search.queue.count
type: alias
- name: rejected
path: elasticsearch.node.stats.thread_pool.search.rejected.count
type: alias
- name: write
type: group
fields:
- name: queue
path: elasticsearch.node.stats.thread_pool.write.queue.count
type: alias
- name: rejected
path: elasticsearch.node.stats.thread_pool.write.rejected.count
type: alias
- name: elasticsearch
type: group
description: >
Expand All @@ -14,23 +284,30 @@
type: keyword
description: >
Elasticsearch cluster name.
- name: cluster.id
type: keyword
description: >
Elasticsearch cluster id.
- name: cluster.state.id
type: keyword
description: >
Elasticsearch state id.
- name: node.id
type: keyword
description: >
Node ID
- name: node.name
type: keyword
description: >
Node name.
- name: node
type: group
fields:
- name: id
type: keyword
description: >
Node ID
- name: name
type: keyword
description: >
Node name.
- name: master
type: boolean
description: >
Is the node the master node?
- name: mlockall
type: boolean
description: >
Is mlockall enabled on the node?
48 changes: 2 additions & 46 deletions metricbeat/module/elasticsearch/elasticsearch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ func NewModule(base mb.BaseModule) (mb.Module, error) {
"index_recovery",
"index_summary",
"ml_job",
"node_stats",
"shard",
}
return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName))
Expand Down Expand Up @@ -137,14 +136,14 @@ func GetClusterID(http *helper.HTTP, uri string, nodeID string) (string, error)
return info.ClusterID, nil
}

// IsMaster checks if the given node host is a master node.
// isMaster checks if the given node host is a master node.
//
// The detection of the master is done in two steps:
// * Fetch node name from /_nodes/_local/name
// * Fetch current master name from cluster state /_cluster/state/master_node
//
// The two names are compared
func IsMaster(http *helper.HTTP, uri string) (bool, error) {
func isMaster(http *helper.HTTP, uri string) (bool, error) {

node, err := getNodeName(http, uri)
if err != nil {
Expand Down Expand Up @@ -415,49 +414,6 @@ func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSet
return ret, nil
}

// IsMLockAllEnabled returns if the given Elasticsearch node has mlockall enabled
func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) {
content, err := fetchPath(http, resetURI, "_nodes/"+nodeID, "filter_path=nodes.*.process.mlockall")
if err != nil {
return false, err
}

var response map[string]map[string]map[string]map[string]bool
err = json.Unmarshal(content, &response)
if err != nil {
return false, err
}

for _, nodeInfo := range response["nodes"] {
mlockall := nodeInfo["process"]["mlockall"]
return mlockall, nil
}

return false, fmt.Errorf("could not determine if mlockall is enabled on node ID = %v", nodeID)
}

// GetMasterNodeID returns the ID of the Elasticsearch cluster's master node
func GetMasterNodeID(http *helper.HTTP, resetURI string) (string, error) {
content, err := fetchPath(http, resetURI, "_nodes/_master", "filter_path=nodes.*.name")
if err != nil {
return "", err
}

var response struct {
Nodes map[string]interface{} `json:"nodes"`
}

if err := json.Unmarshal(content, &response); err != nil {
return "", err
}

for nodeID, _ := range response.Nodes {
return nodeID, nil
}

return "", errors.New("could not determine master node ID")
}

// PassThruField copies the field at the given path from the given source data object into
// the same path in the given target data object.
func PassThruField(fieldPath string, sourceData, targetData common.MapStr) error {
Expand Down
Loading

0 comments on commit 575c571

Please sign in to comment.