Skip to content

Commit

Permalink
Make elasticsearch/ccr metricset work for Stack Monitoring without xp…
Browse files Browse the repository at this point in the history 
…ack.enabled flag (elastic#21348)
  • Loading branch information
@sayden
sayden authored Nov 20, 2020
1 parent 19feed8 commit da353fe
Show file tree
Hide file tree
Showing 12 changed files with 836 additions and 126 deletions.
443 changes: 443 additions & 0 deletions metricbeat/docs/fields.asciidoc
View file

Large diffs are not rendered by default.

94 changes: 94 additions & 0 deletions metricbeat/module/elasticsearch/_meta/fields.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,100 @@
- name: name
type: alias
path: elasticsearch.node.name
- name: ccr_auto_follow_stats
type: group
fields:
- name: number_of_failed_follow_indices
type: alias
path: elasticsearch.ccr.auto_follow.failed.follow_indices.count
- name: number_of_failed_remote_cluster_state_requests
type: alias
path: elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count
- name: number_of_successful_follow_indices
type: alias
path: elasticsearch.ccr.auto_follow.success.follow_indices.count
- name: ccr_stats
type: group
fields:
- name: shard_id
type: alias
path: elasticsearch.ccr.shard_id
- name: remote_cluster
type: alias
path: elasticsearch.ccr.remote_cluster
- name: leader_index
type: alias
path: elasticsearch.ccr.leader.index
- name: follower_index
type: alias
path: elasticsearch.ccr.follower.index
- name: leader_global_checkpoint
type: alias
path: elasticsearch.ccr.leader.global_checkpoint
- name: leader_max_seq_no
type: alias
path: elasticsearch.ccr.leader.max_seq_no
- name: follower_global_checkpoint
type: alias
path: elasticsearch.ccr.follower.global_checkpoint
- name: follower_max_seq_no
type: alias
path: elasticsearch.ccr.follower.max_seq_no
- name: last_requested_seq_no
type: alias
path: elasticsearch.ccr.last_requested_seq_no
- name: outstanding_read_requests
type: alias
path: elasticsearch.ccr.requests.outstanding.read.count
- name: outstanding_write_requests
type: alias
path: elasticsearch.ccr.requests.outstanding.write.count
- name: write_buffer_operation_count
type: alias
path: elasticsearch.ccr.write_buffer.operation.count
- name: write_buffer_size_in_bytes
type: alias
path: elasticsearch.ccr.write_buffer.size.bytes
- name: follower_mapping_version
type: alias
path: elasticsearch.ccr.follower.mapping_version
- name: follower_settings_version
type: alias
path: elasticsearch.ccr.follower.settings_version
- name: follower_aliases_version
type: alias
path: elasticsearch.ccr.follower.aliases_version
- name: total_read_time_millis
type: alias
path: elasticsearch.ccr.total_time.read.ms
- name: total_read_remote_exec_time_millis
type: alias
path: elasticsearch.ccr.total_time.read.remote_exec.ms
- name: successful_read_requests
type: alias
path: elasticsearch.ccr.requests.successful.read.count
- name: failed_read_requests
type: alias
path: elasticsearch.ccr.requests.failed.read.count
- name: operations_read
type: alias
path: elasticsearch.ccr.follower.operations.read.count
- name: operations_written
type: alias
path: elasticsearch.ccr.follower.operations_written
- name: bytes_read
type: alias
path: elasticsearch.ccr.bytes_read
- name: total_write_time_millis
type: alias
path: elasticsearch.ccr.total_time.write.ms
- name: successful_write_requests
type: alias
path: elasticsearch.ccr.requests.successful.write.count
- name: failed_write_requests
type: alias
path: elasticsearch.ccr.requests.failed.write.count

- name: node_stats
type: group
fields:
Expand Down
102 changes: 86 additions & 16 deletions metricbeat/module/elasticsearch/ccr/_meta/data.json
View file
Original file line number Diff line number Diff line change
@@ -1,39 +1,109 @@
{
"@timestamp": "2017-10-12T08:05:34.853Z",
"agent": {
"hostname": "host.example.com",
"name": "host.example.com"
},
"elasticsearch": {
"ccr": {
"auto_follow": {
"failed": {
"follow_indices": {
"count": 0
},
"remote_cluster_state_requests": {
"count": 0
}
},
"success": {
"follow_indices": {
"count": 1
}
}
},
"bytes_read": 32768,
"follower": {
"global_checkpoint": -1,
"index": "my_index_f",
"operations_written": 0,
"global_checkpoint": 768,
"index": "follower_index",
"max_seq_no": 896,
"operations": {
"read": {
"count": 896
}
},
"operations_written": 832,
"settings_version": 2,
"shard": {
"number": 0
},
"time_since_last_read": {
"ms": 42294
"ms": 8
}
},
"leader": {
"index": "my_index",
"max_seq_no": -1
"global_checkpoint": 1024,
"index": "leader_index",
"max_seq_no": 1536
},
"requests": {
"failed": {
"read": {
"count": 0
},
"write": {
"count": 0
}
},
"outstanding": {
"read": {
"count": 8
},
"write": {
"count": 2
}
},
"successful": {
"read": {
"count": 32
},
"write": {
"count": 16
}
}
},
"total_time": {
"read": {
"ms": 32768,
"remote_exec": {
"ms": 16384
}
},
"write": {
"ms": 16384
}
},
"write_buffer": {
"operation": {
"count": 64
},
"size": {
"bytes": 1536
}
}
},
"cluster": {
"id": "3LbUkLkURz--FR-YO0wLNA",
"name": "es1"
"id": "8l_zoGznQRmtoX9iSC-goA",
"name": "docker-cluster"
}
},
"event": {
"dataset": "elasticsearch.ccr",
"duration": 115000,
"module": "elasticsearch"
},
"metricset": {
"host": "127.0.0.1:9200",
"module": "elasticsearch",
"name": "ccr",
"rtt": 115
"period": 10000
},
"service": {
"name": "elasticsearch"
"address": "127.0.0.1:53293",
"name": "elasticsearch",
"type": "elasticsearch"
}
}
83 changes: 83 additions & 0 deletions metricbeat/module/elasticsearch/ccr/_meta/fields.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,74 @@
Cross-cluster replication stats
release: ga
fields:
- name: remote_cluster
type: keyword
- name: bytes_read
type: long
- name: last_requested_seq_no
type: long
- name: shard_id
type: integer

- name: total_time
type: group
fields:
- name: read.ms
type: long
- name: read.remote_exec.ms
type: long
- name: write.ms
type: long

- name: requests
type: group
fields:
- name: successful
type: group
fields:
- name: read.count
type: long
- name: write.count
type: long
- name: failed
type: group
fields:
- name: read.count
type: long
- name: write.count
type: long
- name: outstanding
type: group
fields:
- name: read.count
type: long
- name: write.count
type: long

- name: write_buffer
type: group
fields:
- name: size.bytes
type: long
- name: operation.count
type: long

- name: auto_follow
type: group
fields:
- name: failed
type: group
fields:
- name: follow_indices.count
type: long
- name: remote_cluster_state_requests.count
type: long
- name: success
type: group
fields:
- name: follow_indices.count
type: long

- name: leader
type: group
fields:
Expand All @@ -15,6 +83,9 @@
type: long
description: >
Maximum sequence number of operation on the leader shard
- name: global_checkpoint
type: long

- name: follower
type: group
fields:
Expand All @@ -38,3 +109,15 @@
type: long
description: >
Global checkpoint value on follower shard
- name: max_seq_no
type: long
description: >
Maximum sequence number of operation on the follower shard
- name: mapping_version
type: long
- name: settings_version
type: long
- name: aliases_version
type: long
- name: operations.read.count
type: long
17 changes: 17 additions & 0 deletions metricbeat/module/elasticsearch/ccr/_meta/test/root.710.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"name": "a14cf47ef7f2",
"cluster_name": "docker-cluster",
"cluster_uuid": "8l_zoGznQRmtoX9iSC-goA",
"version": {
"number": "7.10.0",
"build_flavor": "default",
"build_type": "docker",
"build_hash": "43884496262f71aa3f33b34ac2f2271959dbf12a",
"build_date": "2020-10-28T09:54:14.068503Z",
"build_snapshot": true,
"lucene_version": "8.7.0",
"minimum_wire_compatibility_version": "7.11.0",
"minimum_index_compatibility_version": "7.0.0"
},
"tagline": "You Know, for Search"
}
15 changes: 1 addition & 14 deletions metricbeat/module/elasticsearch/ccr/ccr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,20 +89,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
return err
}

if m.XPack {
err = eventsMappingXPack(r, m, *info, content)
if err != nil {
// Since this is an x-pack code path, we log the error but don't
// return it. Otherwise it would get reported into `metricbeat-*`
// indices.
m.Logger().Error(err)
return nil
}
} else {
return eventsMapping(r, *info, content)
}

return nil
return eventsMapping(r, *info, content)
}

func (m *MetricSet) checkCCRAvailability(currentElasticsearchVersion *common.Version) (message string, err error) {
Expand Down
Loading

0 comments on commit da353fe

Please sign in to comment.