From 13a3fdfa013e6e80d133452be10bc213402ee8eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Pie=C5=84kowski?= <8525083+Jakski@users.noreply.github.com> Date: Sun, 13 Jan 2019 17:35:43 +0100 Subject: [PATCH 1/4] cmd/scollector: pull index data from non-master hosts (#2259) --- cmd/scollector/collectors/elasticsearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/scollector/collectors/elasticsearch.go b/cmd/scollector/collectors/elasticsearch.go index db54f99f05..c15be0fca7 100644 --- a/cmd/scollector/collectors/elasticsearch.go +++ b/cmd/scollector/collectors/elasticsearch.go @@ -289,7 +289,7 @@ func c_elasticsearch(collectIndices bool, instance conf.Elastic) (opentsdb.Multi s.add("elastic.jvm.gc", nodeStats.JVM.GC.Collectors.Old, opentsdb.TagSet{"gc": "old"}.Merge(ts)) s.add("elastic.jvm.gc", nodeStats.JVM.GC.Collectors.Young, opentsdb.TagSet{"gc": "young"}.Merge(ts)) } - if collectIndices && isMaster { + if collectIndices { for k, index := range indexStats.Indices { if esSkipIndex(k) { slog.Infof("Skipping index: %v", k) From 30f97296c41c0f7ce5f05d150df850097de7bcf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Pie=C5=84kowski?= <8525083+Jakski@users.noreply.github.com> Date: Wed, 16 Jan 2019 00:25:42 +0100 Subject: [PATCH 2/4] cmd/scollector: add AlwaysCollectIndices setting --- cmd/scollector/collectors/elasticsearch.go | 2 +- cmd/scollector/conf/conf.go | 19 ++++++++++--------- cmd/scollector/doc.go | 5 +++++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/cmd/scollector/collectors/elasticsearch.go b/cmd/scollector/collectors/elasticsearch.go index c15be0fca7..bc651f670c 100644 --- a/cmd/scollector/collectors/elasticsearch.go +++ b/cmd/scollector/collectors/elasticsearch.go @@ -289,7 +289,7 @@ func c_elasticsearch(collectIndices bool, instance conf.Elastic) (opentsdb.Multi s.add("elastic.jvm.gc", nodeStats.JVM.GC.Collectors.Old, opentsdb.TagSet{"gc": "old"}.Merge(ts)) s.add("elastic.jvm.gc", nodeStats.JVM.GC.Collectors.Young, opentsdb.TagSet{"gc": "young"}.Merge(ts)) } - if collectIndices { + if collectIndices && (isMaster || instance.AlwaysCollectIndices) { for k, index := range indexStats.Indices { if esSkipIndex(k) { slog.Infof("Skipping index: %v", k) diff --git a/cmd/scollector/conf/conf.go b/cmd/scollector/conf/conf.go index dc6e413733..eeada8d07f 100644 --- a/cmd/scollector/conf/conf.go +++ b/cmd/scollector/conf/conf.go @@ -271,13 +271,14 @@ type OracleInstance struct { // Optional Elastic instance configuration - if omitted then the defaults are used // You can also define multiple instances where more than one node is running type Elastic struct { - Host string // default is localhost - Port uint16 // default is 9200 - ClusterInterval string // default is DefaultFreq - IndexInterval string // default is 15 mins - User string // default is empty - Password string // default is empty - Disable bool // default is false. - Name string // default is host_port - Scheme string // default is http + Host string // default is localhost + Port uint16 // default is 9200 + ClusterInterval string // default is DefaultFreq + IndexInterval string // default is 15 mins + User string // default is empty + Password string // default is empty + Disable bool // default is false. + Name string // default is host_port + Scheme string // default is http + AlwaysCollectIndices bool // default is false } diff --git a/cmd/scollector/doc.go b/cmd/scollector/doc.go index 180fd3190c..ec5e24ed20 100644 --- a/cmd/scollector/doc.go +++ b/cmd/scollector/doc.go @@ -432,6 +432,11 @@ you specify basic auth credentials and using TLS by setting the Scheme to https: ClusterInterval = "10s" IndexInterval = "1m" +Collecting indices metrics from all nodes may flood database with tags. By +default they are collected, only if current node is master. You can +override this with AlwaysCollectIndices. + + Windows scollector has full Windows support. It can be run standalone, or installed as a From 40208fda15b879d7186b0edfccae80584eadd139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Pie=C5=84kowski?= <8525083+Jakski@users.noreply.github.com> Date: Wed, 16 Jan 2019 00:32:00 +0100 Subject: [PATCH 3/4] cmd/scollector: Fix spacing in documentation --- cmd/scollector/doc.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/scollector/doc.go b/cmd/scollector/doc.go index ec5e24ed20..00bb03004c 100644 --- a/cmd/scollector/doc.go +++ b/cmd/scollector/doc.go @@ -436,7 +436,6 @@ Collecting indices metrics from all nodes may flood database with tags. By default they are collected, only if current node is master. You can override this with AlwaysCollectIndices. - Windows scollector has full Windows support. It can be run standalone, or installed as a From de9dc85ec746830bb88eda5232f612869adbe7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Pie=C5=84kowski?= <8525083+Jakski@users.noreply.github.com> Date: Wed, 16 Jan 2019 01:03:39 +0100 Subject: [PATCH 4/4] cmd/scollector: Change AlwaysCollectIndices to CollectAllMetrics --- cmd/scollector/collectors/elasticsearch.go | 4 ++-- cmd/scollector/conf/conf.go | 20 ++++++++++---------- cmd/scollector/doc.go | 5 ++--- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cmd/scollector/collectors/elasticsearch.go b/cmd/scollector/collectors/elasticsearch.go index bc651f670c..8128d1a3aa 100644 --- a/cmd/scollector/collectors/elasticsearch.go +++ b/cmd/scollector/collectors/elasticsearch.go @@ -266,7 +266,7 @@ func c_elasticsearch(collectIndices bool, instance conf.Elastic) (opentsdb.Multi // As we're pulling _local stats here, this will only process 1 node. for nodeID, nodeStats := range clusterStats.Nodes { isMaster = nodeID == clusterState.MasterNode - if isMaster { + if isMaster || instance.CollectAllMetrics { s.add("elastic.health.cluster", clusterHealth, ts) if statusCode, ok := elasticStatusMap[clusterHealth.Status]; ok { Add(&md, "elastic.health.cluster.status", statusCode, ts, metadata.Gauge, metadata.StatusCode, "The current status of the cluster. Zero for green, one for yellow, two for red.") @@ -289,7 +289,7 @@ func c_elasticsearch(collectIndices bool, instance conf.Elastic) (opentsdb.Multi s.add("elastic.jvm.gc", nodeStats.JVM.GC.Collectors.Old, opentsdb.TagSet{"gc": "old"}.Merge(ts)) s.add("elastic.jvm.gc", nodeStats.JVM.GC.Collectors.Young, opentsdb.TagSet{"gc": "young"}.Merge(ts)) } - if collectIndices && (isMaster || instance.AlwaysCollectIndices) { + if collectIndices && (isMaster || instance.CollectAllMetrics) { for k, index := range indexStats.Indices { if esSkipIndex(k) { slog.Infof("Skipping index: %v", k) diff --git a/cmd/scollector/conf/conf.go b/cmd/scollector/conf/conf.go index eeada8d07f..ddbb6e8708 100644 --- a/cmd/scollector/conf/conf.go +++ b/cmd/scollector/conf/conf.go @@ -271,14 +271,14 @@ type OracleInstance struct { // Optional Elastic instance configuration - if omitted then the defaults are used // You can also define multiple instances where more than one node is running type Elastic struct { - Host string // default is localhost - Port uint16 // default is 9200 - ClusterInterval string // default is DefaultFreq - IndexInterval string // default is 15 mins - User string // default is empty - Password string // default is empty - Disable bool // default is false. - Name string // default is host_port - Scheme string // default is http - AlwaysCollectIndices bool // default is false + Host string // default is localhost + Port uint16 // default is 9200 + ClusterInterval string // default is DefaultFreq + IndexInterval string // default is 15 mins + User string // default is empty + Password string // default is empty + Disable bool // default is false. + Name string // default is host_port + Scheme string // default is http + CollectAllMetrics bool // default is false } diff --git a/cmd/scollector/doc.go b/cmd/scollector/doc.go index 00bb03004c..cefea1b43a 100644 --- a/cmd/scollector/doc.go +++ b/cmd/scollector/doc.go @@ -432,9 +432,8 @@ you specify basic auth credentials and using TLS by setting the Scheme to https: ClusterInterval = "10s" IndexInterval = "1m" -Collecting indices metrics from all nodes may flood database with tags. By -default they are collected, only if current node is master. You can -override this with AlwaysCollectIndices. +By default cluster-wide metrics are collected, only if current node is master. +You can override this behaviour with CollectAllMetrics switch. Windows