Skip to content
This repository has been archived by the owner on Jun 24, 2021. It is now read-only.

Commit

Permalink
Support multiple monitor nodes (#1137) (#1143)
Browse files Browse the repository at this point in the history
* Support multiple monitor nodes
  • Loading branch information
liubo0127 authored Feb 14, 2020
1 parent 0159dae commit 12ecbaf
Show file tree
Hide file tree
Showing 6 changed files with 63 additions and 30 deletions.
6 changes: 3 additions & 3 deletions roles/check_config_static/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
---
# Common Tasks

- name: Ensure only one monitoring host exists
fail: msg="One monitoring host should be specified."
when: groups['monitoring_servers'] | length != 1
- name: Ensure monitoring_servers exists
fail: msg="monitoring_servers should be specified."
when: groups['monitoring_servers'] | length < 1

- name: Ensure monitored_servers exists
fail: msg="monitored_servers should be specified."
Expand Down
17 changes: 14 additions & 3 deletions roles/grafana/templates/data_source.json.j2
Original file line number Diff line number Diff line change
@@ -1,9 +1,20 @@
{% set metric_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) if groups.monitoring_servers else "" -%}
{% set metric_port = hostvars[groups.monitoring_servers[0]].prometheus_port if groups.monitoring_servers else ''-%}
{% if groups.monitoring_servers | length == groups.grafana_servers | length -%}
{% set index = [] -%}
{% for host in groups.monitoring_servers -%}
{% if inventory_hostname == hostvars[host].inventory_hostname -%}
{% set _ = index.append(loop.index0) -%}
{% endif -%}
{% endfor -%}
{% set metric_host = hostvars[groups.monitoring_servers[index.0]].ansible_host | default(hostvars[groups.monitoring_servers[index.0]].inventory_hostname) -%}
{% set metric_port = hostvars[groups.monitoring_servers[index.0]].prometheus_port -%}
{% else -%}
{% set metric_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) -%}
{% set metric_port = hostvars[groups.monitoring_servers[0]].prometheus_port -%}
{% endif -%}
{
"name":"{{ cluster_name }}",
"type":"prometheus",
"access":"proxy",
"url":"http://{{ metric_host }}:{{ metric_port }}/",
"basicAuth":false
}
}
59 changes: 40 additions & 19 deletions roles/prometheus/templates/prometheus.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -27,37 +27,49 @@ rule_files:
- 'lightning.rules.yml'
{% endif %}

{% set alertmanager_host = hostvars[groups.alertmanager_servers[0]].ansible_host | default(hostvars[groups.alertmanager_servers[0]].inventory_hostname)
if groups.get('alertmanager_servers', []) else '' -%}
{% set alertmanager_port = hostvars[groups.alertmanager_servers[0]].alertmanager_port if alertmanager_host else '' -%}

{% if alertmanager_target|default("") %}
{% if alertmanager_target|default("") -%}
alerting:
alertmanagers:
- static_configs:
- targets:
- '{{ alertmanager_target }}'
{% elif alertmanager_host %}
{% elif groups.alertmanager_servers -%}
{% if groups.monitoring_servers | length == groups.alertmanager_servers | length -%}
{% set index = [] -%}
{% for host in groups.monitoring_servers -%}
{% if inventory_hostname == hostvars[host].inventory_hostname -%}
{% set _ = index.append(loop.index0) -%}
{% endif -%}
{% endfor -%}
{% set alertmanager_host = hostvars[groups.alertmanager_servers[index.0]].ansible_host | default(hostvars[groups.alertmanager_servers[index.0]].inventory_hostname) -%}
{% set alertmanager_port = hostvars[groups.alertmanager_servers[index.0]].alertmanager_port -%}
alerting:
alertmanagers:
- static_configs:
- targets:
- '{{ alertmanager_host }}:{{ alertmanager_port }}'
{% else %}
{% endif -%}
{% else -%}
# alerting:
# alertmanagers:
# - static_configs:
# - targets:
# - 'alertmanager_host:9093'
{% endif %}
{% endif -%}

{% set pushgateway_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname)
if groups.get('monitoring_servers', []) else '' -%}
{% set pushgateway_port = hostvars[groups.monitoring_servers[0]].pushgateway_port if pushgateway_host else '' -%}
{% set pushgateway_addrs = [] -%}
{% for host in groups.monitoring_servers -%}
{% set pushgateway_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
{% set pushgateway_port = hostvars[host].pushgateway_port -%}
{% set _ = pushgateway_addrs.append("%s:%s" % (pushgateway_ip, pushgateway_port)) -%}
{% endfor -%}

{% set grafana_host = hostvars[groups.grafana_servers[0]].ansible_host | default(hostvars[groups.grafana_servers[0]].inventory_hostname)
if groups.get('grafana_servers', []) else '' -%}
{% set grafana_port = hostvars[groups.grafana_servers[0]].grafana_port if grafana_host else '' -%}
{% set grafana_addrs = [] -%}
{% for host in groups.grafana_servers -%}
{% set grafana_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
{% set grafana_port = hostvars[host].grafana_port -%}
{% set _ = grafana_addrs.append("%s:%s" % (grafana_ip, grafana_port)) -%}
{% endfor -%}

{% set blackbox_host = hostvars[groups.monitored_servers[0]].ansible_host | default(hostvars[groups.monitored_servers[0]].inventory_hostname) if groups.get('monitored_servers', []) else '' -%}
{% set blackbox_port = hostvars[groups.monitored_servers[0]].blackbox_exporter_port if blackbox_host else '' -%}
Expand Down Expand Up @@ -105,12 +117,15 @@ alerting:
{% endfor -%}

scrape_configs:
{% if pushgateway_host %}
{% if pushgateway_addrs %}
- job_name: 'overwritten-cluster'
scrape_interval: 15s
honor_labels: true # don't overwrite job & instance labels
static_configs:
- targets: ['{{ pushgateway_host }}:{{ pushgateway_port }}']
- targets:
{% for pushgateway_addr in pushgateway_addrs %}
- '{{ pushgateway_addr }}'
{% endfor %}

- job_name: "blackbox_exporter_http"
scrape_interval: 30s
Expand All @@ -119,7 +134,9 @@ scrape_configs:
module: [http_2xx]
static_configs:
- targets:
- 'http://{{ pushgateway_host }}:{{ pushgateway_port }}/metrics'
{% for pushgateway_addr in pushgateway_addrs %}
- 'http://{{ pushgateway_addr }}/metrics'
{% endfor %}
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
Expand Down Expand Up @@ -281,11 +298,15 @@ scrape_configs:
labels:
group: 'pd'
- targets:
- '{{ pushgateway_host }}:{{ pushgateway_port }}'
{% for pushgateway_addr in pushgateway_addrs %}
- '{{ pushgateway_addr }}'
{% endfor %}
labels:
group: 'pushgateway'
- targets:
- '{{ grafana_host }}:{{ grafana_port }}'
{% for grafana_addr in grafana_addrs %}
- '{{ grafana_addr }}'
{% endfor %}
labels:
group: 'grafana'
- targets:
Expand Down
4 changes: 2 additions & 2 deletions rolling_update_monitor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,7 @@
- name: import grafana dashboards - prepare config
delegate_to: localhost
template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests.json
template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests-{{ inventory_hostname }}.json
vars:
- ansible_become: false
- ansible_connection: local
Expand Down Expand Up @@ -426,7 +426,7 @@

- name: import grafana dashboards - run import script
delegate_to: localhost
shell: "python grafana-config-copy.py"
shell: "python grafana-config-copy.py dests-{{ inventory_hostname }}.json"
args:
chdir: "{{ playbook_dir }}/scripts"
vars:
Expand Down
3 changes: 2 additions & 1 deletion scripts/grafana-config-copy.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import print_function, \
unicode_literals

import sys
import urllib
import urllib2
import base64
Expand All @@ -23,7 +24,7 @@
]

if not dests:
with open("./dests.json") as fp:
with open(sys.argv[1]) as fp:
dests = json.load(fp)

src = dict(
Expand Down
4 changes: 2 additions & 2 deletions start.yml
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,7 @@
- name: import grafana dashboards - prepare config
delegate_to: localhost
template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests.json
template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests-{{ inventory_hostname }}.json
vars:
- ansible_become: false
- ansible_connection: local
Expand Down Expand Up @@ -485,7 +485,7 @@

- name: import grafana dashboards - run import script
delegate_to: localhost
shell: "python grafana-config-copy.py"
shell: "python grafana-config-copy.py dests-{{ inventory_hostname }}.json"
args:
chdir: "{{ playbook_dir }}/scripts"
vars:
Expand Down

0 comments on commit 12ecbaf

Please sign in to comment.