Prometheus horizontal sharding and federation

[h0tbird]

For the fun of it:

http://www.robustperception.io/scaling-and-federating-prometheus/
http://www.robustperception.io/life-of-a-label/

[h0tbird]

global:
 external_labels:
  master: 1
 scrape_interval: 1m
 scrape_timeout: 10s
 evaluation_interval: 10s

rule_files:
 - /etc/prometheus/recording.rules
 - /etc/prometheus/alerting.rules

alerting:
 alert_relabel_configs:
 - source_labels: [master]
   action: replace
   replacement: 'all'
   target_label: master

scrape_configs:

 - job_name: 'federate'
   honor_labels: true
   metrics_path: /federate
   params:
    match[]:
     - '{master=~"2|3"}'
   static_configs:
    - targets:
      - 'master-2.cell-1.dub.xnood.com:9191'
      - 'master-3.cell-1.dub.xnood.com:9191'

 - job_name: 'prometheus'
   scrape_interval: 10s
   file_sd_configs:
    - files:
      - /etc/prometheus/targets/prometheus.yml

 - job_name: 'cadvisor'
   scrape_interval: 10s
   file_sd_configs:
    - files:
      - /etc/prometheus/targets/cadvisor.yml
   relabel_configs:
   - source_labels: [__address__]
     modulus: 3
     target_label: __tmp_hash
     action: hashmod
   - source_labels: [__tmp_hash]
     regex: ^0$
     action: keep

 - job_name: 'etcd'
   scrape_interval: 10s
   file_sd_configs:
    - files:
      - /etc/prometheus/targets/etcd.yml
   relabel_configs:
   - source_labels: [__address__]
     modulus: 3
     target_label: __tmp_hash
     action: hashmod
   - source_labels: [__tmp_hash]
     regex: ^0$
     action: keep

 - job_name: 'node'
   scrape_interval: 10s
   file_sd_configs:
    - files:
      - /etc/prometheus/targets/node.yml
   relabel_configs:
   - source_labels: [__address__]
     modulus: 3
     target_label: __tmp_hash
     action: hashmod
   - source_labels: [__tmp_hash]
     regex: ^0$
     action: keep

 - job_name: 'mesos'
   scrape_interval: 10s
   file_sd_configs:
    - files:
      - /etc/prometheus/targets/mesos.yml

 - job_name: 'haproxy'
   scrape_interval: 10s
   file_sd_configs:
    - files:
      - /etc/prometheus/targets/haproxy.yml

 - job_name: 'zookeeper'
   scrape_interval: 10s
   file_sd_configs:
    - files:
      - /etc/prometheus/targets/zookeeper.yml