Docker grafana automatic provisioning

docker/extra/dashboards/metrictank-http-api.json → dashboards/extra/metrictank-http-api.json

@@ -351,7 +351,7 @@
             "points_returned.p90": "#EF843C"
           },
           "bars": false,
-          "datasource": "${DS_RAINTANKPOC}",
+          "datasource": "$datasource",
           "fill": 1,
           "id": 35,
           "legend": {

docker/extra/dashboards/tsdb-gw.json → dashboards/extra/tsdb-gw.json

@@ -265,7 +265,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_GRAPHITE}",
+      "datasource": "$datasource",
       "editable": true,
       "error": false,
       "fill": 1,
@@ -368,7 +368,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_GRAPHITE}",
+      "datasource": "$datasource",
       "editable": true,
       "error": false,
       "fill": 1,
@@ -481,7 +481,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_GRAPHITE}",
+      "datasource": "$datasource",
       "fill": 1,
       "gridPos": {
         "h": 7,
@@ -567,7 +567,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_GRAPHITE}",
+      "datasource": "$datasource",
       "fill": 1,
       "gridPos": {
         "h": 7,
@@ -669,7 +669,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_GRAPHITE}",
+        "datasource": "$datasource",
         "hide": 0,
         "includeAll": true,
         "label": null,
@@ -689,7 +689,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_GRAPHITE}",
+        "datasource": "$datasource",
         "hide": 0,
         "includeAll": true,
         "label": null,

devdocs/docker.md

@@ -0,0 +1,56 @@
+# docker stacks
+
+You may have already followed [our docker quickstart guide](https://github.com/grafana/metrictank/blob/master/docs/quick-start-docker.md) but in fact,
+the metrictank repository contains a variety of docker stacks for different purposes.
+
+They can be started by cd'ing into their directory and running `docker-compose up --force-recreate -V`
+
+Here's an overview.
+Note:
+* not everything is detailed here. Plus this needs some more work.  For specifics on any setup, please refer to the sources.
+* every environment comes with grafana and cassandra unless otherwise noted
+* every environment automatically provisions useful datasources and dashboards unless otherwise noted
+
+### docker-standard
+
+plain, basic environment.
+uses metrictank docker image (with baked in binaries and configs), statsdaemon.
+Comes with 1 dashboard: the metrictank dashboard
+
+### docker-dev
+
+Similar to docker-standard, but custom metrictank build is loaded, custom configs from scripts/config
+also: tag support, main+extra dashboards
+
+
+### docker-dev-scylla
+
+Same as docker-dev but using scylladb instead of cassandra
+
+### docker-dev-custom-cfg-kafka
+
+Similar to docker-dev in terms of build and configuration.
+Also:
+* kafka input and clustering backend
+* short chunkspan & numchunks 
+  (benefits: easy to trigger mem and mem_and_cass requests, frequent cass saves, notifier messages etc)
+* prometheus monitoring and jaeger tracing
+* caddy-proxied endpooints/datasources to check with different org-id auth headers
+* tsdb-gw
+
+### docker-cluster
+
+Similar to docker-dev-custom-cfg-kafka
+
+* uses a cluster of 4 MT's (with 2x replication)
+* separate graphite monitoring server
+
+### docker-chaos
+
+Similar to docker-cluster but used for chaos testing
+
+
+### docker-cosmosdb
+
+Used for testing against cosmosDB. Note: expects you to have a reachable cosmosdb and adjust settings accordingly.
+Does not auto-load datasources or dashboards.

docker/cfg/grafana-dashboards-main-extra-dockprom/config.yaml

@@ -0,0 +1,27 @@
+apiVersion: 1
+
+providers:
+- name: 'main'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: true
+  updateIntervalSeconds: 3
+  options:
+    path: /var/lib/grafana/dashboards/main
+- name: 'extra'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: true
+  updateIntervalSeconds: 3
+  options:
+    path: /var/lib/grafana/dashboards/extra
+- name: 'dockprom'
+  orgId: 1
+  folder: 'dockprom'
+  type: file
+  disableDeletion: true
+  updateIntervalSeconds: 3
+  options:
+    path: /var/lib/grafana/dashboards/dockprom

docker/cfg/grafana-dashboards-main-extra/config.yaml

@@ -0,0 +1,19 @@
+apiVersion: 1
+
+providers:
+- name: 'main'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: true
+  updateIntervalSeconds: 3
+  options:
+    path: /var/lib/grafana/dashboards/main
+- name: 'extra'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: true
+  updateIntervalSeconds: 3
+  options:
+    path: /var/lib/grafana/dashboards/extra

docker/cfg/grafana-dashboards-main/config.yaml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+- name: 'main'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: true
+  updateIntervalSeconds: 3
+  options:
+    path: /var/lib/grafana/dashboards/main

docker/docker-chaos/docker-compose.yml

@@ -176,6 +176,10 @@ services:
     image: grafana/grafana
     ports:
       - "3000:3000"
+    volumes:
+      - ./grafana-datasources:/etc/grafana/provisioning/datasources
+      - ../cfg/grafana-dashboards-main-extra:/etc/grafana/provisioning/dashboards
+      - ../../dashboards:/var/lib/grafana/dashboards
 
   kafka:
     hostname: kafka

docker/docker-chaos/grafana-datasources/graphite.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+datasources:
+- name: graphite
+  type: graphite
+  access: proxy
+  orgId: 1
+  url: http://graphite
+  isDefault: false
+  jsonData:
+     graphiteVersion: "1.1"
+  version: 1
+  editable: true

docker/docker-chaos/grafana-datasources/metrictank.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+datasources:
+- name: metrictank
+  type: graphite
+  access: proxy
+  orgId: 1
+  url: http://metrictank0:6060
+  isDefault: true
+  jsonData:
+     graphiteVersion: "1.1"
+  version: 1
+  editable: true

docker/docker-chaos/grafana-datasources/monitoring.yaml

@@ -0,0 +1,12 @@
+apiVersion: 1
+datasources:
+- name: monitoring
+  type: graphite
+  access: proxy
+  orgId: 1
+  url: http://graphitemon
+  isDefault: false
+  jsonData:
+     graphiteVersion: "1.1"
+  version: 1
+  editable: true

docker/docker-cluster/docker-compose.yml

@@ -127,6 +127,10 @@ services:
     image: grafana/grafana
     ports:
       - "3000:3000"
+    volumes:
+      - ./grafana-datasources:/etc/grafana/provisioning/datasources
+      - ../cfg/grafana-dashboards-main-extra-dockprom:/etc/grafana/provisioning/dashboards
+      - ../../dashboards:/var/lib/grafana/dashboards
 
   kafka:
     hostname: kafka
@@ -167,6 +171,58 @@ services:
       - "6831:6831/udp"
       - "16686:16686"
 
+  prometheus:
+    image: prom/prometheus:v2.2.1
+    container_name: prometheus
+    volumes:
+      - ./prometheus/:/etc/prometheus/
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--storage.tsdb.retention=200h'
+      - '--web.enable-lifecycle'
+    restart: unless-stopped
+    ports:
+     - "9090:9090"
+    labels:
+      org.label-schema.group: "monitoring"
+
+  nodeexporter:
+    image: prom/node-exporter:v0.15.2
+    container_name: nodeexporter
+    user: root
+    privileged: true
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+    restart: unless-stopped
+    expose:
+      - 9100
+    labels:
+      org.label-schema.group: "monitoring"
+
+  cadvisor:
+    image: google/cadvisor:v0.28.3
+    container_name: cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      #- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
+    restart: unless-stopped
+    expose:
+      - 8080
+    labels:
+      org.label-schema.group: "monitoring"
+
   caddy:
     image: stefanprodan/caddy
     container_name: caddy

docker/docker-cluster/grafana-datasources/graphite.yaml

@@ -0,0 +1,22 @@
+apiVersion: 1
+datasources:
+- name: graphite
+  type: graphite
+  access: proxy
+  orgId: 1
+  url: http://caddy:8081
+  isDefault: false
+  jsonData:
+     graphiteVersion: "1.1"
+  version: 1
+  editable: true
+- name: graphite2
+  type: graphite
+  access: proxy
+  orgId: 1
+  url: http://caddy:8082
+  isDefault: false
+  jsonData:
+     graphiteVersion: "1.1"
+  version: 1
+  editable: true