From 65d7d5a65763e7ef299aa648e213777bc38960c6 Mon Sep 17 00:00:00 2001 From: fangfang qi <947321353@qq.com> Date: Mon, 30 Sep 2019 14:09:13 +0800 Subject: [PATCH] Monitoring: Add Node & Pod info in TiDB Grafana (#885) --- .../templates/config/_grafana-datasource.tpl | 14 ------ .../templates/monitor-configmap.yaml | 2 - .../templates/monitor-deployment.yaml | 17 +++++-- charts/tidb-cluster/values.yaml | 4 +- tests/actions.go | 49 ++++++++++++++++++- 5 files changed, 63 insertions(+), 23 deletions(-) delete mode 100644 charts/tidb-cluster/templates/config/_grafana-datasource.tpl diff --git a/charts/tidb-cluster/templates/config/_grafana-datasource.tpl b/charts/tidb-cluster/templates/config/_grafana-datasource.tpl deleted file mode 100644 index 5713bfb6c7..0000000000 --- a/charts/tidb-cluster/templates/config/_grafana-datasource.tpl +++ /dev/null @@ -1,14 +0,0 @@ -{ - "apiVersion": 1, - "datasources": [ - { - "access": "proxy", - "editable": false, - "name": "tidb-cluster", - "orgId": 1, - "type": "prometheus", - "url": "http://127.0.0.1:9090", - "version": 1 - } - ] -} diff --git a/charts/tidb-cluster/templates/monitor-configmap.yaml b/charts/tidb-cluster/templates/monitor-configmap.yaml index 771a90808b..8d6ea1c4b8 100644 --- a/charts/tidb-cluster/templates/monitor-configmap.yaml +++ b/charts/tidb-cluster/templates/monitor-configmap.yaml @@ -13,8 +13,6 @@ data: prometheus-config: |- {{ tuple "config/_prometheus-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- if .Values.monitor.grafana.create }} - datasource-config: |- -{{ tuple "config/_grafana-datasource.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} dashboard-config: |- {{ tuple "config/_grafana-dashboard.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/charts/tidb-cluster/templates/monitor-deployment.yaml b/charts/tidb-cluster/templates/monitor-deployment.yaml index 9a06fc1fe1..e970edf199 100644 --- a/charts/tidb-cluster/templates/monitor-deployment.yaml +++ b/charts/tidb-cluster/templates/monitor-deployment.yaml @@ -56,12 +56,20 @@ spec: env: - name: GF_PROVISIONING_PATH value: /grafana-dashboard-definitions/tidb + - name: GF_DATASOURCE_PATH + value: /etc/grafana/provisioning/datasources - name: TIDB_CLUSTER_NAME value: {{ template "cluster.name" . }} - name: TIDB_ENABLE_BINLOG value: {{ .Values.binlog.pump.create | default false | quote }} - name: PROM_CONFIG_PATH value: /prometheus-rules + - name: GF_K8S_PROMETHEUS_URL + value: {{ .Values.monitor.initializer.config.K8S_PROMETHEUS_URL }} + - name: GF_TIDB_PROMETHEUS_URL + value: http://127.0.0.1:9090 + - name: TIDB_CLUSTER_NAMESPACE + value: {{ .Release.Namespace }} command: - /bin/sh - -c @@ -80,6 +88,9 @@ spec: readOnly: false - name: monitor-data mountPath: /data + - name: datasource + mountPath: /etc/grafana/provisioning/datasources + readOnly: false resources: {{ toYaml .Values.monitor.initializer.resources | indent 10 }} containers: @@ -195,11 +206,7 @@ spec: - key: prometheus-config path: prometheus.yml {{- if .Values.monitor.grafana.create }} - - configMap: - name: {{ template "cluster.name" . }}-monitor - items: - - key: datasource-config - path: datasource.yaml + - emptyDir: {} name: datasource - configMap: name: {{ template "cluster.name" . }}-monitor diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 556a59a0db..bd6e6f9bb7 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -394,7 +394,9 @@ monitor: storage: 10Gi initializer: image: pingcap/tidb-monitor-initializer:v3.0.1 - imagePullPolicy: IfNotPresent + imagePullPolicy: Always + config: + K8S_PROMETHEUS_URL: http://prometheus-k8s.monitoring.svc:9090 resources: {} # limits: # cpu: 50m diff --git a/tests/actions.go b/tests/actions.go index e3389f5055..f057727b25 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -1750,7 +1750,13 @@ func (oa *operatorActions) checkGrafanaData(clusterInfo *TidbClusterConfig) erro values.Set("start", fmt.Sprintf("%d", start.Unix())) values.Set("end", fmt.Sprintf("%d", end.Unix())) values.Set("step", "30") - u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources/proxy/1/api/v1/query_range?%s", svcName, ns, values.Encode()) + + datasourceID, err := getDatasourceID(svcName, ns) + if err != nil { + return err + } + + u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources/proxy/%d/api/v1/query_range?%s", svcName, ns, datasourceID, values.Encode()) req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { return err @@ -1797,6 +1803,47 @@ func (oa *operatorActions) checkGrafanaData(clusterInfo *TidbClusterConfig) erro return nil } +func getDatasourceID(svcName, namespace string) (int, error) { + u := fmt.Sprintf("http://%s.%s.svc.cluster.local:3000/api/datasources", svcName, namespace) + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return 0, err + } + + req.SetBasicAuth(grafanaUsername, grafanaPassword) + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return 0, err + } + defer func() { + err := resp.Body.Close() + glog.Warning("close response failed", err) + }() + + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, err + } + + datasources := []struct { + Id int `json:"id"` + Name string `json:"name"` + }{} + + if err := json.Unmarshal(buf, &datasources); err != nil { + return 0, err + } + + for _, ds := range datasources { + if ds.Name == "tidb-cluster" { + return ds.Id, nil + } + } + + return 0, pingcapErrors.New("not found tidb-cluster datasource") +} + func GetD(ns, tcName, databaseName, password string) string { return fmt.Sprintf("root:%s@(%s-tidb.%s:4000)/%s?charset=utf8", password, tcName, ns, databaseName) }