Skip to content

Commit

Permalink
divide latency of CreateIndex and SaveIndex metrics (#2099)
Browse files Browse the repository at this point in the history 
* divide latency of CreateIndex and SaveIndex metrics

Signed-off-by: Kosuke Morimoto <ksk@vdaas.org>

* divide latency of CreateIndex and SaveIndex Vald Agent metrics

Signed-off-by: Kosuke Morimoto <ksk@vdaas.org>

---------

Signed-off-by: Kosuke Morimoto <ksk@vdaas.org>
Co-authored-by: Yusuke Kato <kpango@vdaas.org>
  • Loading branch information
@kmrmt @kpango @ykadowak
2 people authored and ykadowak committed Jul 25, 2023
1 parent a195594 commit 5a70ba5
Show file tree
Hide file tree
Showing 2 changed files with 294 additions and 54 deletions.
166 changes: 143 additions & 23 deletions k8s/metrics/grafana/dashboards/00-vald-cluster-overview.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ data:
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
Expand Down Expand Up @@ -186,7 +186,7 @@ data:
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
Expand Down Expand Up @@ -264,7 +264,7 @@ data:
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
Expand Down Expand Up @@ -324,7 +324,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -429,7 +429,7 @@ data:
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
Expand Down Expand Up @@ -507,7 +507,7 @@ data:
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
Expand Down Expand Up @@ -590,7 +590,7 @@ data:
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
Expand Down Expand Up @@ -674,7 +674,7 @@ data:
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"targets": [
{
"datasource": {
Expand Down Expand Up @@ -733,7 +733,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -823,7 +823,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -932,7 +932,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -1054,7 +1054,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -1163,7 +1163,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand All @@ -1178,7 +1178,7 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\"}[$interval])) by (le, grpc_server_method))",
"expr": "histogram_quantile(0.5, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\", grpc_server_method!~\".*Index$\"}[$interval])) by (le, grpc_server_method))",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{grpc_server_method}} p50",
Expand All @@ -1191,7 +1191,7 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\"}[$interval])) by (le, grpc_server_method))",
"expr": "histogram_quantile(0.95, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\", grpc_server_method!~\".*Index$\"}[$interval])) by (le, grpc_server_method))",
"interval": "",
"legendFormat": "{{grpc_server_method}} p95",
"range": true,
Expand All @@ -1203,7 +1203,7 @@ data:
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\"}[$interval])) by (le, grpc_server_method))",
"expr": "histogram_quantile(0.99, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\", grpc_server_method!~\".*Index$\"}[$interval])) by (le, grpc_server_method))",
"interval": "",
"legendFormat": "{{grpc_server_method}} p99",
"range": true,
Expand Down Expand Up @@ -1283,7 +1283,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -1379,7 +1379,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -1475,7 +1475,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -1543,6 +1543,126 @@ data:
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 36
},
"hiddenSeries": false,
"id": 183,
"interval": "",
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\", grpc_server_method=~\".*Index$\"}[$interval])) by (le, grpc_server_method))",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{grpc_server_method}} p50",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\", grpc_server_method=~\".*Index$\"}[$interval])) by (le, grpc_server_method))",
"interval": "",
"legendFormat": "{{grpc_server_method}} p95",
"range": true,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(server_latency_bucket{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$ValdAgentPodName\", grpc_server_method=~\".*Index$\"}[$interval])) by (le, grpc_server_method))",
"interval": "",
"legendFormat": "{{grpc_server_method}} p99",
"range": true,
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Latency (Vald Agent Index)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "ms",
"logBase": 1,
"min": "0",
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"collapsed": false,
"datasource": {
Expand Down Expand Up @@ -1620,7 +1740,7 @@ data:
"text": {},
"textMode": "auto"
},
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"repeatDirection": "v",
"targets": [
{
Expand Down Expand Up @@ -1682,7 +1802,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -1851,7 +1971,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down Expand Up @@ -1943,7 +2063,7 @@ data:
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.4.3",
"pluginVersion": "10.0.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
Expand Down
Loading

0 comments on commit 5a70ba5

Please sign in to comment.