From b6e3487807f4abc742886c747acadf9b12c8d7d7 Mon Sep 17 00:00:00 2001 From: ido Date: Mon, 30 Sep 2024 12:44:41 -0400 Subject: [PATCH 1/2] rollup apps monitoring and alerts --- charts/composer/Chart.yaml | 2 +- charts/composer/templates/_helpers.tpl | 21 +++++++++++++++++ charts/composer/templates/deployment.yaml | 2 +- charts/composer/templates/service.yaml | 6 ++--- charts/composer/templates/servicemonitor.yaml | 2 +- charts/composer/values.yaml | 23 +++++++++++++++++++ charts/evm-bridge-withdrawer/Chart.yaml | 2 +- .../templates/_helpers.tpl | 21 +++++++++++++++++ .../templates/prometheusrule.yaml | 20 ++++++++++++++++ .../templates/service.yaml | 4 +++- .../templates/servicemonitor.yaml | 2 +- charts/evm-bridge-withdrawer/values.yaml | 23 +++++++++++++++++++ .../evm-rollup/templates/prometheusrule.yaml | 2 +- charts/evm-stack/Chart.lock | 8 +++---- charts/evm-stack/Chart.yaml | 4 ++-- 15 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 charts/evm-bridge-withdrawer/templates/prometheusrule.yaml diff --git a/charts/composer/Chart.yaml b/charts/composer/Chart.yaml index a5e2ac72fe..26b0666774 100644 --- a/charts/composer/Chart.yaml +++ b/charts/composer/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.4 +version: 0.1.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/composer/templates/_helpers.tpl b/charts/composer/templates/_helpers.tpl index 5d11460167..6e45bc120a 100644 --- a/charts/composer/templates/_helpers.tpl +++ b/charts/composer/templates/_helpers.tpl @@ -5,6 +5,27 @@ Namepsace to deploy elements into. {{- default .Release.Namespace .Values.global.namespaceOverride | trunc 63 | trimSuffix "-" -}} {{- end }} +{{/* +application name to deploy elements into. +*/}} +{{- define "composer.appName" -}} +composer +{{- end }} + +{{/* +Common labels +*/}} +{{- define "composer.labels" -}} +{{ include "rollup.selectorLabels" . }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "composer.selectorLabels" -}} +app: {{ include "rollup.appName" . }} +{{- end }} + {{/* Single entry of rollup names */}} diff --git a/charts/composer/templates/deployment.yaml b/charts/composer/templates/deployment.yaml index 24a19a8759..118356dc84 100644 --- a/charts/composer/templates/deployment.yaml +++ b/charts/composer/templates/deployment.yaml @@ -39,7 +39,7 @@ spec: ports: {{- if .Values.metrics.enabled }} - containerPort: {{ .Values.ports.metrics }} - name: metrics + name: composer-metrics {{- end }} - containerPort: {{ .Values.ports.healthApi }} name: health-api diff --git a/charts/composer/templates/service.yaml b/charts/composer/templates/service.yaml index cf317f8330..b03e325407 100644 --- a/charts/composer/templates/service.yaml +++ b/charts/composer/templates/service.yaml @@ -15,7 +15,7 @@ spec: kind: Service apiVersion: v1 metadata: - name: metrics + name: composer-metrics namespace: {{ include "composer.namespace" . }} labels: app: composer @@ -23,7 +23,7 @@ spec: selector: app: composer ports: - - name: metrics + - name: composer-metrics port: {{ .Values.ports.metrics }} - targetPort: geth-metr + targetPort: composer-metrics {{- end }} diff --git a/charts/composer/templates/servicemonitor.yaml b/charts/composer/templates/servicemonitor.yaml index aa3e4c05b9..b88fc369c1 100644 --- a/charts/composer/templates/servicemonitor.yaml +++ b/charts/composer/templates/servicemonitor.yaml @@ -16,7 +16,7 @@ spec: matchLabels: app: composer endpoints: - - port: metrics + - port: composer-metrics path: / {{- with .Values.serviceMonitor.interval }} interval: {{ . }} diff --git a/charts/composer/values.yaml b/charts/composer/values.yaml index 1aec56c3e1..95e79f9185 100644 --- a/charts/composer/values.yaml +++ b/charts/composer/values.yaml @@ -50,6 +50,29 @@ serviceMonitor: additionalLabels: release: kube-prometheus-stack +alerting: + enabled: false + interval: "" + additionalLabels: + release: kube-prometheus-stack + annotations: {} + # scrapeTimeout: 10s + # path: /metrics + prometheusRule: + enabled: true + additionalLabels: + release: kube-prometheus-stack + namespace: monitoring + rules: + - alert: Composer_Node_Down + expr: up{container="composer"} == 0 # Insert your query Expression + for: 1m # Rough number but should be enough to init warn + labels: + severity: warning + annotations: + summary: Composer is Down (instance {{ $labels.instance }}) + description: "composer node '{{ $labels.namespace }}' has disappeared from Prometheus target discovery.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # When deploying in a production environment should use a secret provider # This is configured for use with GCP, need to set own resource names # and keys diff --git a/charts/evm-bridge-withdrawer/Chart.yaml b/charts/evm-bridge-withdrawer/Chart.yaml index 096054476b..667ef89c1a 100644 --- a/charts/evm-bridge-withdrawer/Chart.yaml +++ b/charts/evm-bridge-withdrawer/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.0 +version: 0.3.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/evm-bridge-withdrawer/templates/_helpers.tpl b/charts/evm-bridge-withdrawer/templates/_helpers.tpl index 84c9085ebd..b4d5894371 100644 --- a/charts/evm-bridge-withdrawer/templates/_helpers.tpl +++ b/charts/evm-bridge-withdrawer/templates/_helpers.tpl @@ -8,3 +8,24 @@ Namepsace to deploy elements into. {{- define "evm-bridge-withdrawer.image" -}} {{ .Values.images.evmBridgeWithdrawer.repo }}:{{ if .Values.global.dev }}{{ .Values.images.evmBridgeWithdrawer.devTag }}{{ else }}{{ .Values.images.evmBridgeWithdrawer.tag }}{{ end }} {{- end }} + +{{/* +application name to deploy elements into. +*/}} +{{- define "evm-bridge-withdrawer.appName" -}} +evm-bridge-withdrawer +{{- end }} + +{{/* +Common labels +*/}} +{{- define "evm-bridge-withdrawer.labels" -}} +{{ include "evm-bridge-withdrawer.selectorLabels" . }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "evm-bridge-withdrawer.selectorLabels" -}} +app: {{ include "evm-bridge-withdrawer.appName" . }} +{{- end }} diff --git a/charts/evm-bridge-withdrawer/templates/prometheusrule.yaml b/charts/evm-bridge-withdrawer/templates/prometheusrule.yaml new file mode 100644 index 0000000000..7d746c9074 --- /dev/null +++ b/charts/evm-bridge-withdrawer/templates/prometheusrule.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alerting.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: evm-bridge-withdrawer-alerting +{{- if .Values.alerting.prometheusRule.namespace }} + namespace: {{ .Values.alerting.prometheusRule.namespace | quote }} +{{- end }} + labels: + {{- include "evm-bridge-withdrawer.labels" . | nindent 4 }} + {{- if .Values.alerting.prometheusRule.additionalLabels }} + {{- toYaml .Values.alerting.prometheusRule.additionalLabels | nindent 4 }} + {{- end }} +spec: +{{- if .Values.alerting.prometheusRule.rules }} + groups: + - name: {{ template "evm-bridge-withdrawer.appName" . }} + rules: {{- toYaml .Values.alerting.prometheusRule.rules | nindent 4 }} +{{- end }} +{{- end }} diff --git a/charts/evm-bridge-withdrawer/templates/service.yaml b/charts/evm-bridge-withdrawer/templates/service.yaml index 6f16cc4540..c66a097ba4 100644 --- a/charts/evm-bridge-withdrawer/templates/service.yaml +++ b/charts/evm-bridge-withdrawer/templates/service.yaml @@ -4,9 +4,11 @@ apiVersion: v1 metadata: name: evm-bridge-withdrawer-metrics namespace: {{ include "evm-bridge-withdrawer.namespace" . }} + labels: + {{- include "evm-bridge-withdrawer.labels" . | nindent 4 }} spec: selector: - app: evm-bridge-withdrawer + {{- include "evm-bridge-withdrawer.labels" . | nindent 4 }} ports: {{- if .Values.metrics.enabled }} - name: metrics diff --git a/charts/evm-bridge-withdrawer/templates/servicemonitor.yaml b/charts/evm-bridge-withdrawer/templates/servicemonitor.yaml index dace167a38..e196d5e545 100644 --- a/charts/evm-bridge-withdrawer/templates/servicemonitor.yaml +++ b/charts/evm-bridge-withdrawer/templates/servicemonitor.yaml @@ -17,7 +17,7 @@ spec: matchLabels: app: evm-bridge-withdrawer endpoints: - - port: metric + - port: metrics path: / {{- with .Values.serviceMonitor.interval }} interval: {{ . }} diff --git a/charts/evm-bridge-withdrawer/values.yaml b/charts/evm-bridge-withdrawer/values.yaml index a84d3e951a..62a723a77e 100644 --- a/charts/evm-bridge-withdrawer/values.yaml +++ b/charts/evm-bridge-withdrawer/values.yaml @@ -54,6 +54,29 @@ serviceMonitor: additionalLabels: release: kube-prometheus-stack +alerting: + enabled: false + interval: "" + additionalLabels: + release: kube-prometheus-stack + annotations: {} + # scrapeTimeout: 10s + # path: /metrics + prometheusRule: + enabled: true + additionalLabels: + release: kube-prometheus-stack + namespace: monitoring + rules: + - alert: Bridge_Withdrawer_Down + expr: up{container="evm-bridge-withdrawer"} == 0 # Insert your query Expression + for: 1m # Rough number but should be enough to init warn + labels: + severity: warning + annotations: + summary: Bridge Withdrawer is Down (instance {{ $labels.instance }}) + description: "bridge withdrawer node '{{ $labels.namespace }}' has disappeared from Prometheus target discovery.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # When deploying in a production environment should use a secret provider # This is configured for use with GCP, need to set own resource names # and keys diff --git a/charts/evm-rollup/templates/prometheusrule.yaml b/charts/evm-rollup/templates/prometheusrule.yaml index 128a52f883..225805dd51 100644 --- a/charts/evm-rollup/templates/prometheusrule.yaml +++ b/charts/evm-rollup/templates/prometheusrule.yaml @@ -12,7 +12,7 @@ metadata: {{- toYaml .Values.alerting.prometheusRule.additionalLabels | nindent 4 }} {{- end }} spec: -{{- if .Values.config.rollup.alerting.prometheusRule.rules }} +{{- if .Values.alerting.prometheusRule.rules }} groups: - name: {{ template "rollup.name" . }} rules: {{- toYaml .Values.alerting.prometheusRule.rules | nindent 4 }} diff --git a/charts/evm-stack/Chart.lock b/charts/evm-stack/Chart.lock index 7b103a22aa..f71bba767e 100644 --- a/charts/evm-stack/Chart.lock +++ b/charts/evm-stack/Chart.lock @@ -7,18 +7,18 @@ dependencies: version: 0.27.5 - name: composer repository: file://../composer - version: 0.1.4 + version: 0.1.5 - name: evm-faucet repository: file://../evm-faucet version: 0.1.2 - name: evm-bridge-withdrawer repository: file://../evm-bridge-withdrawer - version: 0.3.0 + version: 0.3.1 - name: postgresql repository: https://charts.bitnami.com/bitnami version: 15.2.4 - name: blockscout-stack repository: https://blockscout.github.io/helm-charts version: 1.6.2 -digest: sha256:2d7e2f23cd9bbdb43b7cf42112db9ede0a7d5eee9e426b0b2344e43fcf52e1b1 -generated: "2024-10-02T09:43:51.238571-04:00" +digest: sha256:4f00554b05258b2457715803fb1828a60942abb85262967ef8519d733ddee560 +generated: "2024-10-03T17:18:58.503828-04:00" diff --git a/charts/evm-stack/Chart.yaml b/charts/evm-stack/Chart.yaml index e2cf556597..2e5998c88d 100644 --- a/charts/evm-stack/Chart.yaml +++ b/charts/evm-stack/Chart.yaml @@ -26,7 +26,7 @@ dependencies: version: 0.27.5 repository: "file://../evm-rollup" - name: composer - version: 0.1.4 + version: 0.1.5 repository: "file://../composer" condition: composer.enabled - name: evm-faucet @@ -34,7 +34,7 @@ dependencies: repository: "file://../evm-faucet" condition: evm-faucet.enabled - name: evm-bridge-withdrawer - version: 0.3.0 + version: 0.3.1 repository: "file://../evm-bridge-withdrawer" condition: evm-bridge-withdrawer.enabled - name: postgresql From 49b107c5c5fe9a36944eb3eef15a9f2f23d2d4f4 Mon Sep 17 00:00:00 2001 From: ido Date: Mon, 14 Oct 2024 15:12:03 +0200 Subject: [PATCH 2/2] bump chart version --- charts/evm-rollup/Chart.yaml | 2 +- charts/evm-stack/Chart.lock | 6 +++--- charts/evm-stack/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/evm-rollup/Chart.yaml b/charts/evm-rollup/Chart.yaml index 9222e2df02..44489a1b38 100644 --- a/charts/evm-rollup/Chart.yaml +++ b/charts/evm-rollup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.27.5 +version: 0.27.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/evm-stack/Chart.lock b/charts/evm-stack/Chart.lock index f71bba767e..59d9b0d782 100644 --- a/charts/evm-stack/Chart.lock +++ b/charts/evm-stack/Chart.lock @@ -4,7 +4,7 @@ dependencies: version: 0.3.6 - name: evm-rollup repository: file://../evm-rollup - version: 0.27.5 + version: 0.27.6 - name: composer repository: file://../composer version: 0.1.5 @@ -20,5 +20,5 @@ dependencies: - name: blockscout-stack repository: https://blockscout.github.io/helm-charts version: 1.6.2 -digest: sha256:4f00554b05258b2457715803fb1828a60942abb85262967ef8519d733ddee560 -generated: "2024-10-03T17:18:58.503828-04:00" +digest: sha256:0428a6d56fd86c170e322ad79c7b5f87628b6187a1df5ad47ae7c2281b7f12da +generated: "2024-10-14T15:11:45.153501+02:00" diff --git a/charts/evm-stack/Chart.yaml b/charts/evm-stack/Chart.yaml index 2e5998c88d..b208660420 100644 --- a/charts/evm-stack/Chart.yaml +++ b/charts/evm-stack/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.6.2 +version: 0.6.3 dependencies: - name: celestia-node @@ -23,7 +23,7 @@ dependencies: repository: "file://../celestia-node" condition: celestia-node.enabled - name: evm-rollup - version: 0.27.5 + version: 0.27.6 repository: "file://../evm-rollup" - name: composer version: 0.1.5