feat(monitoring): Add optional ServiceMonitors to all JVM based services

charts/datahub/Chart.yaml

@@ -4,25 +4,25 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.106
+version: 0.2.107
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.8.45
 dependencies:
   - name: datahub-gms
-    version: 0.2.9
+    version: 0.2.10
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend
-    version: 0.2.9
+    version: 0.2.10
     repository: file://./subcharts/datahub-frontend
     condition: datahub-frontend.enabled
   - name: datahub-mae-consumer
-    version: 0.2.7
+    version: 0.2.8
     repository: file://./subcharts/datahub-mae-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer
-    version: 0.2.8
+    version: 0.2.9
     repository: file://./subcharts/datahub-mce-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-ingestion-cron

charts/datahub/subcharts/datahub-frontend/Chart.yaml

@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.9
+version: 0.2.10
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

charts/datahub/subcharts/datahub-frontend/README.md

@@ -58,6 +58,7 @@ Current chart version is `0.2.0`
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `nil` |  |
+| serviceMonitoring.create | bool | `false` | If set true and `global.datahub.monitoring.enablePrometheus` is set `true` it will create a ServiceMonitor resource |
 | tolerations | list | `[]` |  |
 | global.elasticsearch.host | string | `"elasticsearch"` |  |
 | global.elasticsearch.port | string | `"9200"` |  |

charts/datahub/subcharts/datahub-frontend/templates/deployment.yaml

@@ -36,11 +36,6 @@ spec:
             defaultMode: 0444
             secretName: {{ .name }}
         {{- end }}
-        {{- if .Values.exporters.jmx.enabled }}
-        - name: config-jmx-exporter
-          configMap:
-            name: {{ include "datahub-gms.fullname" . }}-config-jmx-exporter
-        {{- end }}
       {{- with .Values.extraVolumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

charts/datahub/subcharts/datahub-frontend/templates/service.yaml

@@ -20,5 +20,11 @@ spec:
       nodePort: {{ . }}
       {{- end }}
       {{- end }}
+    {{- if .Values.global.datahub.monitoring.enablePrometheus }}
+    - name: jmx
+      port: 4318
+      targetPort: jmx
+      protocol: TCP
+    {{- end }}
   selector:
     {{- include "datahub-frontend.selectorLabels" . | nindent 4 }}

charts/datahub/subcharts/datahub-frontend/templates/servicemonitor.yaml

@@ -0,0 +1,25 @@
+{{- if and .Values.serviceMonitor.create .Values.global.datahub.monitoring.enablePrometheus -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "datahub-frontend" }}
+  labels:
+    {{- include "datahub-frontend.labels" . | nindent 4 }}
+  {{- with .Values.serviceMonitor.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+  - port: jmx
+    relabelings:
+    - separator: /
+      sourceLabels:
+      - namespace
+      - pod
+      targetLabel: instance
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: Helm
+      app.kubernetes.io/name: datahub-frontend
+{{- end -}}

charts/datahub/subcharts/datahub-frontend/values.yaml

@@ -49,6 +49,9 @@ service:
   annotations: {}
     # service.beta.kubernetes.io/aws-load-balancer-internal: "true"
 
+serviceMonitor:
+  create: false
+
 ingress:
   # className: ""
   enabled: false

charts/datahub/subcharts/datahub-gms/Chart.yaml

@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.9
+version: 0.2.10
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

charts/datahub/subcharts/datahub-gms/README.md

@@ -66,6 +66,7 @@ Current chart version is `0.2.0`
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `nil` |  |
+| serviceMonitoring.create | bool | `false` | If set true and `global.datahub.monitoring.enablePrometheus` is set `true` it will create a ServiceMonitor resource |
 | tolerations | list | `[]` |  |
 | datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. |
 | global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. |

charts/datahub/subcharts/datahub-gms/templates/service.yaml

@@ -20,5 +20,11 @@ spec:
       nodePort: {{ . }}
       {{- end }}
       {{- end }}
+    {{- if .Values.global.datahub.monitoring.enablePrometheus }}
+    - name: jmx
+      port: 4318
+      targetPort: jmx
+      protocol: TCP
+    {{- end }}
   selector:
     {{- include "datahub-gms.selectorLabels" . | nindent 4 }}

charts/datahub/subcharts/datahub-gms/templates/servicemonitor.yaml

@@ -0,0 +1,25 @@
+{{- if and .Values.serviceMonitor.create .Values.global.datahub.monitoring.enablePrometheus -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
+  labels:
+    {{- include "datahub-gms.labels" . | nindent 4 }}
+  {{- with .Values.serviceMonitor.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+  - port: jmx
+    relabelings:
+    - separator: /
+      sourceLabels:
+      - namespace
+      - pod
+      targetLabel: instance
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: Helm
+      app.kubernetes.io/name: datahub-gms
+{{- end -}}

charts/datahub/subcharts/datahub-gms/values.yaml

@@ -24,6 +24,10 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name:
 
+serviceMonitor:
+  create: false
+
+
 podAnnotations: {}
   # co.elastic.logs/enabled: "true"
 

charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml

@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.7
+version: 0.2.8
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

charts/datahub/subcharts/datahub-mae-consumer/README.md

@@ -60,6 +60,7 @@ Current chart version is `0.2.0`
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `nil` |  |
+| serviceMonitoring.create | bool | `false` | If set true and `global.datahub.monitoring.enablePrometheus` is set `true` it will create a ServiceMonitor resource |
 | tolerations | list | `[]` |  |
 | datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. |
 | global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. |

charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml

@@ -40,11 +40,6 @@ spec:
             defaultMode: 0444
             secretName: {{ .name }}
         {{- end }}
-        {{- if .Values.exporters.jmx.enabled }}
-        - name: config-jmx-exporter
-          configMap:
-            name: {{ include "datahub-mae-consumer.fullname" . }}-config-jmx-exporter
-        {{- end }}
       {{- with .Values.extraVolumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

charts/datahub/subcharts/datahub-mae-consumer/templates/service.yaml

@@ -16,5 +16,11 @@ spec:
       nodePort: {{ . }}
       {{- end }}
       {{- end }}
+    {{- if .Values.global.datahub.monitoring.enablePrometheus }}
+    - name: jmx
+      port: 4318
+      targetPort: jmx
+      protocol: TCP
+    {{- end }}
   selector:
     {{- include "datahub-mae-consumer.selectorLabels" . | nindent 4 }}

charts/datahub/subcharts/datahub-mae-consumer/templates/servicemonitor.yaml

@@ -0,0 +1,25 @@
+{{- if and .Values.serviceMonitor.create .Values.global.datahub.monitoring.enablePrometheus -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "datahub-mae-consumer" }}
+  labels:
+    {{- include "datahub-mae-consumer.labels" . | nindent 4 }}
+  {{- with .Values.serviceMonitor.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+  - port: jmx
+    relabelings:
+    - separator: /
+      sourceLabels:
+      - namespace
+      - pod
+      targetLabel: instance
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: Helm
+      app.kubernetes.io/name: datahub-mae-consumer
+{{- end -}}

charts/datahub/subcharts/datahub-mae-consumer/values.yaml

@@ -24,6 +24,9 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name:
 
+serviceMonitor:
+  create: false
+
 podAnnotations: {}
   # co.elastic.logs/enabled: "true"
 

charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml

@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.8
+version: 0.2.9
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

charts/datahub/subcharts/datahub-mce-consumer/README.md

@@ -50,6 +50,7 @@ Current chart version is `0.2.0`
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `nil` |  |
+| serviceMonitoring.create | bool | `false` | If set true and `global.datahub.monitoring.enablePrometheus` is set `true` it will create a ServiceMonitor resource |
 | tolerations | list | `[]` |  |
 | datahub.metadata_service_authentication.enabled | bool | `false` | Whether Metadata Service Authentication is enabled. |
 | global.datahub.metadata_service_authentication.systemClientId | string | `"__datahub_system"` | The internal system id that is used to communicate with DataHub GMS. Required if metadata_service_authentication is 'true'. |

charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml

@@ -40,11 +40,6 @@ spec:
             defaultMode: 0444
             secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
         {{- end }}
-        {{- if .Values.exporters.jmx.enabled }}
-        - name: config-jmx-exporter
-          configMap:
-            name: {{ include "datahub-mce-consumer.fullname" . }}-config-jmx-exporter
-        {{- end }}
       {{- with .Values.extraVolumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
@@ -59,6 +54,9 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           ports:
+            - name: actuator
+              containerPort: 9090
+              protocol: TCP
           {{- if or .Values.global.datahub.monitoring.enablePrometheus .Values.global.datahub.monitoring.enableJMXPort }}
             - name: jmx
               containerPort: 4318
@@ -67,14 +65,14 @@ spec:
           livenessProbe:
             httpGet:
               path: /actuator/health
-              port: 9090
+              port: actuator
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           readinessProbe:
             httpGet:
               path: /actuator/health
-              port: 9090
+              port: actuator
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}

charts/datahub/subcharts/datahub-mce-consumer/templates/service.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "datahub-mce-consumer" }}
+  labels:
+    {{- include "datahub-mce-consumer.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.service.targetPort }}
+      protocol: {{ .Values.service.protocol }}
+      name: {{ .Values.service.name }}
+      {{- if eq .Values.service.type "NodePort" }}
+      {{- with .Values.service.nodePort }}
+      nodePort: {{ . }}
+      {{- end }}
+      {{- end }}
+    {{- if .Values.global.datahub.monitoring.enablePrometheus }}
+    - name: jmx
+      port: 4318
+      targetPort: jmx
+      protocol: TCP
+    {{- end }}
+  selector:
+    {{- include "datahub-mce-consumer.selectorLabels" . | nindent 4 }}

charts/datahub/subcharts/datahub-mce-consumer/templates/servicemonitor.yaml

@@ -0,0 +1,25 @@
+{{- if and .Values.serviceMonitor.create .Values.global.datahub.monitoring.enablePrometheus -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "datahub-mce-consumer" }}
+  labels:
+    {{- include "datahub-mce-consumer.labels" . | nindent 4 }}
+  {{- with .Values.serviceMonitor.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+  - port: jmx
+    relabelings:
+    - separator: /
+      sourceLabels:
+      - namespace
+      - pod
+      targetLabel: instance
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: Helm
+      app.kubernetes.io/name: datahub-mce-consumer
+{{- end -}}

charts/datahub/subcharts/datahub-mce-consumer/values.yaml

@@ -24,6 +24,9 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name:
 
+serviceAccount:
+  create: false
+
 podAnnotations: {}
   # co.elastic.logs/enabled: "true"
 
@@ -39,8 +42,14 @@ securityContext: {}
   # runAsUser: 1000
 
 service:
-  type: ClusterIP
-  port: 80
+  type: ClusterIP # NodePort
+  port: "9090"
+  targetPort: actuator
+  protocol: TCP
+  name: actuator
+
+serviceMonitor:
+  create: false
 
 ingress:
   enabled: false