Replace Prometheus logging in ingestion with StatsD logging (#293)

* Replace Prometheus logging in ingestion with StatsD logging * Added infrastructure for Prometheus statsd exporter
feast-dev · Nov 8, 2019 · 4e7578c · 4e7578c
1 parent 00b59f6
commit 4e7578c
Show file tree

Hide file tree

Showing 23 changed files with 583 additions and 103 deletions.
diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java
@@ -153,9 +153,9 @@ private ImportOptions getPipelineOptions(String jobName, List<FeatureSetSpec> fe
     pipelineOptions.setJobName(jobName);
     if (metrics.isEnabled()) {
       pipelineOptions.setMetricsExporterType(metrics.getType());
-      if (metrics.getType().equals("prometheus")) {
-        pipelineOptions.setPrometheusExporterAddress(
-            String.format("%s:%s", metrics.getHost(), metrics.getPort()));
+      if (metrics.getType().equals("statsd")) {
+        pipelineOptions.setStatsdHost(metrics.getHost());
+        pipelineOptions.setStatsdPort(metrics.getPort());
       }
     }
     return pipelineOptions;

diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java
@@ -37,7 +37,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.beam.runners.direct.DirectRunner;
 import org.apache.beam.sdk.PipelineResult;
@@ -103,9 +102,9 @@ private ImportOptions getPipelineOptions(List<FeatureSetSpec> featureSetSpecs,
     pipelineOptions.setProject(""); // set to default value to satisfy validation
     if (metrics.isEnabled()) {
       pipelineOptions.setMetricsExporterType(metrics.getType());
-      if (metrics.getType().equals("prometheus")) {
-        pipelineOptions.setPrometheusExporterAddress(
-            String.format("%s:%s", metrics.getHost(), metrics.getPort()));
+      if (metrics.getType().equals("statsd")) {
+        pipelineOptions.setStatsdHost(metrics.getHost());
+        pipelineOptions.setStatsdPort(metrics.getPort());
       }
     }
     pipelineOptions.setBlockOnRun(false);

diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/.helmignore b/infra/charts/feast/charts/prometheus-statsd-exporter/.helmignore
@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/Chart.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/Chart.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+appVersion: 0.8.0
+description: A Helm chart for prometheus statsd-exporter Scrape metrics stored statsd
+home: https://github.com/prometheus/statsd_exporter
+keywords:
+  - prometheus
+  - statsd
+maintainers:
+  - name: enflo
+    email: toniflorithomar@gmail.com
+name: prometheus-statsd-exporter
+version: 0.1.2
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/README.md b/infra/charts/feast/charts/prometheus-statsd-exporter/README.md
@@ -0,0 +1,56 @@
+# Prometheus statsd-exporter
+
+ ## TL;DR;
+
+ ```console
+$ helm install incubator/prometheus-statsd-exporter
+```
+
+ ## Introduction
+
+ This chart bootstraps a prometheus-statsd-exporter deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
+
+ ## Installing the Chart
+
+ To install the chart with the release name `my-release`:
+
+ ```console
+$ helm install incubator/prometheus-statsd-exporter --name my-release
+```
+
+
+ The command deploys prometheus-statsd-exporter on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
+
+ ## Uninstalling the Chart
+
+ To uninstall/delete the `my-release` deployment:
+
+ ```console
+$ helm delete my-release
+```
+
+ The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+ ## Configuration
+
+ |Parameter                   | Description                                          | Default                                |
+|`extraArgs`                 | key:value list of extra arguments to give the binary | `{}`                                   |
+|`image.pullPolicy`          | Image pull policy                                    | `IfNotPresent`                         |
+|`image.repository`          | Image repository                                     | `prom/statsd-exporter`                 |
+|`image.tag`                 | Image tag                                            | `v0.8.0`                               |
+|`ingress.enabled`           | enable ingress                                       | `false`                                |
+|`ingress.path`              | ingress base path                                    | `/`                                    |
+|`ingress.host`              | Ingress accepted hostnames                           | `nil`                                  |
+|`ingress.tls`               | Ingress TLS configuration                            | `[]`                                   |
+|`ingress.annotations`       | Ingress annotations                                  | `{}`                                   |
+|`service.type`              | type of service                                      | `ClusterIP`                            |
+|`tolerations`               | List of node taints to tolerate                      | `[]`                                   |
+|`resources`                 | pod resource requests & limits                       | `{}`                                   |   
+| `persistence.enabled`      | Create a volume to store data                        | true                                   |
+
+ Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example,
+
+ ```console
+$ helm install incubator/prometheus-statsd-exporter --name my-release -f values.yaml
+```
+> **Tip**: You can use the default [values.yaml](values.yaml)
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/NOTES.txt b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/NOTES.txt
@@ -0,0 +1,17 @@
+
+To verify that prometheus-statsd-exporter has started, run:
+
+{{- if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-statsd-exporter.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+        You can watch the status of by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "prometheus-statsd-exporter.fullname" . }}'
+
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "prometheus-statsd-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.servicePort }}
+{{- else if contains "ClusterIP"  .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "prometheus-statsd-exporter.name" . }},component={{ .Chart.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 9090
+{{- end }}
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/_helpers.tpl b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/_helpers.tpl
@@ -0,0 +1,44 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "prometheus-statsd-exporter.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "prometheus-statsd-exporter.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "prometheus-statsd-exporter.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "prometheus-statsd-exporter.serviceAccountName" -}}
+{{- if .Values.serviceAccount.enable -}}
+    {{ default (include "prometheus-statsd-expoter.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/config.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/config.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "prometheus-statsd-exporter.fullname" . }}-config
+  labels:
+    app: {{ template "prometheus-statsd-exporter.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  statsd_mappings.yaml: |
+#
+#     defaults:
+#       ttl: "45s"
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/deployment.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/deployment.yaml
@@ -0,0 +1,80 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ template "prometheus-statsd-exporter.fullname" . }}
+  labels:
+    app: {{ template "prometheus-statsd-exporter.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.statsdexporter.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "prometheus-statsd-exporter.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      annotations:
+{{ toYaml .Values.statsdexporter.annotations | indent 8 }}
+      labels:
+        app: {{ template "prometheus-statsd-exporter.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      serviceAccountName: {{ template "prometheus-statsd-exporter.serviceAccountName" . }}
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - --statsd.mapping-config=/etc/statsd_conf/statsd_mappings.yaml
+          {{- range $key, $value := .Values.statsdexporter.extraArgs }}
+            - --{{ $key }}={{ $value }}
+          {{- end }}
+          volumeMounts:
+          - mountPath: /data
+            name: {{ .Values.persistentVolume.name }}
+          - name: statsd-config
+            mountPath: /etc/statsd_conf
+          env:
+          - name: HOME
+            value: /data
+          ports:
+            - name: metrics
+              containerPort: 9102
+              protocol: TCP
+            - name: statsd-tcp
+              containerPort: 9125
+              protocol: TCP
+            - name: statsd-udp
+              containerPort: 9125
+              protocol: UDP
+          livenessProbe:
+            httpGet:
+              path: /#/status
+              port: 9102
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /#/status
+              port: 9102
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+          resources:
+{{ toYaml .Values.statsdexporter.resources | indent 12 }}
+{{- if .Values.statsdexporter.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.statsdexporter.nodeSelector | indent 8 }}
+    {{- end }}
+      volumes:
+      - name: statsd-config
+        configMap:
+          name: {{ template "prometheus-statsd-exporter.fullname" . }}-config
+      - name: {{ .Values.persistentVolume.name }}
+        {{- if .Values.persistentVolume.enabled }}
+        persistentVolumeClaim:
+        claimName: {{ if .Values.persistentVolume.claimName }}{{- else }}{{ template "prometheus-statsd-exporter.fullname" . }}{{- end }}
+        {{- else }}
+        emptyDir: {}
+        {{- end -}}
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/pvc.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/pvc.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    app: {{ template "prometheus-statsd-exporter.fullname" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Chart.Name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus-statsd-exporter.fullname" . }}
+spec:
+  accessModes:
+{{ toYaml .Values.persistentVolume.accessModes | indent 4 }}
+{{- if .Values.persistentVolume.storageClass }}
+{{- if (eq "-" .Values.persistentVolume.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.persistentVolume.storageClass }}"
+{{- end }}
+{{- end }}
+  resources:
+    requests:
+      storage: "{{ .Values.persistentVolume.size }}"
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/service.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/service.yaml
@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.service.annotations }}
+  annotations:
+{{ toYaml .Values.service.annotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus-statsd-exporter.fullname" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Chart.Name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- if .Values.service.labels }}
+{{ toYaml .Values.service.labels | indent 4 }}
+{{- end }}
+  name: {{ template "prometheus-statsd-exporter.fullname" . }}
+spec:
+{{- if .Values.service.clusterIP }}
+  clusterIP: {{ .Values.service.clusterIP }}
+{{- end }}
+{{- if .Values.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.service.externalIPs | indent 4 }}
+{{- end }}
+{{- if .Values.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.service.loadBalancerIP }}
+{{- end }}
+{{- if .Values.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+  {{- range $cidr := .Values.service.loadBalancerSourceRanges }}
+    - {{ $cidr }}
+  {{- end }}
+{{- end }}
+  ports:
+    - name: metrics
+      port: {{ .Values.service.metricsPort }}
+      protocol: TCP
+      targetPort: 9102
+    - name: statsd-tcp
+      port: {{ .Values.service.statsdPort }}
+      protocol: TCP
+      targetPort: 9125
+    - name: statsd-udp
+      port: {{ .Values.service.statsdPort }}
+      protocol: UDP
+      targetPort: 9125
+  selector:
+    app: {{ template "prometheus-statsd-exporter.name" . }}
+    release: {{ .Release.Name }}
+  type: "{{ .Values.service.type }}"
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml
@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.enable -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ template "prometheus-statsd-exporter.fullname" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.serviceaccount.componentName }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus-statsd-exporter.fullname" . }}
+{{- end -}}