Merge pull request #112 from torstenwalter/add-fluent-bit-chart

Add fluent-bit chart

charts/fluent-bit/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/fluent-bit/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: "v1"
+name: fluent-bit
+version: 2.1.0
+appVersion: v2.0.0
+kubeVersion: "^1.10.0-0"
+description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki"
+home: https://grafana.com/loki
+icon: https://raw.githubusercontent.com/grafana/loki/master/docs/sources/logo.png
+sources:
+- https://github.com/grafana/loki
+maintainers:
+- name: Loki Maintainers
+  email: lokiproject@googlegroups.com
+engine: gotpl

charts/fluent-bit/README.md

@@ -0,0 +1,124 @@
+# Fluent Bit Loki chart
+
+This chart install the Fluent Bit application to ship logs to Loki. It defines daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
+
+## Get Repo Info
+
+```console
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo update
+```
+
+_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._
+
+## Installing the Chart
+
+> If you don't have `Helm` installed locally, or `Tiller` installed in your Kubernetes cluster, read the [Using Helm](https://docs.helm.sh/using_helm/) documentation to get started.
+To install the chart with the release name `my-release` using our helm repository:
+
+```bash
+helm repo add grafana https://grafana.github.io/helm-charts
+helm upgrade --install my-release grafana/fluent-bit \
+    --set loki.serviceName=loki.default.svc.cluster.local
+```
+
+If you deploy Loki with a custom namespace or service name, you must change the value above for `loki.serviceName` to the appropriate value.
+
+The command deploys Fluent Bit on the Kubernetes cluster with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
+
+To configure the chart to send to [Grafana Cloud](https://grafana.com/products/cloud) use:
+
+```bash
+helm upgrade --install my-release grafana/fluent-bit \
+    --set loki.serviceName=logs-us-west1.grafana.net,loki.servicePort=80,loki.serviceScheme=https \
+    --set loki.user=2830,loki.password=1234
+```
+
+> **Tip**: List all releases using `helm list`
+
+To install a custom tag use the following command:
+
+```bash
+helm upgrade --install my-release grafana/fluent-bit \
+    --set image.tag=<custom tag>
+```
+
+The full list of available tags on [docker hub](https://cloud.docker.com/u/grafana/repository/docker/grafana/fluent-bit-plugin-loki).
+
+Alternatively you can install the full [Loki stack](../loki-stack) (Loki + Fluent Bit) using:
+
+```bash
+helm upgrade --install my-release grafana/loki-stack \
+    --set fluent-bit.enabled=true,promtail.enabled=false
+```
+
+This will automatically configured the `loki.serviceName` configuration field to the newly created Loki instance.
+
+## RBAC
+
+By default, `rbac.create` is set to true. This enable RBAC support in Fluent Bit and must be true if RBAC is enabled in your cluster.
+
+The chart will take care of creating the required service accounts and roles for Fluent Bit.
+
+If you have RBAC disabled, or to put it another way, ABAC enabled, you should set this value to `false`.
+
+## Uninstalling the Chart
+
+To uninstall/delete the `my-release` deployment:
+
+```bash
+helm delete my-release
+```
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+## Configuration
+
+The following tables lists the configurable parameters of the Fluent Bit chart and their default values.
+
+For more details, read the [Fluent Bit documentation](../../../cmd/fluent-bit/README.md)
+
+| Parameter                | Description                                                                                        | Default                          |
+|--------------------------|----------------------------------------------------------------------------------------------------|----------------------------------|
+| `loki.serviceName`       | The address of the Loki service.                                                                   | `"${RELEASE}-loki"`              |
+| `loki.servicePort`       | The port of the Loki service.                                                                      | `3100`                           |
+| `loki.serviceScheme`     | The scheme of the Loki service.                                                                    | `http`                           |
+| `loki.user`              | The http basic auth username to access the Loki service.                                           |                                  |
+| `loki.password`          | The http basic auth password to access the Loki service.                                           |                                  |
+| `config.port`            | the Fluent Bit port to listen. (This is mainly used to serve metrics)                              | `2020`                           |
+| `config.tenantID`        | The tenantID used by default to push logs to Loki                                                  | `''`                             |
+| `config.batchWait`       | Time to wait before send a log batch to Loki, full or not. (unit: secs)                            | `1`                              |
+| `config.batchSize`       | Log batch size to send a log batch to Loki. (unit: bytes)                                          | `10240` (10KiB)                  |
+| `config.loglevel`        | the Fluent Bit log level (debug,info,warn,error).                                                  | `warn`                           |
+| `config.lineFormat`      | The line format to use to send a record (json/key_value)                                           | `json`                           |
+| `config.k8sLoggingParser`| Allow Kubernetes Pods to suggest a pre-defined Parser. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/kubernetes#kubernetes-annotations).                                                                                      | `Off`                           |
+| `config.k8sLoggingExclude`| Allow Kubernetes Pods to exclude their logs from the log processor. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/pipeline/filters/kubernetes)                                                                                             | `Off`
+| `config.memBufLimit`     | Override the default  Mem_Buf_Limit [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/administration/backpressure#mem_buf_limit) | `5MB`
+| `config.removeKeys`      | The list of key to remove from each record                                                         | `[removeKeys,stream]`            |
+| `config.labels`          | A set of labels to send for every log                                                              | `'{job="fluent-bit"}'`           |
+| `config.autoKubernetesLabels` | If set to true, it will add all Kubernetes labels to Loki labels                                   | `false`                          |
+| `config.labelMap`        | Mapping of labels from a record. See [Fluent Bit documentation](../../../cmd/fluent-bit/README.md) |                                  |
+| `config.parsers`         | Definition of extras fluent bit parsers. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/parser). The format is a sequence of mappings where each key is the same as the one in the [PARSER] section of parsers.conf file       | `[]`                            |
+| `config.extraOutputs`    | Definition of extras fluent bit outputs. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/pipeline/outputs/). The format is a sequence of mappings where each key is the same as the one in the [OUTPUT]                                | `[]`                            |
+| `affinity`               | [affinity][affinity] settings for pod assignment                                                   | `{}`                             |
+| `annotations`            | Annotations to add to Kubernetes resources.                                                        | `{}`                             |
+| `deploymentStrategy`     | The deployment strategy to use with the daemonset                                                  | `RollingUpdate`                  |
+| `image.repository`       | The Fluent Bit docker image repository                                                             | `grafana/fluent-bit-plugin-loki` |
+| `image.tag`              | The Fluent Bit docker image tag                                                                    | `0.1`                            |
+| `image.pullPolicy`       | The Fluent Bit docker image pull policy                                                            | `IfNotPresent`                   |
+| `nodeSelector`           | Fluent Bit [node labels][nodeSelector] for pod assignment                                          | `{}`                             |
+| `podLabels`              | additional Fluent Bit pod labels                                                                   | `{}`                             |
+| `podAnnotations`         | additional Fluent Bit pod annotations                                                              | `Prometheus discovery`           |
+| `rbac.create`            | Activate support for RBAC                                                                          | `true`                           |
+| `resources`              | Resource requests/limit                                                                            |                                  |
+| `tolerations`            | [Toleration][toleration] labels for pod assignment                                                 | `no schedule on master nodes`    |
+| `volumes`                | [Volume]([volumes]) to mount                                                                       | `host containers log`            |
+| `volumeMounts`           | Volume mount mapping                                                                               |                                  |
+| `serviceMonitor.enabled` | Create a [Prometheus Operator](operator) serviceMonitor resource for Fluent Bit                    | `false`                          |
+
+
+[toleration]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+[nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+[affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+[volumes]: https://kubernetes.io/docs/concepts/storage/volumes/
+[operator]: https://github.com/coreos/prometheus-operator

charts/fluent-bit/templates/NOTES.txt

@@ -0,0 +1,3 @@
+Verify the application is working by running these commands:
+  kubectl --namespace {{ .Release.Namespace }} port-forward daemonset/{{ include "fluent-bit-loki.fullname" . }} {{ .Values.config.port }}
+  curl http://127.0.0.1:{{ .Values.config.port }}/api/v1/metrics/prometheus

charts/fluent-bit/templates/_helpers.tpl

@@ -0,0 +1,66 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "fluent-bit-loki.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "fluent-bit-loki.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "fluent-bit-loki.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account
+*/}}
+{{- define "fluent-bit-loki.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "fluent-bit-loki.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+The service name to connect to Loki. Defaults to the same logic as "loki.fullname"
+*/}}
+{{- define "loki.serviceName" -}}
+{{- if .Values.loki.serviceName -}}
+{{- .Values.loki.serviceName -}}
+{{- else if .Values.loki.fullnameOverride -}}
+{{- .Values.loki.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default "loki" .Values.loki.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "helm-toolkit.utils.joinListWithComma" -}}
+{{- $local := dict "first" true -}}
+{{- range $k, $v := . -}}{{- if not $local.first -}},{{- end -}}{{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}}
+{{- end -}}

charts/fluent-bit/templates/clusterrole.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.rbac.create }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  labels:
+    app: {{ template "fluent-bit-loki.name" . }}
+    chart: {{ template "fluent-bit-loki.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "fluent-bit-loki.fullname" . }}-clusterrole
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources:
+  - namespaces
+  - pods
+  verbs: ["get", "watch", "list"]
+{{- end }}

charts/fluent-bit/templates/clusterrolebinding.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.rbac.create }}
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "fluent-bit-loki.fullname" . }}-clusterrolebinding
+  labels:
+    app: {{ template "fluent-bit-loki.name" . }}
+    chart: {{ template "fluent-bit-loki.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "fluent-bit-loki.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ template "fluent-bit-loki.fullname" . }}-clusterrole
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/fluent-bit/templates/configmap.yaml

@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "fluent-bit-loki.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "fluent-bit-loki.name" . }}
+    chart: {{ template "fluent-bit-loki.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  fluent-bit.conf: |-
+    [SERVICE]
+        HTTP_Server    On
+        HTTP_Listen    0.0.0.0
+        HTTP_PORT      {{ .Values.config.port }}
+        Flush          1
+        Daemon         Off
+        Log_Level      {{ .Values.config.loglevel }}
+        Parsers_File   parsers.conf
+    [INPUT]
+        Name           tail
+        Tag            kube.*
+        Path           /var/log/containers/*.log
+        Parser         docker
+        DB             /run/fluent-bit/flb_kube.db
+        Mem_Buf_Limit  {{ .Values.config.memBufLimit }}
+    [FILTER]
+        Name           kubernetes
+        Match          kube.*
+        Kube_URL       https://kubernetes.default.svc:443
+        Merge_Log On
+        K8S-Logging.Exclude {{ .Values.config.k8sLoggingExclude }}
+        K8S-Logging.Parser {{ .Values.config.k8sLoggingParser }}
+    [Output]
+        Name grafana-loki
+        Match *
+        {{- if and .Values.loki.user .Values.loki.password }}
+        Url {{ .Values.loki.serviceScheme }}://{{ .Values.loki.user }}:{{ .Values.loki.password }}@{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}{{ .Values.loki.servicePath }}
+          {{- else }}
+        Url {{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}{{ .Values.loki.servicePath }}
+        {{- end }}
+        TenantID {{ .Values.config.tenantID }}
+        BatchWait {{ .Values.config.batchWait }}
+        BatchSize {{ int .Values.config.batchSize }}
+        Labels {{ .Values.config.labels }}
+        RemoveKeys {{ include "helm-toolkit.utils.joinListWithComma" .Values.config.removeKeys }}
+        AutoKubernetesLabels {{ .Values.config.autoKubernetesLabels }}
+        LabelMapPath /fluent-bit/etc/labelmap.json
+        LineFormat {{ .Values.config.lineFormat }}
+        LogLevel {{ .Values.config.loglevel }}
+    {{- range $extraOutput := .Values.config.extraOutputs }}
+    [OUTPUT]
+    {{- range $key,$value := $extraOutput }}
+        {{ $key }} {{ $value }}
+    {{- end }}
+    {{- end }}
+  parsers.conf: |-
+    [PARSER]
+        Name        docker
+        Format      json
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
+    {{- range $parser:= .Values.config.parsers }}
+    [PARSER]
+    {{- range $key,$value := $parser }}
+        {{ $key }} {{ $value }}
+    {{- end }}
+    {{- end }}
+
+  labelmap.json: |-
+    {{- .Values.config.labelMap | toPrettyJson | nindent 4}}
+