From 511006e57b5250c9918f1250b10641485fd91c1a Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Sun, 15 Apr 2018 14:31:38 +0100 Subject: [PATCH 001/202] Inital commit. From cf03c4337db0bc5070ed09cd40a4d84abebd9830 Mon Sep 17 00:00:00 2001 From: Julien Garcia Gonzalez Date: Mon, 17 Dec 2018 21:17:24 +0100 Subject: [PATCH 002/202] Add helm chart (#107) * Add loki helm chart Signed-off-by: Julien Garcia Gonzalez * Add promtail helm chart Signed-off-by: Julien Garcia Gonzalez * Update loki conf from comment Signed-off-by: Julien Garcia Gonzalez * Update promtail conf from comment Signed-off-by: Julien Garcia Gonzalez * Update values from comment Signed-off-by: Julien Garcia Gonzalez * adding resources limit and request Signed-off-by: Julien Garcia Gonzalez * update README Signed-off-by: Julien Garcia Gonzalez * rename to daemonset promtail Signed-off-by: Julien Garcia Gonzalez * use masger for loki Signed-off-by: Julien Garcia Gonzalez * Move into production repo, re-add readme. Signed-off-by: Tom Wilkie * Move to production folder Signed-off-by: Julien Garcia Gonzalez * Review feedback; add instructions for installing Grafana with helm. Signed-off-by: Tom Wilkie --- charts/Chart.yaml | 13 ++ charts/README.md | 45 ++++++ charts/grafana.yaml | 5 + charts/templates/loki/_helpers.tpl | 43 ++++++ charts/templates/loki/configmap.yaml | 43 ++++++ charts/templates/loki/deployment.yaml | 79 +++++++++++ charts/templates/loki/podsecuritypolicy.yaml | 28 ++++ charts/templates/loki/role.yaml | 18 +++ charts/templates/loki/rolebinding.yaml | 18 +++ charts/templates/loki/service.yaml | 25 ++++ charts/templates/loki/serviceaccount.yaml | 11 ++ charts/templates/promtail/_helpers.tpl | 43 ++++++ charts/templates/promtail/clusterrole.yaml | 24 ++++ .../promtail/clusterrolebinding.yaml | 23 ++++ charts/templates/promtail/configmap.yaml | 78 +++++++++++ charts/templates/promtail/daemonset.yaml | 98 +++++++++++++ .../templates/promtail/podsecuritypolicy.yaml | 30 ++++ charts/templates/promtail/role.yaml | 18 +++ charts/templates/promtail/rolebinding.yaml | 18 +++ charts/templates/promtail/serviceaccount.yaml | 11 ++ charts/values.yaml | 130 ++++++++++++++++++ 21 files changed, 801 insertions(+) create mode 100644 charts/Chart.yaml create mode 100644 charts/README.md create mode 100644 charts/grafana.yaml create mode 100644 charts/templates/loki/_helpers.tpl create mode 100644 charts/templates/loki/configmap.yaml create mode 100644 charts/templates/loki/deployment.yaml create mode 100644 charts/templates/loki/podsecuritypolicy.yaml create mode 100644 charts/templates/loki/role.yaml create mode 100644 charts/templates/loki/rolebinding.yaml create mode 100644 charts/templates/loki/service.yaml create mode 100644 charts/templates/loki/serviceaccount.yaml create mode 100644 charts/templates/promtail/_helpers.tpl create mode 100644 charts/templates/promtail/clusterrole.yaml create mode 100644 charts/templates/promtail/clusterrolebinding.yaml create mode 100644 charts/templates/promtail/configmap.yaml create mode 100644 charts/templates/promtail/daemonset.yaml create mode 100644 charts/templates/promtail/podsecuritypolicy.yaml create mode 100644 charts/templates/promtail/role.yaml create mode 100644 charts/templates/promtail/rolebinding.yaml create mode 100644 charts/templates/promtail/serviceaccount.yaml create mode 100644 charts/values.yaml diff --git a/charts/Chart.yaml b/charts/Chart.yaml new file mode 100644 index 0000000000..3ec1773e63 --- /dev/null +++ b/charts/Chart.yaml @@ -0,0 +1,13 @@ +name: loki +version: 0.0.1 +appVersion: 0.0.1 +kubeVersion: "^1.10.0-0" +description: "Loki: like Prometheus, but for logs." +home: https://grafana.com/loki +icon: https://github.com/grafana/loki/raw/master/docs/logo.png +sources: + - https://github.com/grafana/loki +maintainers: + - name: Loki Maintainers + email: lokiproject@googlegroups.com +engine: gotpl \ No newline at end of file diff --git a/charts/README.md b/charts/README.md new file mode 100644 index 0000000000..2e204ef33f --- /dev/null +++ b/charts/README.md @@ -0,0 +1,45 @@ +# Deploy Loki to Kubernetes with Helm + +## Prerequisites + +Make sure you have the helm configure on your cluster: + +```bash +$ helm init +``` + +Clone `grafana/loki` repository and navigate to `production helm` directory: + +```bash +$ git clone https://github.com/grafana/loki.git +$ cd loki/production/helm +``` + +## Deploy Loki and Promtail to your cluster + +```bash +$ helm install . -n loki --namespace +``` + +## Deploy Grafana to your cluster + +To install Grafana on your cluster with helm, use the following command: + +```bash +$ helm install stable/grafana -n loki-grafana -f grafana.yaml --namespace +``` + +To get the admin password for the Grafana pod, run the following command: + +```bash +$ kubectl get secret --namespace loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo +``` + +To access the Grafana UI, run the following command: + +```bash +$ kubectl port-forward --namespace service/loki-grafana 3000:80 +``` + +Navigate to http://localhost:3000 and login with `admin` and the password output above. +Then follow the [instructions for adding the loki datasource](/docs/usage.md), using the URL `http://loki:3100/`. diff --git a/charts/grafana.yaml b/charts/grafana.yaml new file mode 100644 index 0000000000..0614cdfd39 --- /dev/null +++ b/charts/grafana.yaml @@ -0,0 +1,5 @@ +image: + tag: master + +env: + GF_EXPLORE_ENABLED: true \ No newline at end of file diff --git a/charts/templates/loki/_helpers.tpl b/charts/templates/loki/_helpers.tpl new file mode 100644 index 0000000000..2e333aae60 --- /dev/null +++ b/charts/templates/loki/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "loki.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "loki.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "loki.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "loki.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "loki.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/charts/templates/loki/configmap.yaml b/charts/templates/loki/configmap.yaml new file mode 100644 index 0000000000..12718ceed1 --- /dev/null +++ b/charts/templates/loki/configmap.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + loki.yaml: | + auth_enabled: {{ .Values.loki.config.auth_enabled }} + + server: + http_listen_port: {{ .Values.loki.service.port }} + + ingester: + lifecycler: + ring: + store: {{ .Values.loki.config.ingester.lifecycler.ring.store }} + replication_factor: {{ .Values.loki.config.ingester.lifecycler.ring.replication_factor }} + +{{- if .Values.loki.config.schema_configs }} + schema_config: + configs: + {{- range .Values.loki.config.schema_configs }} + - from: {{ .from }} + store: {{ .store }} + object_store: {{ .object_store }} + schema: {{ .schema }} + index: + prefix: {{ .index.prefix }} + period: {{ .index.period }} + {{- end -}} +{{- end -}} + +{{- if .Values.loki.config.storage_configs }} + storage_config: + {{- range .Values.loki.config.storage_configs }} + {{ .name }}: + directory: {{ .directory }} + {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml new file mode 100644 index 0000000000..b491d6ff21 --- /dev/null +++ b/charts/templates/loki/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.loki.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.loki.replicas }} + selector: + matchLabels: + app: {{ template "loki.name" . }} + release: {{ .Release.Name }} + strategy: + type: {{ .Values.loki.deploymentStrategy }} + {{- if ne .Values.loki.deploymentStrategy "RollingUpdate" }} + rollingUpdate: null + {{- end }} + template: + metadata: + labels: + app: {{ template "loki.name" . }} + release: {{ .Release.Name }} +{{- with .Values.loki.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ template "loki.serviceAccountName" . }} +{{- if .Values.loki.priorityClassName }} + priorityClassName: {{ .Values.loki.priorityClassName }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.loki.image.repository }}:{{ .Values.loki.image.tag }}" + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + args: + - "-config.file=/etc/loki/loki.yaml" + volumeMounts: + - name: config + mountPath: /etc/loki + ports: + - name: {{ template "loki.name" . }} + containerPort: {{ .Values.loki.service.port }} + protocol: TCP +{{- with .Values.loki.livenessProbe }} + livenessProbe: +{{ toYaml . | indent 12 }} +{{- end }} +{{- with .Values.loki.readinessProbe }} + readinessProbe: +{{ toYaml . | indent 12 }} +{{- end }} +{{- with .Values.loki.resources }} + resources: +{{ toYaml . | indent 12 }} +{{- end }} + {{- with .Values.loki.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.loki.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.loki.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "loki.fullname" . }} \ No newline at end of file diff --git a/charts/templates/loki/podsecuritypolicy.yaml b/charts/templates/loki/podsecuritypolicy.yaml new file mode 100644 index 0000000000..a7c8751916 --- /dev/null +++ b/charts/templates/loki/podsecuritypolicy.yaml @@ -0,0 +1,28 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + privileged: false + allowPrivilegeEscalation: false + volumes: + - 'configMap' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: false +{{- end }} \ No newline at end of file diff --git a/charts/templates/loki/role.yaml b/charts/templates/loki/role.yaml new file mode 100644 index 0000000000..cc09eb3de8 --- /dev/null +++ b/charts/templates/loki/role.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.rbac.pspEnabled }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "loki.fullname" . }}] +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/templates/loki/rolebinding.yaml b/charts/templates/loki/rolebinding.yaml new file mode 100644 index 0000000000..587ac9585a --- /dev/null +++ b/charts/templates/loki/rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "loki.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "loki.serviceAccountName" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/templates/loki/service.yaml b/charts/templates/loki/service.yaml new file mode 100644 index 0000000000..db03cf4a67 --- /dev/null +++ b/charts/templates/loki/service.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.loki.service.labels }} +{{ toYaml .Values.loki.service.labels | indent 4 }} +{{- end }} +{{- with .Values.loki.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + ports: + - name: {{ template "loki.name" . }} + port: {{ .Values.loki.service.port }} + protocol: TCP + targetPort: {{ .Values.loki.service.port }} + selector: + app: {{ template "loki.name" . }} + release: {{ .Release.Name }} \ No newline at end of file diff --git a/charts/templates/loki/serviceaccount.yaml b/charts/templates/loki/serviceaccount.yaml new file mode 100644 index 0000000000..5ad0dcf6d5 --- /dev/null +++ b/charts/templates/loki/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "loki.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "loki.serviceAccountName" . }} +{{- end }} \ No newline at end of file diff --git a/charts/templates/promtail/_helpers.tpl b/charts/templates/promtail/_helpers.tpl new file mode 100644 index 0000000000..a1a5bccefc --- /dev/null +++ b/charts/templates/promtail/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "promtail.name" -}} +{{- default .Chart.Name .Values.promtail.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "promtail.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.promtail.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "promtail.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "promtail.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "promtail.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/charts/templates/promtail/clusterrole.yaml b/charts/templates/promtail/clusterrole.yaml new file mode 100644 index 0000000000..84bff8d2e3 --- /dev/null +++ b/charts/templates/promtail/clusterrole.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + app: {{ template "promtail.name" . }} + chart: {{ template "promtail.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.promtail.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "promtail.fullname" . }}-clusterrole +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "watch", "list"] +{{- end}} \ No newline at end of file diff --git a/charts/templates/promtail/clusterrolebinding.yaml b/charts/templates/promtail/clusterrolebinding.yaml new file mode 100644 index 0000000000..f958a38915 --- /dev/null +++ b/charts/templates/promtail/clusterrolebinding.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rbac.create }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "promtail.fullname" . }}-clusterrolebinding + labels: + app: {{ template "promtail.name" . }} + chart: {{ template "promtail.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.promtail.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +subjects: + - kind: ServiceAccount + name: {{ template "promtail.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "promtail.fullname" . }}-clusterrole + apiGroup: rbac.authorization.k8s.io +{{- end}} \ No newline at end of file diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml new file mode 100644 index 0000000000..c7f9c2ce02 --- /dev/null +++ b/charts/templates/promtail/configmap.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "promtail.fullname" . }} + labels: + app: {{ template "promtail.name" . }} + chart: {{ template "promtail.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + promtail.yaml: | + scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __meta_kubernetes_pod_label_name + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_label_name + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - replacement: /var/log/pods/$1 + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-app + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __meta_kubernetes_pod_label_app + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_label_app + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1 + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ \ No newline at end of file diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml new file mode 100644 index 0000000000..8da09af44c --- /dev/null +++ b/charts/templates/promtail/daemonset.yaml @@ -0,0 +1,98 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: {{ template "promtail.fullname" . }} + labels: + app: {{ template "promtail.name" . }} + chart: {{ template "promtail.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.promtail.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.promtail.replicas }} + selector: + matchLabels: + app: {{ template "promtail.name" . }} + release: {{ .Release.Name }} + strategy: + type: {{ .Values.promtail.deploymentStrategy }} + {{- if ne .Values.promtail.deploymentStrategy "RollingUpdate" }} + rollingUpdate: null + {{- end }} + template: + metadata: + labels: + app: {{ template "promtail.name" . }} + release: {{ .Release.Name }} +{{- with .Values.promtail.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ template "promtail.serviceAccountName" . }} +{{- if .Values.promtail.priorityClassName }} + priorityClassName: {{ .Values.promtail.priorityClassName }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.promtail.image.repository }}:{{ .Values.promtail.image.tag }}" + imagePullPolicy: {{ .Values.promtail.image.pullPolicy }} + args: + - "-config.file=/etc/promtail/promtail.yaml" + - "-client.url=http://{{ template "loki.fullname" . }}:{{ .Values.loki.service.port }}/api/prom/push" + volumeMounts: + - name: config + mountPath: /etc/promtail + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 80 + name: http-metrics + securityContext: + privileged: true + runAsUser: 0 +{{- with .Values.promtail.livenessProbe }} + livenessProbe: +{{ toYaml . | indent 12 }} +{{- end }} +{{- with .Values.promtail.readinessProbe }} + readinessProbe: +{{ toYaml . | indent 12 }} +{{- end }} +{{- with .Values.promtail.resources }} + resources: +{{ toYaml . | indent 12 }} +{{- end }} + {{- with .Values.promtail.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.promtail.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.promtail.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "promtail.fullname" . }} + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers \ No newline at end of file diff --git a/charts/templates/promtail/podsecuritypolicy.yaml b/charts/templates/promtail/podsecuritypolicy.yaml new file mode 100644 index 0000000000..72e57f3524 --- /dev/null +++ b/charts/templates/promtail/podsecuritypolicy.yaml @@ -0,0 +1,30 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "promtail.fullname" . }} + labels: + app: {{ template "promtail.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + privileged: true + allowPrivilegeEscalation: true + volumes: + - 'secret' + - 'configMap' + - 'hostPath' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: false + {{- end }} \ No newline at end of file diff --git a/charts/templates/promtail/role.yaml b/charts/templates/promtail/role.yaml new file mode 100644 index 0000000000..4de03d9e28 --- /dev/null +++ b/charts/templates/promtail/role.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ template "promtail.fullname" . }} + labels: + app: {{ template "promtail.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.rbac.pspEnabled }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "promtail.fullname" . }}] +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/templates/promtail/rolebinding.yaml b/charts/templates/promtail/rolebinding.yaml new file mode 100644 index 0000000000..b23fb1374b --- /dev/null +++ b/charts/templates/promtail/rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ template "promtail.fullname" . }} + labels: + app: {{ template "promtail.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "promtail.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "promtail.serviceAccountName" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/templates/promtail/serviceaccount.yaml b/charts/templates/promtail/serviceaccount.yaml new file mode 100644 index 0000000000..cce8581e20 --- /dev/null +++ b/charts/templates/promtail/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "promtail.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "promtail.serviceAccountName" . }} +{{- end }} \ No newline at end of file diff --git a/charts/values.yaml b/charts/values.yaml new file mode 100644 index 0000000000..4628fbef0a --- /dev/null +++ b/charts/values.yaml @@ -0,0 +1,130 @@ +rbac: + create: true + pspEnabled: true + +serviceAccount: + create: true + name: + +loki: + replicas: 1 + deploymentStrategy: RollingUpdate + + image: + repository: grafana/loki + tag: master + pullPolicy: IfNotPresent + + service: + port: 3100 + annotations: {} + labels: {} + + readinessProbe: {} + + livenessProbe: {} + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + ## Pod Annotations + # podAnnotations: {} + + ## Deployment annotations + # annotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: + + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + + config: + auth_enabled: false + ingester: + lifecycler: + ring: + store: inmemory + replication_factor: 1 + schema_configs: + - from: 0 + store: boltdb + object_store: filesystem + schema: v9 + index: + prefix: index_ + period: 168h + storage_configs: + - name: boltdb + directory: /tmp/loki/index + - name: filesystem + directory: /tmp/loki/chunks + +promtail: + nameOverride: promtail + replicas: 1 + deploymentStrategy: RollingUpdate + + image: + repository: grafana/promtail + tag: master + pullPolicy: IfNotPresent + + service: + port: 3100 + annotations: {} + labels: {} + + readinessProbe: {} + + livenessProbe: {} + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + ## Pod Annotations + # podAnnotations: {} + + ## Deployment annotations + # annotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: + + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} \ No newline at end of file From d62325d9b286b08669f5e5632cc2a237e1b6f45f Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 11 Jan 2019 05:23:58 +0800 Subject: [PATCH 003/202] deploy promtail on master node as default (#184) Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/values.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/charts/values.yaml b/charts/values.yaml index 4628fbef0a..2a293ed0d5 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -122,9 +122,12 @@ promtail: ## Tolerations for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## - tolerations: [] + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + ## Affinity for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## - affinity: {} \ No newline at end of file + affinity: {} From 8384b20e4ae7d2d819998f12603f67aa0a6bbfa1 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 17 Jan 2019 19:31:12 +0800 Subject: [PATCH 004/202] helm: remove promtail sa setting (#160) Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/templates/promtail/_helpers.tpl | 2 +- charts/templates/promtail/serviceaccount.yaml | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 charts/templates/promtail/serviceaccount.yaml diff --git a/charts/templates/promtail/_helpers.tpl b/charts/templates/promtail/_helpers.tpl index a1a5bccefc..dcbe0ea260 100644 --- a/charts/templates/promtail/_helpers.tpl +++ b/charts/templates/promtail/_helpers.tpl @@ -36,7 +36,7 @@ Create the name of the service account */}} {{- define "promtail.serviceAccountName" -}} {{- if .Values.serviceAccount.create -}} - {{ default (include "promtail.fullname" .) .Values.serviceAccount.name }} + {{ default (include "loki.fullname" .) .Values.serviceAccount.name }} {{- else -}} {{ default "default" .Values.serviceAccount.name }} {{- end -}} diff --git a/charts/templates/promtail/serviceaccount.yaml b/charts/templates/promtail/serviceaccount.yaml deleted file mode 100644 index cce8581e20..0000000000 --- a/charts/templates/promtail/serviceaccount.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: {{ template "promtail.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - name: {{ template "promtail.serviceAccountName" . }} -{{- end }} \ No newline at end of file From dd65ccc7dd267fd4c369810c7696a46f42259f28 Mon Sep 17 00:00:00 2001 From: Anthony Woods Date: Thu, 17 Jan 2019 19:22:45 +0100 Subject: [PATCH 005/202] allow promtail path to be a glob match (#206) * allow promtail target path to be a glob match Promtail now expects __path__ to be a glob. All files that match the glob expression will be tailed. If new files are created that match the glob, they will immediately be tailed. * remove entry from positions file when file is deleted * fix syncing the current positions to positions file * handle file renames When a file is renamed a "rename" and a "create" event are sent. We need to treat the rename as a remove. If the new filename matches the path glob, then we will start tailing it when the "create" event is reveived. * lint --- charts/templates/promtail/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index c7f9c2ce02..e90695e935 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -36,7 +36,7 @@ data: source_labels: - __meta_kubernetes_pod_name target_label: instance - - replacement: /var/log/pods/$1 + - replacement: /var/log/pods/$1/0.log separator: / source_labels: - __meta_kubernetes_pod_uid @@ -70,7 +70,7 @@ data: target_label: instance - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1 + - replacement: /var/log/pods/$1/0.log separator: / source_labels: - __meta_kubernetes_pod_uid From b1a473002f392151a2e308ab096bc6f015e589d2 Mon Sep 17 00:00:00 2001 From: Anthony Woods Date: Sat, 19 Jan 2019 10:28:29 +0800 Subject: [PATCH 006/202] add container_name label to logs fixes #190 Pods can have multiple containers. This fix ensures that we collect logs for all containers. --- charts/templates/promtail/configmap.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index e90695e935..291320cb57 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -36,6 +36,12 @@ data: source_labels: - __meta_kubernetes_pod_name target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) - replacement: /var/log/pods/$1/0.log separator: / source_labels: @@ -68,6 +74,10 @@ data: source_labels: - __meta_kubernetes_pod_name target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_container_name + target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - replacement: /var/log/pods/$1/0.log From fda278e0ab5b8f727bf8d240435b0ecad4910dd9 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Tue, 29 Jan 2019 09:39:26 +0000 Subject: [PATCH 007/202] s/__meta_kubernetes_container_name/__meta_kubernetes_pod_container_name/ for the helm chart too. Signed-off-by: Tom Wilkie --- charts/templates/promtail/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index 291320cb57..39f7182a2e 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -38,7 +38,7 @@ data: target_label: instance - action: replace source_labels: - - __meta_kubernetes_container_name + - __meta_kubernetes_pod_container_name target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) @@ -76,7 +76,7 @@ data: target_label: instance - action: replace source_labels: - - __meta_kubernetes_container_name + - __meta_kubernetes_pod_container_name target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) From c7ad177e62574a69a95eaade04d0fae46f6fd63f Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 31 Jan 2019 19:40:24 +0100 Subject: [PATCH 008/202] Fix validation; allow 'metrics' without a metric name. (#268) * Update cortex to include optionally-enforce-metric-name Signed-off-by: Tom Wilkie * Configure validation to allow 'metrics' without metric names. Signed-off-by: Tom Wilkie --- charts/templates/loki/configmap.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/templates/loki/configmap.yaml b/charts/templates/loki/configmap.yaml index 12718ceed1..f7eba41fd4 100644 --- a/charts/templates/loki/configmap.yaml +++ b/charts/templates/loki/configmap.yaml @@ -14,6 +14,9 @@ data: server: http_listen_port: {{ .Values.loki.service.port }} + limits_config: + enforce_metric_name: false + ingester: lifecycler: ring: From 3c0d0ed4e813450e083e8baee09a5e60791e38ab Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Fri, 1 Feb 2019 14:27:10 +0100 Subject: [PATCH 009/202] Flush idle chunks properly (#209) * Flush idle chunks properly Signed-off-by: Goutham Veeramachaneni * Review feedback Signed-off-by: Goutham Veeramachaneni * Review feedback. Signed-off-by: Tom Wilkie * Check the correct chunk is full before appending a new chunk. Signed-off-by: Tom Wilkie * Add the config to helm and ksonnet too. Signed-off-by: Tom Wilkie --- charts/templates/loki/configmap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/templates/loki/configmap.yaml b/charts/templates/loki/configmap.yaml index f7eba41fd4..2eff8df813 100644 --- a/charts/templates/loki/configmap.yaml +++ b/charts/templates/loki/configmap.yaml @@ -22,6 +22,7 @@ data: ring: store: {{ .Values.loki.config.ingester.lifecycler.ring.store }} replication_factor: {{ .Values.loki.config.ingester.lifecycler.ring.replication_factor }} + chunk_idle_period: 15m {{- if .Values.loki.config.schema_configs }} schema_config: From b4fb6e96cf892d890f805e80979cdac1ad242260 Mon Sep 17 00:00:00 2001 From: Mike Splain Date: Sun, 3 Feb 2019 05:02:22 -0500 Subject: [PATCH 010/202] Add pvc and move data to data (#156) --- charts/templates/loki/deployment.yaml | 12 +++++++++- charts/templates/loki/podsecuritypolicy.yaml | 2 ++ charts/templates/loki/pvc.yaml | 24 ++++++++++++++++++++ charts/values.yaml | 16 +++++++++++-- 4 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 charts/templates/loki/pvc.yaml diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index b491d6ff21..d8dab03637 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -45,6 +45,9 @@ spec: volumeMounts: - name: config mountPath: /etc/loki + - name: storage + mountPath: "/data" + subPath: {{ .Values.loki.persistence.subPath }} ports: - name: {{ template "loki.name" . }} containerPort: {{ .Values.loki.service.port }} @@ -76,4 +79,11 @@ spec: volumes: - name: config configMap: - name: {{ template "loki.fullname" . }} \ No newline at end of file + name: {{ template "loki.fullname" . }} + - name: storage + {{- if .Values.loki.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.loki.persistence.existingClaim | default (include "loki.fullname" .) }} + {{- else }} + emptyDir: {} + {{- end -}} diff --git a/charts/templates/loki/podsecuritypolicy.yaml b/charts/templates/loki/podsecuritypolicy.yaml index a7c8751916..ea36a45e5b 100644 --- a/charts/templates/loki/podsecuritypolicy.yaml +++ b/charts/templates/loki/podsecuritypolicy.yaml @@ -13,6 +13,8 @@ spec: allowPrivilegeEscalation: false volumes: - 'configMap' + - 'emptyDir' + - 'persistentVolumeClaim' hostNetwork: false hostIPC: false hostPID: false diff --git a/charts/templates/loki/pvc.yaml b/charts/templates/loki/pvc.yaml new file mode 100644 index 0000000000..b31e91bb59 --- /dev/null +++ b/charts/templates/loki/pvc.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.loki.persistence.enabled (not .Values.loki.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + {{- with .Values.loki.persistence.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + accessModes: + {{- range .Values.loki.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.loki.persistence.size | quote }} + storageClassName: {{ .Values.loki.persistence.storageClassName }} +{{- end -}} diff --git a/charts/values.yaml b/charts/values.yaml index 2a293ed0d5..4f182706d8 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -56,6 +56,18 @@ loki: ## affinity: {} + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + enabled: true + accessModes: + - ReadWriteOnce + size: 10Gi + # storageClassName: default + # annotations: {} + # subPath: "" + # existingClaim: config: auth_enabled: false @@ -74,9 +86,9 @@ loki: period: 168h storage_configs: - name: boltdb - directory: /tmp/loki/index + directory: /data/loki/index - name: filesystem - directory: /tmp/loki/chunks + directory: /data/loki/chunks promtail: nameOverride: promtail From 34c3b3179b2c8eb894ba562335e9e64f1290d2d5 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Mon, 4 Feb 2019 14:27:51 +0800 Subject: [PATCH 011/202] tail all container log files kubelet creates a new log file when a container is restarted. So we need to catpure all log files in the `/var/logs/pods///` dir. --- charts/templates/promtail/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index 39f7182a2e..f0ecb9b49e 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -42,7 +42,7 @@ data: target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/0.log + - replacement: /var/log/pods/$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid @@ -80,7 +80,7 @@ data: target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/0.log + - replacement: /var/log/pods/$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid From 0a080558bc93e423959ac210130d62b003d59ad8 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Tue, 12 Feb 2019 01:39:35 +0800 Subject: [PATCH 012/202] dont add pods twice if they have app and name labels issue #313 --- charts/templates/promtail/configmap.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index f0ecb9b49e..9f09bbb5b8 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -59,6 +59,10 @@ data: regex: ^$ source_labels: - __meta_kubernetes_pod_label_app + - action: drop + regex: .+ + source_labels: + - __meta_kubernetes_pod_label_name - action: replace replacement: $1 separator: / From 79a583c02cf55d718df76b7fc29c8e1127403a95 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 12 Feb 2019 07:31:27 +0800 Subject: [PATCH 013/202] Fix helm install error (#305) * update helm values yaml - set persistence.enabled as 'false' as default since there is no pv as default - set persistence.storageClassName as 'default' since even if pv exists, pvc still need storageClassName: ``` Normal FailedBinding 8s (x4 over 35s) persistentvolume-controller no persistent volumes available for this claim and no storage class is set ``` Signed-off-by: Xiang Dai <764524258@qq.com> * helm: delete duplicate empty blank Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/README.md | 2 +- charts/templates/loki/deployment.yaml | 2 +- charts/templates/promtail/clusterrole.yaml | 2 +- charts/templates/promtail/daemonset.yaml | 2 +- charts/values.yaml | 14 ++++++++------ 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/charts/README.md b/charts/README.md index 2e204ef33f..36b5590621 100644 --- a/charts/README.md +++ b/charts/README.md @@ -26,7 +26,7 @@ $ helm install . -n loki --namespace To install Grafana on your cluster with helm, use the following command: ```bash -$ helm install stable/grafana -n loki-grafana -f grafana.yaml --namespace +$ helm install stable/grafana -n loki-grafana -f grafana.yaml --namespace ``` To get the admin password for the Grafana pod, run the following command: diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index d8dab03637..c8dc8bf1a4 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -40,7 +40,7 @@ spec: - name: {{ .Chart.Name }} image: "{{ .Values.loki.image.repository }}:{{ .Values.loki.image.tag }}" imagePullPolicy: {{ .Values.loki.image.pullPolicy }} - args: + args: - "-config.file=/etc/loki/loki.yaml" volumeMounts: - name: config diff --git a/charts/templates/promtail/clusterrole.yaml b/charts/templates/promtail/clusterrole.yaml index 84bff8d2e3..8a1f207029 100644 --- a/charts/templates/promtail/clusterrole.yaml +++ b/charts/templates/promtail/clusterrole.yaml @@ -19,6 +19,6 @@ rules: - nodes/proxy - services - endpoints - - pods + - pods verbs: ["get", "watch", "list"] {{- end}} \ No newline at end of file diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index 8da09af44c..8e897b7016 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -40,7 +40,7 @@ spec: - name: {{ .Chart.Name }} image: "{{ .Values.promtail.image.repository }}:{{ .Values.promtail.image.tag }}" imagePullPolicy: {{ .Values.promtail.image.pullPolicy }} - args: + args: - "-config.file=/etc/promtail/promtail.yaml" - "-client.url=http://{{ template "loki.fullname" . }}:{{ .Values.loki.service.port }}/api/prom/push" volumeMounts: diff --git a/charts/values.yaml b/charts/values.yaml index 4f182706d8..ae55dfec2d 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -4,9 +4,9 @@ rbac: serviceAccount: create: true - name: + name: -loki: +loki: replicas: 1 deploymentStrategy: RollingUpdate @@ -58,13 +58,15 @@ loki: ## Enable persistence using Persistent Volume Claims ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ - ## + ## If you set enabled as "True", you need : + ## - create a pv which above 10Gi and has same namespace with loki + ## - keep storageClassName same with below setting persistence: - enabled: true + enabled: false accessModes: - ReadWriteOnce size: 10Gi - # storageClassName: default + storageClassName: default # annotations: {} # subPath: "" # existingClaim: @@ -97,7 +99,7 @@ promtail: image: repository: grafana/promtail - tag: master + tag: master pullPolicy: IfNotPresent service: From 03e13a1040399b5e7a2754adcb2f110b7003a511 Mon Sep 17 00:00:00 2001 From: erkan Date: Wed, 13 Feb 2019 00:49:05 +0100 Subject: [PATCH 014/202] fixed promtail ds, it had deployment options --- charts/templates/promtail/daemonset.yaml | 5 ++--- charts/values.yaml | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index 8e897b7016..24370380a0 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -12,12 +12,11 @@ metadata: {{ toYaml . | indent 4 }} {{- end }} spec: - replicas: {{ .Values.promtail.replicas }} selector: matchLabels: app: {{ template "promtail.name" . }} release: {{ .Release.Name }} - strategy: + updateStrategy: type: {{ .Values.promtail.deploymentStrategy }} {{- if ne .Values.promtail.deploymentStrategy "RollingUpdate" }} rollingUpdate: null @@ -95,4 +94,4 @@ spec: path: /var/log - name: varlibdockercontainers hostPath: - path: /var/lib/docker/containers \ No newline at end of file + path: /var/lib/docker/containers diff --git a/charts/values.yaml b/charts/values.yaml index ae55dfec2d..0754974e9e 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -94,7 +94,6 @@ loki: promtail: nameOverride: promtail - replicas: 1 deploymentStrategy: RollingUpdate image: From 53f8385342e5f20c05f4e677a8e0358c3d34bf19 Mon Sep 17 00:00:00 2001 From: Ha Son Hai Date: Fri, 15 Feb 2019 15:54:17 +0100 Subject: [PATCH 015/202] Loki pod needs secret mounting privileges --- charts/templates/loki/podsecuritypolicy.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/templates/loki/podsecuritypolicy.yaml b/charts/templates/loki/podsecuritypolicy.yaml index ea36a45e5b..a5007bdbba 100644 --- a/charts/templates/loki/podsecuritypolicy.yaml +++ b/charts/templates/loki/podsecuritypolicy.yaml @@ -15,6 +15,7 @@ spec: - 'configMap' - 'emptyDir' - 'persistentVolumeClaim' + - 'secret' hostNetwork: false hostIPC: false hostPID: false @@ -27,4 +28,4 @@ spec: fsGroup: rule: 'RunAsAny' readOnlyRootFilesystem: false -{{- end }} \ No newline at end of file +{{- end }} From 264013a9e8eb53391d34b975ceaacda72ba10ff9 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Fri, 15 Feb 2019 16:58:55 +0100 Subject: [PATCH 016/202] Change image pull default policy to Always - Loki and promtail are under heavy development, always pulling makes sure bug fixes are shipped --- charts/values.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/charts/values.yaml b/charts/values.yaml index 0754974e9e..dee4dc5e3b 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -13,7 +13,7 @@ loki: image: repository: grafana/loki tag: master - pullPolicy: IfNotPresent + pullPolicy: Always # Always pull while in BETA service: port: 3100 @@ -24,7 +24,8 @@ loki: livenessProbe: {} - resources: {} + resources: + {} # limits: # cpu: 100m # memory: 128Mi @@ -99,7 +100,7 @@ promtail: image: repository: grafana/promtail tag: master - pullPolicy: IfNotPresent + pullPolicy: Always # Always pull while in BETA service: port: 3100 @@ -110,7 +111,8 @@ promtail: livenessProbe: {} - resources: {} + resources: + {} # limits: # cpu: 100m # memory: 128Mi @@ -139,7 +141,6 @@ promtail: - key: node-role.kubernetes.io/master effect: NoSchedule - ## Affinity for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## From 028f53179d6a168486c0907f3892d1a6a3e4eaeb Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Sun, 24 Feb 2019 20:03:11 -0600 Subject: [PATCH 017/202] Add minReadySeconds --- charts/templates/loki/deployment.yaml | 3 +++ charts/values.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index c8dc8bf1a4..860f46e9f9 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -13,6 +13,9 @@ metadata: {{- end }} spec: replicas: {{ .Values.loki.replicas }} + {{- if .Values.loki.minReadySeconds}} + minReadySeconds: {{ .Values.loki.minReadySeconds }} + {{- end }} selector: matchLabels: app: {{ template "loki.name" . }} diff --git a/charts/values.yaml b/charts/values.yaml index dee4dc5e3b..884a98dccb 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -8,6 +8,7 @@ serviceAccount: loki: replicas: 1 + minReadySeconds: 60 deploymentStrategy: RollingUpdate image: From 5ef9576daaec9cb7fc6bf0a18f5b40ff4ae477fc Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Sun, 24 Feb 2019 20:15:46 -0600 Subject: [PATCH 018/202] Add support for setting terminationGracePeriodSeconds. Add example readiness proble for loki and example antiaffinity. --- charts/templates/loki/deployment.yaml | 3 +++ charts/values.yaml | 24 ++++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index 860f46e9f9..7644e2fe0e 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -78,6 +78,9 @@ spec: {{- with .Values.loki.tolerations }} tolerations: {{ toYaml . | indent 8 }} + {{- end }} + {{- if .Values.loki.terminationGracePeriodSeconds}} + terminationGracePeriodSeconds: {{ .Values.loki.terminationGracePeriodSeconds }} {{- end }} volumes: - name: config diff --git a/charts/values.yaml b/charts/values.yaml index 884a98dccb..bf78fd0b7a 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -8,7 +8,8 @@ serviceAccount: loki: replicas: 1 - minReadySeconds: 60 + # minReadySeconds: 60 + # terminationGracePeriodSeconds: 3600 deploymentStrategy: RollingUpdate image: @@ -22,11 +23,14 @@ loki: labels: {} readinessProbe: {} + # httpGet: + # path: /ready + # port: 80 + # initialDelaySeconds: 15 livenessProbe: {} - resources: - {} + resources: {} # limits: # cpu: 100m # memory: 128Mi @@ -57,7 +61,16 @@ loki: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## affinity: {} - + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - loki + # topologyKey: "kubernetes.io/hostname" + ## Enable persistence using Persistent Volume Claims ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ ## If you set enabled as "True", you need : @@ -112,8 +125,7 @@ promtail: livenessProbe: {} - resources: - {} + resources: {} # limits: # cpu: 100m # memory: 128Mi From 7506b4a7aa292a93ef5706ba2e62125ebf09b49d Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Sun, 24 Feb 2019 20:29:39 -0600 Subject: [PATCH 019/202] fix port in readinessProbe example --- charts/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/values.yaml b/charts/values.yaml index bf78fd0b7a..8e71cc095b 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -25,7 +25,7 @@ loki: readinessProbe: {} # httpGet: # path: /ready - # port: 80 + # port: 3100 # initialDelaySeconds: 15 livenessProbe: {} From a91c3881fc034b7168eee0d317f224777e1327d7 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Mon, 25 Feb 2019 08:28:46 -0800 Subject: [PATCH 020/202] Add -metrics to port name for loki so our Prometheus will scrape it. Signed-off-by: Tom Wilkie --- charts/templates/loki/deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index c8dc8bf1a4..1da9554d89 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -26,6 +26,7 @@ spec: metadata: labels: app: {{ template "loki.name" . }} + name: {{ template "loki.name" . }} release: {{ .Release.Name }} {{- with .Values.loki.podAnnotations }} annotations: @@ -49,7 +50,7 @@ spec: mountPath: "/data" subPath: {{ .Values.loki.persistence.subPath }} ports: - - name: {{ template "loki.name" . }} + - name: {{ template "loki.name" . }}-metrics containerPort: {{ .Values.loki.service.port }} protocol: TCP {{- with .Values.loki.livenessProbe }} From 50952083f066ef44e5bfec4fd8ada0a2c0c16032 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Tue, 5 Mar 2019 12:19:27 -0600 Subject: [PATCH 021/202] Fix fullnameOverride in Helm chart Signed-off-by: Steven Sheehy --- charts/templates/loki/_helpers.tpl | 6 +++--- charts/templates/promtail/_helpers.tpl | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/templates/loki/_helpers.tpl b/charts/templates/loki/_helpers.tpl index 2e333aae60..71fca849b7 100644 --- a/charts/templates/loki/_helpers.tpl +++ b/charts/templates/loki/_helpers.tpl @@ -12,10 +12,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this If release name contains chart name it will be used as a full name. */}} {{- define "loki.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- if .Values.loki.fullnameOverride -}} +{{- .Values.loki.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- $name := default .Chart.Name .Values.loki.nameOverride -}} {{- if contains $name .Release.Name -}} {{- .Release.Name | trunc 63 | trimSuffix "-" -}} {{- else -}} diff --git a/charts/templates/promtail/_helpers.tpl b/charts/templates/promtail/_helpers.tpl index dcbe0ea260..92f1d483ba 100644 --- a/charts/templates/promtail/_helpers.tpl +++ b/charts/templates/promtail/_helpers.tpl @@ -12,8 +12,8 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this If release name contains chart name it will be used as a full name. */}} {{- define "promtail.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- if .Values.promtail.fullnameOverride -}} +{{- .Values.promtail.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} {{- $name := default .Chart.Name .Values.promtail.nameOverride -}} {{- if contains $name .Release.Name -}} From 51cd60aa7c39b3b6dee277dde4d6a407b406fef3 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Tue, 5 Mar 2019 12:28:30 -0600 Subject: [PATCH 022/202] Add entryParser option to helm chart Signed-off-by: Steven Sheehy --- charts/Chart.yaml | 4 ++-- charts/templates/promtail/configmap.yaml | 4 +++- charts/values.yaml | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 3ec1773e63..071080c8bf 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.0.1 +version: 0.1.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." @@ -10,4 +10,4 @@ sources: maintainers: - name: Loki Maintainers email: lokiproject@googlegroups.com -engine: gotpl \ No newline at end of file +engine: gotpl diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index 9f09bbb5b8..c1b996ec07 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -11,6 +11,7 @@ data: promtail.yaml: | scrape_configs: - job_name: kubernetes-pods + entry_parser: {{ .Values.promtail.entryParser }} kubernetes_sd_configs: - role: pod relabel_configs: @@ -49,6 +50,7 @@ data: - __meta_kubernetes_pod_container_name target_label: __path__ - job_name: kubernetes-pods-app + entry_parser: {{ .Values.promtail.entryParser }} kubernetes_sd_configs: - role: pod relabel_configs: @@ -89,4 +91,4 @@ data: source_labels: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name - target_label: __path__ \ No newline at end of file + target_label: __path__ diff --git a/charts/values.yaml b/charts/values.yaml index dee4dc5e3b..943e182849 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -97,6 +97,8 @@ promtail: nameOverride: promtail deploymentStrategy: RollingUpdate + entryParser: docker + image: repository: grafana/promtail tag: master From df8c2148e8873ec303dfd404a66ef20396ee0de7 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Thu, 7 Mar 2019 17:28:38 +0000 Subject: [PATCH 023/202] Made the docker root configurable. Default is '/var/lib/docker' --- charts/templates/promtail/daemonset.yaml | 4 ++-- charts/values.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index 24370380a0..56c4bce210 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -48,7 +48,7 @@ spec: - name: varlog mountPath: /var/log - name: varlibdockercontainers - mountPath: /var/lib/docker/containers + mountPath: {{ .Values.promtail.dockerRoot }}/containers readOnly: true env: - name: HOSTNAME @@ -94,4 +94,4 @@ spec: path: /var/log - name: varlibdockercontainers hostPath: - path: /var/lib/docker/containers + path: {{ .Values.promtail.dockerRoot }}/containers diff --git a/charts/values.yaml b/charts/values.yaml index 943e182849..5def64d8a9 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -98,6 +98,7 @@ promtail: deploymentStrategy: RollingUpdate entryParser: docker + dockerRoot: /var/lib/docker image: repository: grafana/promtail From e97bbd56cba0f7d221958e7a0add9957840c5fa2 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Thu, 7 Mar 2019 19:32:26 +0000 Subject: [PATCH 024/202] Changed 'dockerRoot' to a more generic name --- charts/templates/promtail/daemonset.yaml | 4 ++-- charts/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index 56c4bce210..b3cbd6ce00 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -48,7 +48,7 @@ spec: - name: varlog mountPath: /var/log - name: varlibdockercontainers - mountPath: {{ .Values.promtail.dockerRoot }}/containers + mountPath: {{ .Values.promtail.containerRootPath }}/containers readOnly: true env: - name: HOSTNAME @@ -94,4 +94,4 @@ spec: path: /var/log - name: varlibdockercontainers hostPath: - path: {{ .Values.promtail.dockerRoot }}/containers + path: {{ .Values.promtail.containerRootPath }}/containers diff --git a/charts/values.yaml b/charts/values.yaml index 5def64d8a9..c1364a9b8a 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -98,7 +98,7 @@ promtail: deploymentStrategy: RollingUpdate entryParser: docker - dockerRoot: /var/lib/docker + containerRootPath: /var/lib/docker image: repository: grafana/promtail From ac99957bad9eef41149bafa53ec13639b1d0af9a Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Fri, 8 Mar 2019 10:28:34 -0600 Subject: [PATCH 025/202] Set defaults for minReadySeconds and terminationGracePeriodSeconds and removed if statements. --- charts/templates/loki/deployment.yaml | 4 ---- charts/values.yaml | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index 5974362b06..3c3c73d0ca 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -13,9 +13,7 @@ metadata: {{- end }} spec: replicas: {{ .Values.loki.replicas }} - {{- if .Values.loki.minReadySeconds}} minReadySeconds: {{ .Values.loki.minReadySeconds }} - {{- end }} selector: matchLabels: app: {{ template "loki.name" . }} @@ -80,9 +78,7 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} - {{- if .Values.loki.terminationGracePeriodSeconds}} terminationGracePeriodSeconds: {{ .Values.loki.terminationGracePeriodSeconds }} - {{- end }} volumes: - name: config configMap: diff --git a/charts/values.yaml b/charts/values.yaml index 88176cd036..08a60578b3 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -8,8 +8,8 @@ serviceAccount: loki: replicas: 1 - # minReadySeconds: 60 - # terminationGracePeriodSeconds: 3600 + minReadySeconds: 0 + terminationGracePeriodSeconds: 30 deploymentStrategy: RollingUpdate image: From b6005ae7f9313f9318f4e2e2f52e2aeb82784fc2 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Fri, 8 Mar 2019 16:40:48 +0000 Subject: [PATCH 026/202] More generic handling of volumes and volumeMounts in Promtail helm template --- charts/templates/promtail/daemonset.yaml | 21 +++++++++++---------- charts/values.yaml | 14 +++++++++++++- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index b3cbd6ce00..9063102019 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -45,11 +45,13 @@ spec: volumeMounts: - name: config mountPath: /etc/promtail - - name: varlog - mountPath: /var/log - - name: varlibdockercontainers - mountPath: {{ .Values.promtail.containerRootPath }}/containers - readOnly: true + {{- range .Values.promtail.volumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- if .readOnly }} + readOnly: {{ .readOnly}} + {{- end }} + {{- end }} env: - name: HOSTNAME valueFrom: @@ -89,9 +91,8 @@ spec: - name: config configMap: name: {{ template "promtail.fullname" . }} - - name: varlog + {{- range .Values.promtail.volumes }} + - name: {{ .name }} hostPath: - path: /var/log - - name: varlibdockercontainers - hostPath: - path: {{ .Values.promtail.containerRootPath }}/containers + path: {{ .path }} + {{- end }} diff --git a/charts/values.yaml b/charts/values.yaml index c1364a9b8a..cb067d597a 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -98,13 +98,25 @@ promtail: deploymentStrategy: RollingUpdate entryParser: docker - containerRootPath: /var/lib/docker image: repository: grafana/promtail tag: master pullPolicy: Always # Always pull while in BETA + volumes: + - name: varlog + path: /var/log + - name: varlibdockercontainers + path: /var/lib/docker/containers + + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + service: port: 3100 annotations: {} From 84c673e18e074c7d9e37f2e9aa39839edbb2c118 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Fri, 8 Mar 2019 17:04:11 +0000 Subject: [PATCH 027/202] Updated with cleaner toYaml Helm logic --- charts/templates/promtail/daemonset.yaml | 16 +++++----------- charts/values.yaml | 6 ++++-- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index 9063102019..1b23e132b1 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -45,12 +45,8 @@ spec: volumeMounts: - name: config mountPath: /etc/promtail - {{- range .Values.promtail.volumeMounts }} - - name: {{ .name }} - mountPath: {{ .mountPath }} - {{- if .readOnly }} - readOnly: {{ .readOnly}} - {{- end }} + {{- if .Values.promtail.volumeMounts -}} + {{ toYaml .Values.promtail.volumeMounts | nindent 12 }} {{- end }} env: - name: HOSTNAME @@ -91,8 +87,6 @@ spec: - name: config configMap: name: {{ template "promtail.fullname" . }} - {{- range .Values.promtail.volumes }} - - name: {{ .name }} - hostPath: - path: {{ .path }} - {{- end }} + {{- if .Values.promtail.volumes -}} + {{ toYaml .Values.promtail.volumes | nindent 8 }} + {{- end }} diff --git a/charts/values.yaml b/charts/values.yaml index cb067d597a..57f7617238 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -106,9 +106,11 @@ promtail: volumes: - name: varlog - path: /var/log + hostPath: + path: /var/log - name: varlibdockercontainers - path: /var/lib/docker/containers + hostPath: + path: /var/lib/docker/containers volumeMounts: - name: varlog From 8a4834c0c467bb0b88ae815ef05485f21a8e3fa9 Mon Sep 17 00:00:00 2001 From: Belmin Fernandez Date: Sat, 9 Mar 2019 15:16:18 -0500 Subject: [PATCH 028/202] Adds netpol to allow promtail to reach loki --- charts/Chart.yaml | 2 +- charts/templates/loki/networkpolicy.yaml | 25 ++++++++++++++++++++++++ charts/values.yaml | 3 +++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 charts/templates/loki/networkpolicy.yaml diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 071080c8bf..1b9def1e23 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.1.0 +version: 0.2.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/templates/loki/networkpolicy.yaml b/charts/templates/loki/networkpolicy.yaml new file mode 100644 index 0000000000..67281e71f9 --- /dev/null +++ b/charts/templates/loki/networkpolicy.yaml @@ -0,0 +1,25 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + podSelector: + matchLabels: + name: {{ template "loki.fullname" . }} + app: {{ template "loki.name" . }} + release: {{ .Release.Name }} + ingress: + - from: + - podSelector: + matchLabels: + app: {{ template "promtail.name" . }} + release: {{ .Release.Name }} + - ports: + - port: {{ .Values.loki.service.port }} +{{- end -}} diff --git a/charts/values.yaml b/charts/values.yaml index 943e182849..60bc37f481 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -2,6 +2,9 @@ rbac: create: true pspEnabled: true +networkPolicy: + enabled: false + serviceAccount: create: true name: From 8bc67001dfb0c69617ca2c43f402a72bbb0f069a Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Sat, 16 Mar 2019 10:28:23 +0800 Subject: [PATCH 029/202] Support only deploy loki or promtail and fix helm lint issue Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/templates/loki/configmap.yaml | 10 ++++++---- charts/templates/loki/deployment.yaml | 4 +++- charts/templates/loki/podsecuritypolicy.yaml | 2 ++ charts/templates/loki/pvc.yaml | 4 +++- charts/templates/loki/role.yaml | 4 +++- charts/templates/loki/rolebinding.yaml | 6 ++++-- charts/templates/loki/service.yaml | 4 +++- charts/templates/loki/serviceaccount.yaml | 4 +++- charts/templates/promtail/clusterrole.yaml | 4 +++- charts/templates/promtail/clusterrolebinding.yaml | 4 +++- charts/templates/promtail/configmap.yaml | 2 ++ charts/templates/promtail/daemonset.yaml | 2 ++ charts/templates/promtail/podsecuritypolicy.yaml | 4 +++- charts/templates/promtail/role.yaml | 4 +++- charts/templates/promtail/rolebinding.yaml | 6 ++++-- charts/values.yaml | 4 ++++ 16 files changed, 51 insertions(+), 17 deletions(-) diff --git a/charts/templates/loki/configmap.yaml b/charts/templates/loki/configmap.yaml index 2eff8df813..5b93c3f6de 100644 --- a/charts/templates/loki/configmap.yaml +++ b/charts/templates/loki/configmap.yaml @@ -1,3 +1,4 @@ +{{- if .Values.loki.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -35,13 +36,14 @@ data: index: prefix: {{ .index.prefix }} period: {{ .index.period }} - {{- end -}} -{{- end -}} + {{- end }} +{{- end }} {{- if .Values.loki.config.storage_configs }} storage_config: {{- range .Values.loki.config.storage_configs }} {{ .name }}: directory: {{ .directory }} - {{- end -}} -{{- end -}} \ No newline at end of file + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index 1da9554d89..0f50bd47b7 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.loki.enabled }} apiVersion: apps/v1beta2 kind: Deployment metadata: @@ -87,4 +88,5 @@ spec: claimName: {{ .Values.loki.persistence.existingClaim | default (include "loki.fullname" .) }} {{- else }} emptyDir: {} - {{- end -}} + {{- end }} +{{- end }} diff --git a/charts/templates/loki/podsecuritypolicy.yaml b/charts/templates/loki/podsecuritypolicy.yaml index a5007bdbba..9c2175ba45 100644 --- a/charts/templates/loki/podsecuritypolicy.yaml +++ b/charts/templates/loki/podsecuritypolicy.yaml @@ -1,3 +1,4 @@ +{{- if .Values.loki.enabled }} {{- if .Values.rbac.pspEnabled }} apiVersion: extensions/v1beta1 kind: PodSecurityPolicy @@ -29,3 +30,4 @@ spec: rule: 'RunAsAny' readOnlyRootFilesystem: false {{- end }} +{{- end }} diff --git a/charts/templates/loki/pvc.yaml b/charts/templates/loki/pvc.yaml index b31e91bb59..9bba88becb 100644 --- a/charts/templates/loki/pvc.yaml +++ b/charts/templates/loki/pvc.yaml @@ -1,3 +1,4 @@ +{{- if .Values.loki.enabled }} {{- if and .Values.loki.persistence.enabled (not .Values.loki.persistence.existingClaim) }} apiVersion: v1 kind: PersistentVolumeClaim @@ -21,4 +22,5 @@ spec: requests: storage: {{ .Values.loki.persistence.size | quote }} storageClassName: {{ .Values.loki.persistence.storageClassName }} -{{- end -}} +{{- end }} +{{- end }} diff --git a/charts/templates/loki/role.yaml b/charts/templates/loki/role.yaml index cc09eb3de8..68951cf814 100644 --- a/charts/templates/loki/role.yaml +++ b/charts/templates/loki/role.yaml @@ -1,3 +1,4 @@ +{{- if .Values.loki.enabled }} {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role @@ -15,4 +16,5 @@ rules: verbs: ['use'] resourceNames: [{{ template "loki.fullname" . }}] {{- end }} -{{- end }} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/charts/templates/loki/rolebinding.yaml b/charts/templates/loki/rolebinding.yaml index 587ac9585a..afe1af010d 100644 --- a/charts/templates/loki/rolebinding.yaml +++ b/charts/templates/loki/rolebinding.yaml @@ -1,4 +1,5 @@ -{{- if .Values.rbac.create -}} +{{- if .Values.loki.enabled }} +{{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: @@ -15,4 +16,5 @@ roleRef: subjects: - kind: ServiceAccount name: {{ template "loki.serviceAccountName" . }} -{{- end -}} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/charts/templates/loki/service.yaml b/charts/templates/loki/service.yaml index db03cf4a67..0ad7289231 100644 --- a/charts/templates/loki/service.yaml +++ b/charts/templates/loki/service.yaml @@ -1,3 +1,4 @@ +{{- if .Values.loki.enabled }} apiVersion: v1 kind: Service metadata: @@ -22,4 +23,5 @@ spec: targetPort: {{ .Values.loki.service.port }} selector: app: {{ template "loki.name" . }} - release: {{ .Release.Name }} \ No newline at end of file + release: {{ .Release.Name }} +{{- end }} diff --git a/charts/templates/loki/serviceaccount.yaml b/charts/templates/loki/serviceaccount.yaml index 5ad0dcf6d5..302cafdb45 100644 --- a/charts/templates/loki/serviceaccount.yaml +++ b/charts/templates/loki/serviceaccount.yaml @@ -1,3 +1,4 @@ +{{- if .Values.loki.enabled }} {{- if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount @@ -8,4 +9,5 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} name: {{ template "loki.serviceAccountName" . }} -{{- end }} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/charts/templates/promtail/clusterrole.yaml b/charts/templates/promtail/clusterrole.yaml index 8a1f207029..75be62d03d 100644 --- a/charts/templates/promtail/clusterrole.yaml +++ b/charts/templates/promtail/clusterrole.yaml @@ -1,3 +1,4 @@ +{{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -21,4 +22,5 @@ rules: - endpoints - pods verbs: ["get", "watch", "list"] -{{- end}} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/charts/templates/promtail/clusterrolebinding.yaml b/charts/templates/promtail/clusterrolebinding.yaml index f958a38915..7a842922c8 100644 --- a/charts/templates/promtail/clusterrolebinding.yaml +++ b/charts/templates/promtail/clusterrolebinding.yaml @@ -1,3 +1,4 @@ +{{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -20,4 +21,5 @@ roleRef: kind: ClusterRole name: {{ template "promtail.fullname" . }}-clusterrole apiGroup: rbac.authorization.k8s.io -{{- end}} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index c1b996ec07..e6c88b5e1f 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -1,3 +1,4 @@ +{{- if .Values.promtail.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -92,3 +93,4 @@ data: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ +{{- end }} diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index 24370380a0..9012dc13e4 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -1,3 +1,4 @@ +{{- if .Values.promtail.enabled }} apiVersion: extensions/v1beta1 kind: DaemonSet metadata: @@ -95,3 +96,4 @@ spec: - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers +{{- end }} diff --git a/charts/templates/promtail/podsecuritypolicy.yaml b/charts/templates/promtail/podsecuritypolicy.yaml index 72e57f3524..547f1a544e 100644 --- a/charts/templates/promtail/podsecuritypolicy.yaml +++ b/charts/templates/promtail/podsecuritypolicy.yaml @@ -1,3 +1,4 @@ +{{- if .Values.promtail.enabled }} {{- if .Values.rbac.pspEnabled }} apiVersion: extensions/v1beta1 kind: PodSecurityPolicy @@ -27,4 +28,5 @@ spec: fsGroup: rule: 'RunAsAny' readOnlyRootFilesystem: false - {{- end }} \ No newline at end of file + {{- end }} +{{- end }} diff --git a/charts/templates/promtail/role.yaml b/charts/templates/promtail/role.yaml index 4de03d9e28..d1a97cbaf6 100644 --- a/charts/templates/promtail/role.yaml +++ b/charts/templates/promtail/role.yaml @@ -1,3 +1,4 @@ +{{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role @@ -15,4 +16,5 @@ rules: verbs: ['use'] resourceNames: [{{ template "promtail.fullname" . }}] {{- end }} -{{- end }} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/charts/templates/promtail/rolebinding.yaml b/charts/templates/promtail/rolebinding.yaml index b23fb1374b..aa3521b2cb 100644 --- a/charts/templates/promtail/rolebinding.yaml +++ b/charts/templates/promtail/rolebinding.yaml @@ -1,4 +1,5 @@ -{{- if .Values.rbac.create -}} +{{- if .Values.promtail.enabled }} +{{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: @@ -15,4 +16,5 @@ roleRef: subjects: - kind: ServiceAccount name: {{ template "promtail.serviceAccountName" . }} -{{- end -}} \ No newline at end of file +{{- end }} +{{- end }} diff --git a/charts/values.yaml b/charts/values.yaml index 943e182849..94e194539b 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -7,6 +7,8 @@ serviceAccount: name: loki: + enabled: true + replicas: 1 deploymentStrategy: RollingUpdate @@ -94,6 +96,8 @@ loki: directory: /data/loki/chunks promtail: + enabled: true + nameOverride: promtail deploymentStrategy: RollingUpdate From 9a947c071b350c87d928abe2b3c1eaccce95ce81 Mon Sep 17 00:00:00 2001 From: granular-ryanbonham <41303664+granular-ryanbonham@users.noreply.github.com> Date: Tue, 19 Mar 2019 10:23:48 -0500 Subject: [PATCH 030/202] Support AWS Storage w/ HELM template (#337) * Migrate config from configmap to secret. Add support for AWS Storage Config. * Add checksome for secret as pod annoation to ensure pod rotates if we update secret. * Move aws_storage_configs to storage_configs. Move local storage options under a local_storage key. * Allow specification of storage_config directly as yaml in values.yaml * Fix syntax for _helpers.tpl --- charts/templates/loki/_helpers.tpl | 37 ++++++++++++++++++++ charts/templates/loki/configmap.yaml | 49 --------------------------- charts/templates/loki/deployment.yaml | 7 ++-- charts/templates/loki/secret.yaml | 11 ++++++ charts/values.yaml | 6 ++-- 5 files changed, 55 insertions(+), 55 deletions(-) delete mode 100644 charts/templates/loki/configmap.yaml create mode 100644 charts/templates/loki/secret.yaml diff --git a/charts/templates/loki/_helpers.tpl b/charts/templates/loki/_helpers.tpl index 71fca849b7..d57dffe417 100644 --- a/charts/templates/loki/_helpers.tpl +++ b/charts/templates/loki/_helpers.tpl @@ -41,3 +41,40 @@ Create the name of the service account {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} + +{{- define "configSecret" }} +auth_enabled: {{ .Values.loki.config.auth_enabled }} + +server: + http_listen_port: {{ .Values.loki.service.port }} + +limits_config: + enforce_metric_name: false + +ingester: + lifecycler: + ring: + store: {{ .Values.loki.config.ingester.lifecycler.ring.store }} + replication_factor: {{ .Values.loki.config.ingester.lifecycler.ring.replication_factor }} + chunk_idle_period: 15m + +{{- if .Values.loki.config.schema_configs }} +schema_config: + configs: +{{- range .Values.loki.config.schema_configs }} + - from: {{ .from }} + store: {{ .store }} + object_store: {{ .object_store }} + schema: {{ .schema }} + index: + prefix: {{ .index.prefix }} + period: {{ .index.period }} +{{- end -}} +{{- end -}} + +{{- with .Values.loki.config.storage_config }} +storage_config: +{{ toYaml . | indent 2 }} +{{- end }} + +{{- end}} \ No newline at end of file diff --git a/charts/templates/loki/configmap.yaml b/charts/templates/loki/configmap.yaml deleted file mode 100644 index 5b93c3f6de..0000000000 --- a/charts/templates/loki/configmap.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if .Values.loki.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "loki.fullname" . }} - labels: - app: {{ template "loki.name" . }} - chart: {{ template "loki.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - loki.yaml: | - auth_enabled: {{ .Values.loki.config.auth_enabled }} - - server: - http_listen_port: {{ .Values.loki.service.port }} - - limits_config: - enforce_metric_name: false - - ingester: - lifecycler: - ring: - store: {{ .Values.loki.config.ingester.lifecycler.ring.store }} - replication_factor: {{ .Values.loki.config.ingester.lifecycler.ring.replication_factor }} - chunk_idle_period: 15m - -{{- if .Values.loki.config.schema_configs }} - schema_config: - configs: - {{- range .Values.loki.config.schema_configs }} - - from: {{ .from }} - store: {{ .store }} - object_store: {{ .object_store }} - schema: {{ .schema }} - index: - prefix: {{ .index.prefix }} - period: {{ .index.period }} - {{- end }} -{{- end }} - -{{- if .Values.loki.config.storage_configs }} - storage_config: - {{- range .Values.loki.config.storage_configs }} - {{ .name }}: - directory: {{ .directory }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index 05edf1369b..404301c2e1 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -30,8 +30,9 @@ spec: app: {{ template "loki.name" . }} name: {{ template "loki.name" . }} release: {{ .Release.Name }} -{{- with .Values.loki.podAnnotations }} annotations: + checksum/config: {{ include (print $.Template.BasePath "/loki/secret.yaml") . | sha256sum }} +{{- with .Values.loki.podAnnotations }} {{ toYaml . | indent 8 }} {{- end }} spec: @@ -82,8 +83,8 @@ spec: terminationGracePeriodSeconds: {{ .Values.loki.terminationGracePeriodSeconds }} volumes: - name: config - configMap: - name: {{ template "loki.fullname" . }} + secret: + secretName: {{ template "loki.fullname" . }} - name: storage {{- if .Values.loki.persistence.enabled }} persistentVolumeClaim: diff --git a/charts/templates/loki/secret.yaml b/charts/templates/loki/secret.yaml new file mode 100644 index 0000000000..66819067e5 --- /dev/null +++ b/charts/templates/loki/secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + loki.yaml: {{ include "configSecret" . | b64enc}} \ No newline at end of file diff --git a/charts/values.yaml b/charts/values.yaml index e669d76266..c47889d8ff 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -106,10 +106,10 @@ loki: index: prefix: index_ period: 168h - storage_configs: - - name: boltdb + storage_config: + boltdb: directory: /data/loki/index - - name: filesystem + filesystem: directory: /data/loki/chunks promtail: From fe32f17fe8e38f3ee97af0cb1274684bed06b65f Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Wed, 20 Mar 2019 20:22:17 +0800 Subject: [PATCH 031/202] Tag with lastest (#236) * Tag with latest Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/grafana.yaml | 2 +- charts/values.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/grafana.yaml b/charts/grafana.yaml index 0614cdfd39..a80753d353 100644 --- a/charts/grafana.yaml +++ b/charts/grafana.yaml @@ -1,5 +1,5 @@ image: - tag: master + tag: latest env: GF_EXPLORE_ENABLED: true \ No newline at end of file diff --git a/charts/values.yaml b/charts/values.yaml index c47889d8ff..f03f050891 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -19,7 +19,7 @@ loki: image: repository: grafana/loki - tag: master + tag: latest pullPolicy: Always # Always pull while in BETA service: @@ -122,7 +122,7 @@ promtail: image: repository: grafana/promtail - tag: master + tag: latest pullPolicy: Always # Always pull while in BETA volumes: From 3fd4f9d26537fb9e64b82713a9b25c92db3b2aff Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Tue, 19 Mar 2019 11:42:10 -0500 Subject: [PATCH 032/202] Add readiness probe to Helm chart and cleanup Signed-off-by: Steven Sheehy --- charts/Chart.yaml | 2 +- charts/templates/loki/_helpers.tpl | 4 +- charts/templates/loki/deployment.yaml | 45 ++++++--------- charts/templates/loki/podsecuritypolicy.yaml | 2 +- charts/templates/loki/role.yaml | 2 +- charts/templates/loki/rolebinding.yaml | 2 +- charts/templates/loki/service.yaml | 14 ++--- charts/templates/promtail/daemonset.yaml | 53 +++++++----------- .../templates/promtail/podsecuritypolicy.yaml | 2 +- charts/templates/promtail/role.yaml | 2 +- charts/templates/promtail/rolebinding.yaml | 2 +- charts/values.yaml | 55 ++++++++++--------- 12 files changed, 80 insertions(+), 105 deletions(-) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 1b9def1e23..40b88cca03 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.2.0 +version: 0.3.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/templates/loki/_helpers.tpl b/charts/templates/loki/_helpers.tpl index d57dffe417..c04c2950f9 100644 --- a/charts/templates/loki/_helpers.tpl +++ b/charts/templates/loki/_helpers.tpl @@ -46,7 +46,7 @@ Create the name of the service account auth_enabled: {{ .Values.loki.config.auth_enabled }} server: - http_listen_port: {{ .Values.loki.service.port }} + http_listen_port: {{ .Values.loki.port }} limits_config: enforce_metric_name: false @@ -77,4 +77,4 @@ storage_config: {{ toYaml . | indent 2 }} {{- end }} -{{- end}} \ No newline at end of file +{{- end}} diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index 404301c2e1..93eff4d02c 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -1,5 +1,5 @@ {{- if .Values.loki.enabled }} -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ template "loki.fullname" . }} @@ -8,10 +8,8 @@ metadata: chart: {{ template "loki.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} -{{- with .Values.loki.annotations }} annotations: -{{ toYaml . | indent 4 }} -{{- end }} + {{- toYaml .Values.loki.annotations | nindent 4 }} spec: replicas: {{ .Values.loki.replicas }} minReadySeconds: {{ .Values.loki.minReadySeconds }} @@ -32,14 +30,14 @@ spec: release: {{ .Release.Name }} annotations: checksum/config: {{ include (print $.Template.BasePath "/loki/secret.yaml") . | sha256sum }} -{{- with .Values.loki.podAnnotations }} -{{ toYaml . | indent 8 }} -{{- end }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ template "loki.serviceAccountName" . }} -{{- if .Values.loki.priorityClassName }} + {{- if .Values.loki.priorityClassName }} priorityClassName: {{ .Values.loki.priorityClassName }} -{{- end }} + {{- end }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.loki.image.repository }}:{{ .Values.loki.image.tag }}" @@ -53,33 +51,21 @@ spec: mountPath: "/data" subPath: {{ .Values.loki.persistence.subPath }} ports: - - name: {{ template "loki.name" . }}-metrics - containerPort: {{ .Values.loki.service.port }} + - name: http-metrics + containerPort: {{ .Values.loki.port }} protocol: TCP -{{- with .Values.loki.livenessProbe }} livenessProbe: -{{ toYaml . | indent 12 }} -{{- end }} -{{- with .Values.loki.readinessProbe }} + {{- toYaml .Values.loki.livenessProbe | nindent 12 }} readinessProbe: -{{ toYaml . | indent 12 }} -{{- end }} -{{- with .Values.loki.resources }} + {{- toYaml .Values.loki.readinessProbe | nindent 12 }} resources: -{{ toYaml . | indent 12 }} -{{- end }} - {{- with .Values.loki.nodeSelector }} + {{- toYaml .Values.loki.resources | nindent 12 }} nodeSelector: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.loki.affinity }} + {{- toYaml .Values.loki.nodeSelector | nindent 8 }} affinity: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.loki.tolerations }} + {{- toYaml .Values.loki.affinity | nindent 8 }} tolerations: -{{ toYaml . | indent 8 }} - {{- end }} + {{- toYaml .Values.loki.tolerations | nindent 8 }} terminationGracePeriodSeconds: {{ .Values.loki.terminationGracePeriodSeconds }} volumes: - name: config @@ -93,3 +79,4 @@ spec: emptyDir: {} {{- end }} {{- end }} + diff --git a/charts/templates/loki/podsecuritypolicy.yaml b/charts/templates/loki/podsecuritypolicy.yaml index 9c2175ba45..c81894d613 100644 --- a/charts/templates/loki/podsecuritypolicy.yaml +++ b/charts/templates/loki/podsecuritypolicy.yaml @@ -1,6 +1,6 @@ {{- if .Values.loki.enabled }} {{- if .Values.rbac.pspEnabled }} -apiVersion: extensions/v1beta1 +apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: {{ template "loki.fullname" . }} diff --git a/charts/templates/loki/role.yaml b/charts/templates/loki/role.yaml index 68951cf814..eb9c508f25 100644 --- a/charts/templates/loki/role.yaml +++ b/charts/templates/loki/role.yaml @@ -1,6 +1,6 @@ {{- if .Values.loki.enabled }} {{- if .Values.rbac.create }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ template "loki.fullname" . }} diff --git a/charts/templates/loki/rolebinding.yaml b/charts/templates/loki/rolebinding.yaml index afe1af010d..4aa765bb4a 100644 --- a/charts/templates/loki/rolebinding.yaml +++ b/charts/templates/loki/rolebinding.yaml @@ -1,6 +1,6 @@ {{- if .Values.loki.enabled }} {{- if .Values.rbac.create }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ template "loki.fullname" . }} diff --git a/charts/templates/loki/service.yaml b/charts/templates/loki/service.yaml index 0ad7289231..830c18d662 100644 --- a/charts/templates/loki/service.yaml +++ b/charts/templates/loki/service.yaml @@ -8,19 +8,17 @@ metadata: chart: {{ template "loki.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} -{{- if .Values.loki.service.labels }} -{{ toYaml .Values.loki.service.labels | indent 4 }} -{{- end }} -{{- with .Values.loki.service.annotations }} + {{- with .Values.loki.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} annotations: -{{ toYaml . | indent 4 }} -{{- end }} + {{- toYaml .Values.loki.service.annotations | nindent 4 }} spec: ports: - - name: {{ template "loki.name" . }} + - name: http-metrics port: {{ .Values.loki.service.port }} protocol: TCP - targetPort: {{ .Values.loki.service.port }} + targetPort: http-metrics selector: app: {{ template "loki.name" . }} release: {{ .Release.Name }} diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index d8fa9065f3..1f02fd5e2c 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -1,5 +1,5 @@ {{- if .Values.promtail.enabled }} -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: name: {{ template "promtail.fullname" . }} @@ -8,10 +8,8 @@ metadata: chart: {{ template "promtail.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} -{{- with .Values.promtail.annotations }} annotations: -{{ toYaml . | indent 4 }} -{{- end }} + {{- toYaml .Values.promtail.annotations | nindent 4 }} spec: selector: matchLabels: @@ -27,17 +25,15 @@ spec: labels: app: {{ template "promtail.name" . }} release: {{ .Release.Name }} -{{- with .Values.promtail.podAnnotations }} annotations: -{{ toYaml . | indent 8 }} -{{- end }} + {{ toYaml .Values.promtail.podAnnotations | nindent 8 }} spec: serviceAccountName: {{ template "promtail.serviceAccountName" . }} -{{- if .Values.promtail.priorityClassName }} + {{- if .Values.promtail.priorityClassName }} priorityClassName: {{ .Values.promtail.priorityClassName }} -{{- end }} + {{- end }} containers: - - name: {{ .Chart.Name }} + - name: promtail image: "{{ .Values.promtail.image.repository }}:{{ .Values.promtail.image.tag }}" imagePullPolicy: {{ .Values.promtail.image.pullPolicy }} args: @@ -46,9 +42,9 @@ spec: volumeMounts: - name: config mountPath: /etc/promtail - {{- if .Values.promtail.volumeMounts -}} - {{ toYaml .Values.promtail.volumeMounts | nindent 12 }} - {{- end }} + {{- with .Values.promtail.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} env: - name: HOSTNAME valueFrom: @@ -60,35 +56,24 @@ spec: securityContext: privileged: true runAsUser: 0 -{{- with .Values.promtail.livenessProbe }} livenessProbe: -{{ toYaml . | indent 12 }} -{{- end }} -{{- with .Values.promtail.readinessProbe }} + {{- toYaml .Values.promtail.livenessProbe | nindent 12 }} readinessProbe: -{{ toYaml . | indent 12 }} -{{- end }} -{{- with .Values.promtail.resources }} + {{- toYaml .Values.promtail.readinessProbe | nindent 12 }} resources: -{{ toYaml . | indent 12 }} -{{- end }} - {{- with .Values.promtail.nodeSelector }} + {{- toYaml .Values.promtail.resources | nindent 12 }} nodeSelector: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.promtail.affinity }} + {{- toYaml .Values.promtail.nodeSelector | nindent 8 }} affinity: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.promtail.tolerations }} + {{- toYaml .Values.promtail.affinity | nindent 8 }} tolerations: -{{ toYaml . | indent 8 }} - {{- end }} + {{- toYaml .Values.promtail.tolerations | nindent 8 }} volumes: - name: config configMap: name: {{ template "promtail.fullname" . }} - {{- if .Values.promtail.volumes -}} - {{ toYaml .Values.promtail.volumes | nindent 8 }} + {{- with .Values.promtail.volumes }} + {{- toYaml . | nindent 8 }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} + diff --git a/charts/templates/promtail/podsecuritypolicy.yaml b/charts/templates/promtail/podsecuritypolicy.yaml index 547f1a544e..2666f6bc22 100644 --- a/charts/templates/promtail/podsecuritypolicy.yaml +++ b/charts/templates/promtail/podsecuritypolicy.yaml @@ -1,6 +1,6 @@ {{- if .Values.promtail.enabled }} {{- if .Values.rbac.pspEnabled }} -apiVersion: extensions/v1beta1 +apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: {{ template "promtail.fullname" . }} diff --git a/charts/templates/promtail/role.yaml b/charts/templates/promtail/role.yaml index d1a97cbaf6..e37a92f737 100644 --- a/charts/templates/promtail/role.yaml +++ b/charts/templates/promtail/role.yaml @@ -1,6 +1,6 @@ {{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ template "promtail.fullname" . }} diff --git a/charts/templates/promtail/rolebinding.yaml b/charts/templates/promtail/rolebinding.yaml index aa3521b2cb..e2787b2c61 100644 --- a/charts/templates/promtail/rolebinding.yaml +++ b/charts/templates/promtail/rolebinding.yaml @@ -1,6 +1,6 @@ {{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ template "promtail.fullname" . }} diff --git a/charts/values.yaml b/charts/values.yaml index c47889d8ff..b63fb91ab1 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -16,6 +16,7 @@ loki: minReadySeconds: 0 terminationGracePeriodSeconds: 30 deploymentStrategy: RollingUpdate + port: 3100 image: repository: grafana/loki @@ -25,29 +26,35 @@ loki: service: port: 3100 annotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "http-metrics" labels: {} - readinessProbe: {} - # httpGet: - # path: /ready - # port: 3100 - # initialDelaySeconds: 15 + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 - livenessProbe: {} + livenessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 resources: {} # limits: - # cpu: 100m - # memory: 128Mi + # cpu: 200m + # memory: 256Mi # requests: # cpu: 100m # memory: 128Mi ## Pod Annotations - # podAnnotations: {} + podAnnotations: {} ## Deployment annotations - # annotations: {} + annotations: {} ## Assign a PriorityClassName to pods if set # priorityClassName: @@ -140,28 +147,25 @@ promtail: mountPath: /var/lib/docker/containers readOnly: true - service: - port: 3100 - annotations: {} - labels: {} - - readinessProbe: {} + readinessProbe: null - livenessProbe: {} + livenessProbe: null resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 200m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi ## Pod Annotations - # podAnnotations: {} + podAnnotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "http-metrics" ## Deployment annotations - # annotations: {} + annotations: {} ## Assign a PriorityClassName to pods if set # priorityClassName: @@ -182,3 +186,4 @@ promtail: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## affinity: {} + From 88bdf0efe40a36914889520df489a679830dd4c3 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Wed, 20 Mar 2019 17:52:20 +0000 Subject: [PATCH 033/202] Use __meta_kubernetes_pod_controller_name to ensure we scrape all pods. (#154) * Add a scrape config for pods without name or app labels. Signed-off-by: Tom Wilkie * rebase with master and update to prefer name label but fallback to controller name * Update scrape_config to use pod controller_name in the job If a pod has a "name" label we use that in the job name. Otherwise, - If the pod was created directly by a controller (statefulset, daemonset, job) then we use that controller_name. - if the pod was created by controller created by a controller (deployment, cronJob) then we use the parent controller name. - add a scrape_config.sh tool that generates the scrap_configs for promtail.sh and helm chart from the ksonnet lib. * Reduce duplication in jsonnet. Signed-off-by: Tom Wilkie * Fix rebase Signed-off-by: Tom Wilkie * Tidy up promtail config imports. Signed-off-by: Tom Wilkie --- charts/templates/promtail/configmap.yaml | 126 +++++++++++++++++++++-- 1 file changed, 119 insertions(+), 7 deletions(-) diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index e6c88b5e1f..ec6f36db3e 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -11,24 +11,27 @@ metadata: data: promtail.yaml: | scrape_configs: - - job_name: kubernetes-pods - entry_parser: {{ .Values.promtail.entryParser }} + - entry_parser: '{{ .Values.promtail.entryParser }}' + job_name: kubernetes-pods-name kubernetes_sd_configs: - role: pod relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_name + target_label: __service__ - source_labels: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop regex: ^$ source_labels: - - __meta_kubernetes_pod_label_name + - __service__ - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace - - __meta_kubernetes_pod_label_name + - __service__ target_label: job - action: replace source_labels: @@ -50,28 +53,137 @@ data: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - - job_name: kubernetes-pods-app - entry_parser: {{ .Values.promtail.entryParser }} + - entry_parser: '{{ .Values.promtail.entryParser }}' + job_name: kubernetes-pods-app kubernetes_sd_configs: - role: pod relabel_configs: + - action: drop + regex: .+ + source_labels: + - __meta_kubernetes_pod_label_name + - source_labels: + - __meta_kubernetes_pod_label_app + target_label: __service__ - source_labels: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop regex: ^$ source_labels: - - __meta_kubernetes_pod_label_app + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - entry_parser: '{{ .Values.promtail.entryParser }}' + job_name: kubernetes-pods-direct-controllers + kubernetes_sd_configs: + - role: pod + relabel_configs: - action: drop regex: .+ + separator: '' source_labels: - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: drop + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - entry_parser: '{{ .Values.promtail.entryParser }}' + job_name: kubernetes-pods-indirect-controller + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name - __meta_kubernetes_pod_label_app + - action: keep + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - action: replace + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ target_label: job - action: replace source_labels: From 41951d36ab1f901c2fd93023ccf6dffb43391bd7 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 21 Mar 2019 11:08:19 +0800 Subject: [PATCH 034/202] Add backoff_config in helm Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/templates/promtail/configmap.yaml | 5 +++++ charts/values.yaml | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index ec6f36db3e..5a85f0c932 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -10,6 +10,11 @@ metadata: heritage: {{ .Release.Service }} data: promtail.yaml: | + backoff_config: + minbackoff: {{ .Values.promtail.config.backoff_config.minbackoff }} + maxbackoff: {{ .Values.promtail.config.backoff_config.maxbackoff }} + maxretries: {{ .Values.promtail.config.backoff_config.maxretries }} + scrape_configs: - entry_parser: '{{ .Values.promtail.entryParser }}' job_name: kubernetes-pods-name diff --git a/charts/values.yaml b/charts/values.yaml index ff6da44408..d31f5ef7de 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -127,6 +127,15 @@ promtail: entryParser: docker + config: + backoff_config: + # Initial backoff time between retries + minbackoff: 100ms + # Maximum backoff time between retries + maxbackoff: 5s + # Maximum number of retires when sending batches, 0 means infinite retries + maxretries: 5 + image: repository: grafana/promtail tag: latest From 2be2a1532b5a705091ae6973a380b0fd5f9f6683 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 21 Mar 2019 11:26:36 +0800 Subject: [PATCH 035/202] remove white noise Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/values.yaml b/charts/values.yaml index d31f5ef7de..853470ef90 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -82,7 +82,7 @@ loki: # values: # - loki # topologyKey: "kubernetes.io/hostname" - + ## Enable persistence using Persistent Volume Claims ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ ## If you set enabled as "True", you need : From 975aba067734a25ddb62b45bb301ed7d768fd462 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Wed, 20 Mar 2019 12:19:40 -0500 Subject: [PATCH 036/202] Helm: Run with reduced privileges Signed-off-by: Steven Sheehy --- charts/Chart.yaml | 2 +- charts/templates/loki/deployment.yaml | 2 ++ charts/templates/loki/podsecuritypolicy.yaml | 23 ++++++++++---- charts/templates/loki/pvc.yaml | 6 ++-- charts/templates/loki/role.yaml | 2 +- charts/templates/loki/rolebinding.yaml | 2 +- charts/templates/loki/serviceaccount.yaml | 2 +- charts/templates/promtail/clusterrole.yaml | 4 +-- .../promtail/clusterrolebinding.yaml | 4 +-- charts/templates/promtail/configmap.yaml | 5 ++- charts/templates/promtail/daemonset.yaml | 10 ++++-- .../templates/promtail/podsecuritypolicy.yaml | 10 +++--- charts/templates/promtail/role.yaml | 2 +- charts/templates/promtail/rolebinding.yaml | 2 +- charts/values.yaml | 31 ++++++++++++++----- 15 files changed, 70 insertions(+), 37 deletions(-) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 40b88cca03..6eebbd4549 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.3.0 +version: 0.4.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index 93eff4d02c..c221489b17 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -60,6 +60,8 @@ spec: {{- toYaml .Values.loki.readinessProbe | nindent 12 }} resources: {{- toYaml .Values.loki.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.loki.securityContext | nindent 12 }} nodeSelector: {{- toYaml .Values.loki.nodeSelector | nindent 8 }} affinity: diff --git a/charts/templates/loki/podsecuritypolicy.yaml b/charts/templates/loki/podsecuritypolicy.yaml index c81894d613..a5ef5c3f27 100644 --- a/charts/templates/loki/podsecuritypolicy.yaml +++ b/charts/templates/loki/podsecuritypolicy.yaml @@ -6,7 +6,7 @@ metadata: name: {{ template "loki.fullname" . }} labels: app: {{ template "loki.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} + chart: {{ template "loki.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} spec: @@ -21,13 +21,24 @@ spec: hostIPC: false hostPID: false runAsUser: - rule: 'RunAsAny' + rule: 'MustRunAsNonRoot' seLinux: - rule: 'RunAsAny' + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 supplementalGroups: - rule: 'RunAsAny' + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 fsGroup: - rule: 'RunAsAny' - readOnlyRootFilesystem: false + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL {{- end }} {{- end }} diff --git a/charts/templates/loki/pvc.yaml b/charts/templates/loki/pvc.yaml index 9bba88becb..f9041d551d 100644 --- a/charts/templates/loki/pvc.yaml +++ b/charts/templates/loki/pvc.yaml @@ -6,13 +6,11 @@ metadata: name: {{ template "loki.fullname" . }} labels: app: {{ template "loki.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} + chart: {{ template "loki.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} - {{- with .Values.loki.persistence.annotations }} annotations: -{{ toYaml . | indent 4 }} - {{- end }} + {{- toYaml .Values.loki.persistence.annotations | nindent 4 }} spec: accessModes: {{- range .Values.loki.persistence.accessModes }} diff --git a/charts/templates/loki/role.yaml b/charts/templates/loki/role.yaml index eb9c508f25..794bf3aac0 100644 --- a/charts/templates/loki/role.yaml +++ b/charts/templates/loki/role.yaml @@ -6,7 +6,7 @@ metadata: name: {{ template "loki.fullname" . }} labels: app: {{ template "loki.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} + chart: {{ template "loki.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} {{- if .Values.rbac.pspEnabled }} diff --git a/charts/templates/loki/rolebinding.yaml b/charts/templates/loki/rolebinding.yaml index 4aa765bb4a..734fc9aeb7 100644 --- a/charts/templates/loki/rolebinding.yaml +++ b/charts/templates/loki/rolebinding.yaml @@ -6,7 +6,7 @@ metadata: name: {{ template "loki.fullname" . }} labels: app: {{ template "loki.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} + chart: {{ template "loki.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} roleRef: diff --git a/charts/templates/loki/serviceaccount.yaml b/charts/templates/loki/serviceaccount.yaml index 302cafdb45..fae926023d 100644 --- a/charts/templates/loki/serviceaccount.yaml +++ b/charts/templates/loki/serviceaccount.yaml @@ -5,7 +5,7 @@ kind: ServiceAccount metadata: labels: app: {{ template "loki.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} + chart: {{ template "loki.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} name: {{ template "loki.serviceAccountName" . }} diff --git a/charts/templates/promtail/clusterrole.yaml b/charts/templates/promtail/clusterrole.yaml index 75be62d03d..b9d3709013 100644 --- a/charts/templates/promtail/clusterrole.yaml +++ b/charts/templates/promtail/clusterrole.yaml @@ -8,10 +8,8 @@ metadata: chart: {{ template "promtail.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} -{{- with .Values.promtail.annotations }} annotations: -{{ toYaml . | indent 4 }} -{{- end }} + {{- toYaml .Values.promtail.annotations | nindent 4 }} name: {{ template "promtail.fullname" . }}-clusterrole rules: - apiGroups: [""] # "" indicates the core API group diff --git a/charts/templates/promtail/clusterrolebinding.yaml b/charts/templates/promtail/clusterrolebinding.yaml index 7a842922c8..a6d9984325 100644 --- a/charts/templates/promtail/clusterrolebinding.yaml +++ b/charts/templates/promtail/clusterrolebinding.yaml @@ -9,10 +9,8 @@ metadata: chart: {{ template "promtail.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} -{{- with .Values.promtail.annotations }} annotations: -{{ toYaml . | indent 4 }} -{{- end }} + {{- toYaml .Values.promtail.annotations | nindent 4 }} subjects: - kind: ServiceAccount name: {{ template "promtail.serviceAccountName" . }} diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index 5a85f0c932..71d6c0617a 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -14,7 +14,10 @@ data: minbackoff: {{ .Values.promtail.config.backoff_config.minbackoff }} maxbackoff: {{ .Values.promtail.config.backoff_config.maxbackoff }} maxretries: {{ .Values.promtail.config.backoff_config.maxretries }} - + server: + http_listen_port: {{ .Values.promtail.port }} + positions: + filename: /run/promtail/positions.yaml scrape_configs: - entry_parser: '{{ .Values.promtail.entryParser }}' job_name: kubernetes-pods-name diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index 1f02fd5e2c..a3bdd627e9 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -42,6 +42,8 @@ spec: volumeMounts: - name: config mountPath: /etc/promtail + - name: run + mountPath: /run/promtail {{- with .Values.promtail.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -51,11 +53,10 @@ spec: fieldRef: fieldPath: spec.nodeName ports: - - containerPort: 80 + - containerPort: {{ .Values.promtail.port }} name: http-metrics securityContext: - privileged: true - runAsUser: 0 + {{- toYaml .Values.promtail.securityContext | nindent 12 }} livenessProbe: {{- toYaml .Values.promtail.livenessProbe | nindent 12 }} readinessProbe: @@ -72,6 +73,9 @@ spec: - name: config configMap: name: {{ template "promtail.fullname" . }} + - name: run + hostPath: + path: /run/promtail {{- with .Values.promtail.volumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/templates/promtail/podsecuritypolicy.yaml b/charts/templates/promtail/podsecuritypolicy.yaml index 2666f6bc22..96d761caa8 100644 --- a/charts/templates/promtail/podsecuritypolicy.yaml +++ b/charts/templates/promtail/podsecuritypolicy.yaml @@ -6,12 +6,12 @@ metadata: name: {{ template "promtail.fullname" . }} labels: app: {{ template "promtail.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} + chart: {{ template "promtail.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} spec: - privileged: true - allowPrivilegeEscalation: true + privileged: false + allowPrivilegeEscalation: false volumes: - 'secret' - 'configMap' @@ -27,6 +27,8 @@ spec: rule: 'RunAsAny' fsGroup: rule: 'RunAsAny' - readOnlyRootFilesystem: false + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL {{- end }} {{- end }} diff --git a/charts/templates/promtail/role.yaml b/charts/templates/promtail/role.yaml index e37a92f737..96ead979c7 100644 --- a/charts/templates/promtail/role.yaml +++ b/charts/templates/promtail/role.yaml @@ -6,7 +6,7 @@ metadata: name: {{ template "promtail.fullname" . }} labels: app: {{ template "promtail.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} + chart: {{ template "promtail.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} {{- if .Values.rbac.pspEnabled }} diff --git a/charts/templates/promtail/rolebinding.yaml b/charts/templates/promtail/rolebinding.yaml index e2787b2c61..9f77e754c1 100644 --- a/charts/templates/promtail/rolebinding.yaml +++ b/charts/templates/promtail/rolebinding.yaml @@ -6,7 +6,7 @@ metadata: name: {{ template "promtail.fullname" . }} labels: app: {{ template "promtail.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} + chart: {{ template "promtail.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} roleRef: diff --git a/charts/values.yaml b/charts/values.yaml index 853470ef90..ee7df7a807 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -50,6 +50,13 @@ loki: # cpu: 100m # memory: 128Mi + securityContext: + fsGroup: 10001 + readOnlyRootFilesystem: true + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + ## Pod Annotations podAnnotations: {} @@ -94,7 +101,7 @@ loki: - ReadWriteOnce size: 10Gi storageClassName: default - # annotations: {} + annotations: {} # subPath: "" # existingClaim: @@ -141,18 +148,22 @@ promtail: tag: latest pullPolicy: Always # Always pull while in BETA + port: 3101 + + # Extra volumes to scrape logs from volumes: - - name: varlog + - name: pods hostPath: - path: /var/log - - name: varlibdockercontainers + path: /var/log/pods + - name: docker hostPath: path: /var/lib/docker/containers volumeMounts: - - name: varlog - mountPath: /var/log - - name: varlibdockercontainers + - name: pods + mountPath: /var/log/pods + readOnly: true + - name: docker mountPath: /var/lib/docker/containers readOnly: true @@ -168,6 +179,12 @@ promtail: # cpu: 100m # memory: 128Mi + securityContext: + fsGroup: 0 + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsUser: 0 + ## Pod Annotations podAnnotations: {} # prometheus.io/scrape: "true" From de12350a3f16333e5df243d183820f1c87045d98 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Mon, 25 Mar 2019 11:45:53 -0500 Subject: [PATCH 037/202] Helm: Allow custom promtail configuration Signed-off-by: Steven Sheehy --- charts/Chart.yaml | 2 +- charts/templates/promtail/configmap.yaml | 204 +-------------------- charts/values.yaml | 218 ++++++++++++++++++++++- 3 files changed, 211 insertions(+), 213 deletions(-) diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 6eebbd4549..57501cb32c 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.4.0 +version: 0.5.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/templates/promtail/configmap.yaml b/charts/templates/promtail/configmap.yaml index 71d6c0617a..25fdc366b6 100644 --- a/charts/templates/promtail/configmap.yaml +++ b/charts/templates/promtail/configmap.yaml @@ -10,207 +10,5 @@ metadata: heritage: {{ .Release.Service }} data: promtail.yaml: | - backoff_config: - minbackoff: {{ .Values.promtail.config.backoff_config.minbackoff }} - maxbackoff: {{ .Values.promtail.config.backoff_config.maxbackoff }} - maxretries: {{ .Values.promtail.config.backoff_config.maxretries }} - server: - http_listen_port: {{ .Values.promtail.port }} - positions: - filename: /run/promtail/positions.yaml - scrape_configs: - - entry_parser: '{{ .Values.promtail.entryParser }}' - job_name: kubernetes-pods-name - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: - - __meta_kubernetes_pod_label_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - entry_parser: '{{ .Values.promtail.entryParser }}' - job_name: kubernetes-pods-app - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - source_labels: - - __meta_kubernetes_pod_label_name - - source_labels: - - __meta_kubernetes_pod_label_app - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - entry_parser: '{{ .Values.promtail.entryParser }}' - job_name: kubernetes-pods-direct-controllers - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - separator: '' - source_labels: - - __meta_kubernetes_pod_label_name - - __meta_kubernetes_pod_label_app - - action: drop - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - - source_labels: - - __meta_kubernetes_pod_controller_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - entry_parser: '{{ .Values.promtail.entryParser }}' - job_name: kubernetes-pods-indirect-controller - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - separator: '' - source_labels: - - __meta_kubernetes_pod_label_name - - __meta_kubernetes_pod_label_app - - action: keep - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - - action: replace - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ + {{- tpl (toYaml .Values.promtail.config | nindent 4 ) . }} {{- end }} diff --git a/charts/values.yaml b/charts/values.yaml index ee7df7a807..a6b6ed1bcc 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -134,20 +134,12 @@ promtail: entryParser: docker - config: - backoff_config: - # Initial backoff time between retries - minbackoff: 100ms - # Maximum backoff time between retries - maxbackoff: 5s - # Maximum number of retires when sending batches, 0 means infinite retries - maxretries: 5 - image: repository: grafana/promtail tag: latest pullPolicy: Always # Always pull while in BETA + # This should match promtail.config.server.http_listen_port port: 3101 # Extra volumes to scrape logs from @@ -213,3 +205,211 @@ promtail: ## affinity: {} + config: + client: + backoff_config: + # Initial backoff time between retries + minbackoff: 100ms + # Maximum backoff time between retries + maxbackoff: 5s + # Maximum number of retires when sending batches, 0 means infinite retries + maxretries: 5 + server: + http_listen_port: 3101 + positions: + filename: /run/promtail/positions.yaml + scrape_configs: + - entry_parser: '{{ .Values.promtail.entryParser }}' + job_name: kubernetes-pods-name + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - entry_parser: '{{ .Values.promtail.entryParser }}' + job_name: kubernetes-pods-app + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + source_labels: + - __meta_kubernetes_pod_label_name + - source_labels: + - __meta_kubernetes_pod_label_app + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - entry_parser: '{{ .Values.promtail.entryParser }}' + job_name: kubernetes-pods-direct-controllers + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: drop + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - entry_parser: '{{ .Values.promtail.entryParser }}' + job_name: kubernetes-pods-indirect-controller + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: keep + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - action: replace + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ From f85b0c3a0e8c6f02a5d38c236ad2758d55ac34cc Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Mon, 25 Mar 2019 12:34:04 -0500 Subject: [PATCH 038/202] Fix liveness/readyness probes, they can not be empty {} --- charts/templates/promtail/daemonset.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index a3bdd627e9..f913536189 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -57,10 +57,14 @@ spec: name: http-metrics securityContext: {{- toYaml .Values.promtail.securityContext | nindent 12 }} + {{- if .Values.promtail.livenessProbe }} livenessProbe: {{- toYaml .Values.promtail.livenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.promtail.livenessProbe }} readinessProbe: {{- toYaml .Values.promtail.readinessProbe | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.promtail.resources | nindent 12 }} nodeSelector: From 868f3ded4ae2c4032303d5494d7a43ed4bfb9bdc Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Mon, 25 Mar 2019 13:50:41 -0500 Subject: [PATCH 039/202] range is not a valid option for seLinux --- charts/templates/loki/podsecuritypolicy.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/charts/templates/loki/podsecuritypolicy.yaml b/charts/templates/loki/podsecuritypolicy.yaml index a5ef5c3f27..02e1e8615d 100644 --- a/charts/templates/loki/podsecuritypolicy.yaml +++ b/charts/templates/loki/podsecuritypolicy.yaml @@ -23,10 +23,7 @@ spec: runAsUser: rule: 'MustRunAsNonRoot' seLinux: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 + rule: 'RunAsAny' supplementalGroups: rule: 'MustRunAs' ranges: From b76c28d1c6e6ba6ade8992e2bbf16b5a829ee7dd Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Mon, 25 Mar 2019 14:01:06 -0500 Subject: [PATCH 040/202] Fix security context remove fsGroup --- charts/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/values.yaml b/charts/values.yaml index ee7df7a807..1dd0d3b87d 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -51,7 +51,6 @@ loki: # memory: 128Mi securityContext: - fsGroup: 10001 readOnlyRootFilesystem: true runAsGroup: 10001 runAsNonRoot: true @@ -180,7 +179,6 @@ promtail: # memory: 128Mi securityContext: - fsGroup: 0 readOnlyRootFilesystem: true runAsGroup: 0 runAsUser: 0 From 5ee72a60ba0f9ac68dc4613de92f02ddd33683e9 Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Mon, 25 Mar 2019 15:11:48 -0500 Subject: [PATCH 041/202] Change Loki's Pod and Conainter Security Policies. --- charts/templates/loki/deployment.yaml | 4 +++- charts/values.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/charts/templates/loki/deployment.yaml b/charts/templates/loki/deployment.yaml index c221489b17..e357708e13 100644 --- a/charts/templates/loki/deployment.yaml +++ b/charts/templates/loki/deployment.yaml @@ -37,6 +37,8 @@ spec: serviceAccountName: {{ template "loki.serviceAccountName" . }} {{- if .Values.loki.priorityClassName }} priorityClassName: {{ .Values.loki.priorityClassName }} + securityContext: + {{- toYaml .Values.loki.securityContext | nindent 8 }} {{- end }} containers: - name: {{ .Chart.Name }} @@ -61,7 +63,7 @@ spec: resources: {{- toYaml .Values.loki.resources | nindent 12 }} securityContext: - {{- toYaml .Values.loki.securityContext | nindent 12 }} + readOnlyRootFilesystem: true nodeSelector: {{- toYaml .Values.loki.nodeSelector | nindent 8 }} affinity: diff --git a/charts/values.yaml b/charts/values.yaml index 1dd0d3b87d..b18e0044d4 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -51,7 +51,7 @@ loki: # memory: 128Mi securityContext: - readOnlyRootFilesystem: true + fsGroup: 10001 runAsGroup: 10001 runAsNonRoot: true runAsUser: 10001 From 28d9f634748195eec3fdfc5f4f5950e0cf76e6fc Mon Sep 17 00:00:00 2001 From: Ryan Bonham Date: Mon, 25 Mar 2019 15:26:14 -0500 Subject: [PATCH 042/202] Dropping liveness/readiness probe from promtail helm template as there is not health checks currently.. Can add back once something exist and we can do so without if statements.. --- charts/templates/promtail/daemonset.yaml | 8 -------- charts/values.yaml | 4 ---- 2 files changed, 12 deletions(-) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index f913536189..ca9464651e 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -57,14 +57,6 @@ spec: name: http-metrics securityContext: {{- toYaml .Values.promtail.securityContext | nindent 12 }} - {{- if .Values.promtail.livenessProbe }} - livenessProbe: - {{- toYaml .Values.promtail.livenessProbe | nindent 12 }} - {{- end }} - {{- if .Values.promtail.livenessProbe }} - readinessProbe: - {{- toYaml .Values.promtail.readinessProbe | nindent 12 }} - {{- end }} resources: {{- toYaml .Values.promtail.resources | nindent 12 }} nodeSelector: diff --git a/charts/values.yaml b/charts/values.yaml index b18e0044d4..199fbb881f 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -166,10 +166,6 @@ promtail: mountPath: /var/lib/docker/containers readOnly: true - readinessProbe: null - - livenessProbe: null - resources: {} # limits: # cpu: 200m From 01934346fef9ee32351be34d38dc44e72fc7cce3 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 26 Mar 2019 17:22:19 +0800 Subject: [PATCH 043/202] Add health check Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/templates/promtail/daemonset.yaml | 8 ++++++++ charts/values.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index ca9464651e..f913536189 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -57,6 +57,14 @@ spec: name: http-metrics securityContext: {{- toYaml .Values.promtail.securityContext | nindent 12 }} + {{- if .Values.promtail.livenessProbe }} + livenessProbe: + {{- toYaml .Values.promtail.livenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.promtail.livenessProbe }} + readinessProbe: + {{- toYaml .Values.promtail.readinessProbe | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.promtail.resources | nindent 12 }} nodeSelector: diff --git a/charts/values.yaml b/charts/values.yaml index bac543bda1..5f3cf351e5 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -158,6 +158,30 @@ promtail: mountPath: /var/lib/docker/containers readOnly: true + readinessProbe: + failureThreshold: 5 + exec: + command: + - /bin/sh + - -c + - /bin/ps -ef | grep promtail | grep -v grep + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + livenessProbe: + failureThreshold: 5 + exec: + command: + - /bin/sh + - -c + - /bin/ps -ef | grep promtail | grep -v grep + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} # limits: # cpu: 200m From a2944e1cc27790fd3bc6f483568842428832f6d2 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 26 Mar 2019 19:27:59 +0800 Subject: [PATCH 044/202] add miss promtail client config option Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/values.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/charts/values.yaml b/charts/values.yaml index bac543bda1..4600a008c9 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -201,6 +201,14 @@ promtail: config: client: + # Maximum wait period before sending batch + batchwait: 1s + # Maximum batch size to accrue before sending, unit is byte + batchsize: 102400 + + # Maximum time to wait for server to respond to a request + timeout: 10s + backoff_config: # Initial backoff time between retries minbackoff: 100ms @@ -208,6 +216,10 @@ promtail: maxbackoff: 5s # Maximum number of retires when sending batches, 0 means infinite retries maxretries: 5 + + # The labels to add to any time series or alerts when communicating with loki + external_labels: {} + server: http_listen_port: 3101 positions: From 4e69510491ac12765b3c30c093b53c8af3339f1f Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 29 Mar 2019 18:47:26 +0800 Subject: [PATCH 045/202] update check Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/values.yaml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/charts/values.yaml b/charts/values.yaml index 5f3cf351e5..6f819612f4 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -160,11 +160,9 @@ promtail: readinessProbe: failureThreshold: 5 - exec: - command: - - /bin/sh - - -c - - /bin/ps -ef | grep promtail | grep -v grep + httpGet: + path: /metrics + port: http-metrics initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 @@ -172,11 +170,9 @@ promtail: livenessProbe: failureThreshold: 5 - exec: - command: - - /bin/sh - - -c - - /bin/ps -ef | grep promtail | grep -v grep + httpGet: + path: /metrics + port: http-metrics initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 From bb7a99b93d0126526d405a68e97b48d4bd465d50 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Sat, 30 Mar 2019 11:31:14 +0800 Subject: [PATCH 046/202] fix typo Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/templates/promtail/daemonset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/templates/promtail/daemonset.yaml b/charts/templates/promtail/daemonset.yaml index f913536189..ca88143355 100644 --- a/charts/templates/promtail/daemonset.yaml +++ b/charts/templates/promtail/daemonset.yaml @@ -61,7 +61,7 @@ spec: livenessProbe: {{- toYaml .Values.promtail.livenessProbe | nindent 12 }} {{- end }} - {{- if .Values.promtail.livenessProbe }} + {{- if .Values.promtail.readinessProbe }} readinessProbe: {{- toYaml .Values.promtail.readinessProbe | nindent 12 }} {{- end }} From 513388ad7d041127262ca31be6fd1fa11e171b2f Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Sat, 30 Mar 2019 19:39:07 -0500 Subject: [PATCH 047/202] Setup Helm chart repository Signed-off-by: Steven Sheehy --- charts/{ => loki}/Chart.yaml | 0 charts/{ => loki}/README.md | 17 ++++++----------- charts/{ => loki}/grafana.yaml | 0 charts/{ => loki}/templates/loki/_helpers.tpl | 0 .../{ => loki}/templates/loki/deployment.yaml | 0 .../templates/loki/networkpolicy.yaml | 0 .../templates/loki/podsecuritypolicy.yaml | 0 charts/{ => loki}/templates/loki/pvc.yaml | 0 charts/{ => loki}/templates/loki/role.yaml | 0 .../{ => loki}/templates/loki/rolebinding.yaml | 0 charts/{ => loki}/templates/loki/secret.yaml | 0 charts/{ => loki}/templates/loki/service.yaml | 0 .../templates/loki/serviceaccount.yaml | 0 .../{ => loki}/templates/promtail/_helpers.tpl | 0 .../templates/promtail/clusterrole.yaml | 0 .../templates/promtail/clusterrolebinding.yaml | 0 .../templates/promtail/configmap.yaml | 0 .../templates/promtail/daemonset.yaml | 0 .../templates/promtail/podsecuritypolicy.yaml | 0 charts/{ => loki}/templates/promtail/role.yaml | 0 .../templates/promtail/rolebinding.yaml | 0 charts/{ => loki}/values.yaml | 0 22 files changed, 6 insertions(+), 11 deletions(-) rename charts/{ => loki}/Chart.yaml (100%) rename charts/{ => loki}/README.md (72%) rename charts/{ => loki}/grafana.yaml (100%) rename charts/{ => loki}/templates/loki/_helpers.tpl (100%) rename charts/{ => loki}/templates/loki/deployment.yaml (100%) rename charts/{ => loki}/templates/loki/networkpolicy.yaml (100%) rename charts/{ => loki}/templates/loki/podsecuritypolicy.yaml (100%) rename charts/{ => loki}/templates/loki/pvc.yaml (100%) rename charts/{ => loki}/templates/loki/role.yaml (100%) rename charts/{ => loki}/templates/loki/rolebinding.yaml (100%) rename charts/{ => loki}/templates/loki/secret.yaml (100%) rename charts/{ => loki}/templates/loki/service.yaml (100%) rename charts/{ => loki}/templates/loki/serviceaccount.yaml (100%) rename charts/{ => loki}/templates/promtail/_helpers.tpl (100%) rename charts/{ => loki}/templates/promtail/clusterrole.yaml (100%) rename charts/{ => loki}/templates/promtail/clusterrolebinding.yaml (100%) rename charts/{ => loki}/templates/promtail/configmap.yaml (100%) rename charts/{ => loki}/templates/promtail/daemonset.yaml (100%) rename charts/{ => loki}/templates/promtail/podsecuritypolicy.yaml (100%) rename charts/{ => loki}/templates/promtail/role.yaml (100%) rename charts/{ => loki}/templates/promtail/rolebinding.yaml (100%) rename charts/{ => loki}/values.yaml (100%) diff --git a/charts/Chart.yaml b/charts/loki/Chart.yaml similarity index 100% rename from charts/Chart.yaml rename to charts/loki/Chart.yaml diff --git a/charts/README.md b/charts/loki/README.md similarity index 72% rename from charts/README.md rename to charts/loki/README.md index 36b5590621..7461a2b1d7 100644 --- a/charts/README.md +++ b/charts/loki/README.md @@ -1,24 +1,19 @@ -# Deploy Loki to Kubernetes with Helm +# Loki Helm Chart ## Prerequisites -Make sure you have the helm configure on your cluster: +Make sure you have Helm [installed](https://helm.sh/docs/using_helm/#installing-helm) and +[deployed](https://helm.sh/docs/using_helm/#installing-tiller) to your cluster. Then add +Loki's chart repository to Helm: ```bash -$ helm init -``` - -Clone `grafana/loki` repository and navigate to `production helm` directory: - -```bash -$ git clone https://github.com/grafana/loki.git -$ cd loki/production/helm +$ helm repo add loki https://grafana.github.io/loki/charts ``` ## Deploy Loki and Promtail to your cluster ```bash -$ helm install . -n loki --namespace +$ helm upgrade --install loki loki/loki ``` ## Deploy Grafana to your cluster diff --git a/charts/grafana.yaml b/charts/loki/grafana.yaml similarity index 100% rename from charts/grafana.yaml rename to charts/loki/grafana.yaml diff --git a/charts/templates/loki/_helpers.tpl b/charts/loki/templates/loki/_helpers.tpl similarity index 100% rename from charts/templates/loki/_helpers.tpl rename to charts/loki/templates/loki/_helpers.tpl diff --git a/charts/templates/loki/deployment.yaml b/charts/loki/templates/loki/deployment.yaml similarity index 100% rename from charts/templates/loki/deployment.yaml rename to charts/loki/templates/loki/deployment.yaml diff --git a/charts/templates/loki/networkpolicy.yaml b/charts/loki/templates/loki/networkpolicy.yaml similarity index 100% rename from charts/templates/loki/networkpolicy.yaml rename to charts/loki/templates/loki/networkpolicy.yaml diff --git a/charts/templates/loki/podsecuritypolicy.yaml b/charts/loki/templates/loki/podsecuritypolicy.yaml similarity index 100% rename from charts/templates/loki/podsecuritypolicy.yaml rename to charts/loki/templates/loki/podsecuritypolicy.yaml diff --git a/charts/templates/loki/pvc.yaml b/charts/loki/templates/loki/pvc.yaml similarity index 100% rename from charts/templates/loki/pvc.yaml rename to charts/loki/templates/loki/pvc.yaml diff --git a/charts/templates/loki/role.yaml b/charts/loki/templates/loki/role.yaml similarity index 100% rename from charts/templates/loki/role.yaml rename to charts/loki/templates/loki/role.yaml diff --git a/charts/templates/loki/rolebinding.yaml b/charts/loki/templates/loki/rolebinding.yaml similarity index 100% rename from charts/templates/loki/rolebinding.yaml rename to charts/loki/templates/loki/rolebinding.yaml diff --git a/charts/templates/loki/secret.yaml b/charts/loki/templates/loki/secret.yaml similarity index 100% rename from charts/templates/loki/secret.yaml rename to charts/loki/templates/loki/secret.yaml diff --git a/charts/templates/loki/service.yaml b/charts/loki/templates/loki/service.yaml similarity index 100% rename from charts/templates/loki/service.yaml rename to charts/loki/templates/loki/service.yaml diff --git a/charts/templates/loki/serviceaccount.yaml b/charts/loki/templates/loki/serviceaccount.yaml similarity index 100% rename from charts/templates/loki/serviceaccount.yaml rename to charts/loki/templates/loki/serviceaccount.yaml diff --git a/charts/templates/promtail/_helpers.tpl b/charts/loki/templates/promtail/_helpers.tpl similarity index 100% rename from charts/templates/promtail/_helpers.tpl rename to charts/loki/templates/promtail/_helpers.tpl diff --git a/charts/templates/promtail/clusterrole.yaml b/charts/loki/templates/promtail/clusterrole.yaml similarity index 100% rename from charts/templates/promtail/clusterrole.yaml rename to charts/loki/templates/promtail/clusterrole.yaml diff --git a/charts/templates/promtail/clusterrolebinding.yaml b/charts/loki/templates/promtail/clusterrolebinding.yaml similarity index 100% rename from charts/templates/promtail/clusterrolebinding.yaml rename to charts/loki/templates/promtail/clusterrolebinding.yaml diff --git a/charts/templates/promtail/configmap.yaml b/charts/loki/templates/promtail/configmap.yaml similarity index 100% rename from charts/templates/promtail/configmap.yaml rename to charts/loki/templates/promtail/configmap.yaml diff --git a/charts/templates/promtail/daemonset.yaml b/charts/loki/templates/promtail/daemonset.yaml similarity index 100% rename from charts/templates/promtail/daemonset.yaml rename to charts/loki/templates/promtail/daemonset.yaml diff --git a/charts/templates/promtail/podsecuritypolicy.yaml b/charts/loki/templates/promtail/podsecuritypolicy.yaml similarity index 100% rename from charts/templates/promtail/podsecuritypolicy.yaml rename to charts/loki/templates/promtail/podsecuritypolicy.yaml diff --git a/charts/templates/promtail/role.yaml b/charts/loki/templates/promtail/role.yaml similarity index 100% rename from charts/templates/promtail/role.yaml rename to charts/loki/templates/promtail/role.yaml diff --git a/charts/templates/promtail/rolebinding.yaml b/charts/loki/templates/promtail/rolebinding.yaml similarity index 100% rename from charts/templates/promtail/rolebinding.yaml rename to charts/loki/templates/promtail/rolebinding.yaml diff --git a/charts/values.yaml b/charts/loki/values.yaml similarity index 100% rename from charts/values.yaml rename to charts/loki/values.yaml From 342c40be0a84f6665ac8058730b0b6bd86752793 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 4 Apr 2019 18:35:13 +0800 Subject: [PATCH 048/202] Fix service account error Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/loki/templates/loki/serviceaccount.yaml | 2 +- charts/loki/values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/loki/templates/loki/serviceaccount.yaml b/charts/loki/templates/loki/serviceaccount.yaml index fae926023d..80424aef31 100644 --- a/charts/loki/templates/loki/serviceaccount.yaml +++ b/charts/loki/templates/loki/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if .Values.loki.enabled }} +{{- if or (.Values.loki.enabled) (.Values.promtail.enabled) }} {{- if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 8af9f3a8ae..94ec46f4a6 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -6,6 +6,7 @@ networkPolicy: enabled: false serviceAccount: + # if has deployed loki or promtail, must set false create: true name: From e5ed45afe764952b4364923bce63cb5aeab3442c Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Thu, 4 Apr 2019 10:58:30 -0500 Subject: [PATCH 049/202] Separate loki & promtail charts Signed-off-by: Steven Sheehy --- charts/loki/Chart.yaml | 2 +- charts/loki/README.md | 12 + charts/loki/requirements.yaml | 6 + charts/loki/templates/NOTES.txt | 3 + charts/loki/templates/{loki => }/_helpers.tpl | 20 +- .../loki/templates/{loki => }/deployment.yaml | 48 +- charts/loki/templates/loki/pvc.yaml | 24 - .../templates/{loki => }/networkpolicy.yaml | 2 +- .../{loki => }/podsecuritypolicy.yaml | 3 +- charts/loki/templates/pvc.yaml | 22 + charts/loki/templates/{loki => }/role.yaml | 3 +- .../templates/{loki => }/rolebinding.yaml | 3 +- charts/loki/templates/{loki => }/secret.yaml | 0 charts/loki/templates/{loki => }/service.yaml | 9 +- .../templates/{loki => }/serviceaccount.yaml | 3 +- charts/loki/values.yaml | 533 ++++-------------- charts/promtail/Chart.yaml | 13 + charts/promtail/templates/NOTES.txt | 3 + .../templates}/_helpers.tpl | 19 +- .../templates}/clusterrole.yaml | 4 - .../templates}/clusterrolebinding.yaml | 4 - .../templates}/configmap.yaml | 4 +- .../templates}/daemonset.yaml | 44 +- .../templates}/podsecuritypolicy.yaml | 2 - .../promtail => promtail/templates}/role.yaml | 2 - .../templates}/rolebinding.yaml | 3 +- charts/promtail/templates/serviceaccount.yaml | 12 + charts/promtail/values.yaml | 313 ++++++++++ 28 files changed, 573 insertions(+), 543 deletions(-) create mode 100644 charts/loki/requirements.yaml create mode 100644 charts/loki/templates/NOTES.txt rename charts/loki/templates/{loki => }/_helpers.tpl (73%) rename charts/loki/templates/{loki => }/deployment.yaml (52%) delete mode 100644 charts/loki/templates/loki/pvc.yaml rename charts/loki/templates/{loki => }/networkpolicy.yaml (93%) rename charts/loki/templates/{loki => }/podsecuritypolicy.yaml (95%) create mode 100644 charts/loki/templates/pvc.yaml rename charts/loki/templates/{loki => }/role.yaml (92%) rename charts/loki/templates/{loki => }/rolebinding.yaml (92%) rename charts/loki/templates/{loki => }/secret.yaml (100%) rename charts/loki/templates/{loki => }/service.yaml (71%) rename charts/loki/templates/{loki => }/serviceaccount.yaml (80%) create mode 100644 charts/promtail/Chart.yaml create mode 100644 charts/promtail/templates/NOTES.txt rename charts/{loki/templates/promtail => promtail/templates}/_helpers.tpl (64%) rename charts/{loki/templates/promtail => promtail/templates}/clusterrole.yaml (80%) rename charts/{loki/templates/promtail => promtail/templates}/clusterrolebinding.yaml (83%) rename charts/{loki/templates/promtail => promtail/templates}/configmap.yaml (71%) rename charts/{loki/templates/promtail => promtail/templates}/daemonset.yaml (53%) rename charts/{loki/templates/promtail => promtail/templates}/podsecuritypolicy.yaml (93%) rename charts/{loki/templates/promtail => promtail/templates}/role.yaml (91%) rename charts/{loki/templates/promtail => promtail/templates}/rolebinding.yaml (91%) create mode 100644 charts/promtail/templates/serviceaccount.yaml create mode 100644 charts/promtail/values.yaml diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 57501cb32c..a6b3fbb8ae 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.5.0 +version: 0.6.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/README.md b/charts/loki/README.md index 7461a2b1d7..9d19a306a4 100644 --- a/charts/loki/README.md +++ b/charts/loki/README.md @@ -16,6 +16,18 @@ $ helm repo add loki https://grafana.github.io/loki/charts $ helm upgrade --install loki loki/loki ``` +## Deploy Loki only + +```bash +$ helm upgrade --install loki loki/loki --set 'promtail.enabled=false' +``` + +## Deploy Promtail only + +```bash +$ helm upgrade --install promtail loki/promtail +``` + ## Deploy Grafana to your cluster To install Grafana on your cluster with helm, use the following command: diff --git a/charts/loki/requirements.yaml b/charts/loki/requirements.yaml new file mode 100644 index 0000000000..426a6970a7 --- /dev/null +++ b/charts/loki/requirements.yaml @@ -0,0 +1,6 @@ +dependencies: +- name: "promtail" + condition: promtail.enabled + repository: "file://../promtail" + version: "^0.6.0" + diff --git a/charts/loki/templates/NOTES.txt b/charts/loki/templates/NOTES.txt new file mode 100644 index 0000000000..095580bc4d --- /dev/null +++ b/charts/loki/templates/NOTES.txt @@ -0,0 +1,3 @@ +Verify the application is working by running these commands: + kubectl port-forward service/{{ include "loki.fullname" . }} {{ .Values.service.port }} + curl http://127.0.0.1:{{ .Values.service.port }}/api/prom/label diff --git a/charts/loki/templates/loki/_helpers.tpl b/charts/loki/templates/_helpers.tpl similarity index 73% rename from charts/loki/templates/loki/_helpers.tpl rename to charts/loki/templates/_helpers.tpl index c04c2950f9..9187b96add 100644 --- a/charts/loki/templates/loki/_helpers.tpl +++ b/charts/loki/templates/_helpers.tpl @@ -12,10 +12,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this If release name contains chart name it will be used as a full name. */}} {{- define "loki.fullname" -}} -{{- if .Values.loki.fullnameOverride -}} -{{- .Values.loki.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} -{{- $name := default .Chart.Name .Values.loki.nameOverride -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} {{- if contains $name .Release.Name -}} {{- .Release.Name | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -43,10 +43,10 @@ Create the name of the service account {{- end -}} {{- define "configSecret" }} -auth_enabled: {{ .Values.loki.config.auth_enabled }} +auth_enabled: {{ .Values.config.auth_enabled }} server: - http_listen_port: {{ .Values.loki.port }} + http_listen_port: {{ .Values.port }} limits_config: enforce_metric_name: false @@ -54,14 +54,14 @@ limits_config: ingester: lifecycler: ring: - store: {{ .Values.loki.config.ingester.lifecycler.ring.store }} - replication_factor: {{ .Values.loki.config.ingester.lifecycler.ring.replication_factor }} + store: {{ .Values.config.ingester.lifecycler.ring.store }} + replication_factor: {{ .Values.config.ingester.lifecycler.ring.replication_factor }} chunk_idle_period: 15m -{{- if .Values.loki.config.schema_configs }} +{{- if .Values.config.schema_configs }} schema_config: configs: -{{- range .Values.loki.config.schema_configs }} +{{- range .Values.config.schema_configs }} - from: {{ .from }} store: {{ .store }} object_store: {{ .object_store }} @@ -72,7 +72,7 @@ schema_config: {{- end -}} {{- end -}} -{{- with .Values.loki.config.storage_config }} +{{- with .Values.config.storage_config }} storage_config: {{ toYaml . | indent 2 }} {{- end }} diff --git a/charts/loki/templates/loki/deployment.yaml b/charts/loki/templates/deployment.yaml similarity index 52% rename from charts/loki/templates/loki/deployment.yaml rename to charts/loki/templates/deployment.yaml index e357708e13..c6fd7b67ea 100644 --- a/charts/loki/templates/loki/deployment.yaml +++ b/charts/loki/templates/deployment.yaml @@ -1,4 +1,3 @@ -{{- if .Values.loki.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -9,17 +8,17 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} annotations: - {{- toYaml .Values.loki.annotations | nindent 4 }} + {{- toYaml .Values.annotations | nindent 4 }} spec: - replicas: {{ .Values.loki.replicas }} - minReadySeconds: {{ .Values.loki.minReadySeconds }} + replicas: {{ .Values.replicas }} + minReadySeconds: {{ .Values.minReadySeconds }} selector: matchLabels: app: {{ template "loki.name" . }} release: {{ .Release.Name }} strategy: - type: {{ .Values.loki.deploymentStrategy }} - {{- if ne .Values.loki.deploymentStrategy "RollingUpdate" }} + type: {{ .Values.deploymentStrategy }} + {{- if ne .Values.deploymentStrategy "RollingUpdate" }} rollingUpdate: null {{- end }} template: @@ -29,21 +28,21 @@ spec: name: {{ template "loki.name" . }} release: {{ .Release.Name }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/loki/secret.yaml") . | sha256sum }} - {{- with .Values.loki.podAnnotations }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} spec: serviceAccountName: {{ template "loki.serviceAccountName" . }} - {{- if .Values.loki.priorityClassName }} - priorityClassName: {{ .Values.loki.priorityClassName }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} securityContext: - {{- toYaml .Values.loki.securityContext | nindent 8 }} + {{- toYaml .Values.securityContext | nindent 8 }} {{- end }} containers: - name: {{ .Chart.Name }} - image: "{{ .Values.loki.image.repository }}:{{ .Values.loki.image.tag }}" - imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "-config.file=/etc/loki/loki.yaml" volumeMounts: @@ -51,36 +50,35 @@ spec: mountPath: /etc/loki - name: storage mountPath: "/data" - subPath: {{ .Values.loki.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} ports: - name: http-metrics - containerPort: {{ .Values.loki.port }} + containerPort: {{ .Values.port }} protocol: TCP livenessProbe: - {{- toYaml .Values.loki.livenessProbe | nindent 12 }} + {{- toYaml .Values.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.loki.readinessProbe | nindent 12 }} + {{- toYaml .Values.readinessProbe | nindent 12 }} resources: - {{- toYaml .Values.loki.resources | nindent 12 }} + {{- toYaml .Values.resources | nindent 12 }} securityContext: readOnlyRootFilesystem: true nodeSelector: - {{- toYaml .Values.loki.nodeSelector | nindent 8 }} + {{- toYaml .Values.nodeSelector | nindent 8 }} affinity: - {{- toYaml .Values.loki.affinity | nindent 8 }} + {{- toYaml .Values.affinity | nindent 8 }} tolerations: - {{- toYaml .Values.loki.tolerations | nindent 8 }} - terminationGracePeriodSeconds: {{ .Values.loki.terminationGracePeriodSeconds }} + {{- toYaml .Values.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} volumes: - name: config secret: secretName: {{ template "loki.fullname" . }} - name: storage - {{- if .Values.loki.persistence.enabled }} + {{- if .Values.persistence.enabled }} persistentVolumeClaim: - claimName: {{ .Values.loki.persistence.existingClaim | default (include "loki.fullname" .) }} + claimName: {{ .Values.persistence.existingClaim | default (include "loki.fullname" .) }} {{- else }} emptyDir: {} {{- end }} -{{- end }} diff --git a/charts/loki/templates/loki/pvc.yaml b/charts/loki/templates/loki/pvc.yaml deleted file mode 100644 index f9041d551d..0000000000 --- a/charts/loki/templates/loki/pvc.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- if .Values.loki.enabled }} -{{- if and .Values.loki.persistence.enabled (not .Values.loki.persistence.existingClaim) }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ template "loki.fullname" . }} - labels: - app: {{ template "loki.name" . }} - chart: {{ template "loki.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - annotations: - {{- toYaml .Values.loki.persistence.annotations | nindent 4 }} -spec: - accessModes: - {{- range .Values.loki.persistence.accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.loki.persistence.size | quote }} - storageClassName: {{ .Values.loki.persistence.storageClassName }} -{{- end }} -{{- end }} diff --git a/charts/loki/templates/loki/networkpolicy.yaml b/charts/loki/templates/networkpolicy.yaml similarity index 93% rename from charts/loki/templates/loki/networkpolicy.yaml rename to charts/loki/templates/networkpolicy.yaml index 67281e71f9..5becbcdb59 100644 --- a/charts/loki/templates/loki/networkpolicy.yaml +++ b/charts/loki/templates/networkpolicy.yaml @@ -21,5 +21,5 @@ spec: app: {{ template "promtail.name" . }} release: {{ .Release.Name }} - ports: - - port: {{ .Values.loki.service.port }} + - port: {{ .Values.service.port }} {{- end -}} diff --git a/charts/loki/templates/loki/podsecuritypolicy.yaml b/charts/loki/templates/podsecuritypolicy.yaml similarity index 95% rename from charts/loki/templates/loki/podsecuritypolicy.yaml rename to charts/loki/templates/podsecuritypolicy.yaml index 02e1e8615d..9968b4844b 100644 --- a/charts/loki/templates/loki/podsecuritypolicy.yaml +++ b/charts/loki/templates/podsecuritypolicy.yaml @@ -1,4 +1,3 @@ -{{- if .Values.loki.enabled }} {{- if .Values.rbac.pspEnabled }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -38,4 +37,4 @@ spec: requiredDropCapabilities: - ALL {{- end }} -{{- end }} + diff --git a/charts/loki/templates/pvc.yaml b/charts/loki/templates/pvc.yaml new file mode 100644 index 0000000000..350c04b120 --- /dev/null +++ b/charts/loki/templates/pvc.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + {{- toYaml .Values.persistence.annotations | nindent 4 }} +spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + storageClassName: {{ .Values.persistence.storageClassName }} +{{- end }} diff --git a/charts/loki/templates/loki/role.yaml b/charts/loki/templates/role.yaml similarity index 92% rename from charts/loki/templates/loki/role.yaml rename to charts/loki/templates/role.yaml index 794bf3aac0..756c42b57e 100644 --- a/charts/loki/templates/loki/role.yaml +++ b/charts/loki/templates/role.yaml @@ -1,4 +1,3 @@ -{{- if .Values.loki.enabled }} {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -17,4 +16,4 @@ rules: resourceNames: [{{ template "loki.fullname" . }}] {{- end }} {{- end }} -{{- end }} + diff --git a/charts/loki/templates/loki/rolebinding.yaml b/charts/loki/templates/rolebinding.yaml similarity index 92% rename from charts/loki/templates/loki/rolebinding.yaml rename to charts/loki/templates/rolebinding.yaml index 734fc9aeb7..de1990b77b 100644 --- a/charts/loki/templates/loki/rolebinding.yaml +++ b/charts/loki/templates/rolebinding.yaml @@ -1,4 +1,3 @@ -{{- if .Values.loki.enabled }} {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -17,4 +16,4 @@ subjects: - kind: ServiceAccount name: {{ template "loki.serviceAccountName" . }} {{- end }} -{{- end }} + diff --git a/charts/loki/templates/loki/secret.yaml b/charts/loki/templates/secret.yaml similarity index 100% rename from charts/loki/templates/loki/secret.yaml rename to charts/loki/templates/secret.yaml diff --git a/charts/loki/templates/loki/service.yaml b/charts/loki/templates/service.yaml similarity index 71% rename from charts/loki/templates/loki/service.yaml rename to charts/loki/templates/service.yaml index 830c18d662..a72c4128f4 100644 --- a/charts/loki/templates/loki/service.yaml +++ b/charts/loki/templates/service.yaml @@ -1,4 +1,3 @@ -{{- if .Values.loki.enabled }} apiVersion: v1 kind: Service metadata: @@ -8,18 +7,18 @@ metadata: chart: {{ template "loki.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - {{- with .Values.loki.service.labels }} + {{- with .Values.service.labels }} {{- toYaml . | nindent 4 }} {{- end }} annotations: - {{- toYaml .Values.loki.service.annotations | nindent 4 }} + {{- toYaml .Values.service.annotations | nindent 4 }} spec: ports: - name: http-metrics - port: {{ .Values.loki.service.port }} + port: {{ .Values.service.port }} protocol: TCP targetPort: http-metrics selector: app: {{ template "loki.name" . }} release: {{ .Release.Name }} -{{- end }} + diff --git a/charts/loki/templates/loki/serviceaccount.yaml b/charts/loki/templates/serviceaccount.yaml similarity index 80% rename from charts/loki/templates/loki/serviceaccount.yaml rename to charts/loki/templates/serviceaccount.yaml index 80424aef31..f223f0a1c2 100644 --- a/charts/loki/templates/loki/serviceaccount.yaml +++ b/charts/loki/templates/serviceaccount.yaml @@ -1,4 +1,3 @@ -{{- if or (.Values.loki.enabled) (.Values.promtail.enabled) }} {{- if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount @@ -10,4 +9,4 @@ metadata: release: {{ .Release.Name }} name: {{ template "loki.serviceAccountName" . }} {{- end }} -{{- end }} + diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 94ec46f4a6..b31874069a 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -1,442 +1,127 @@ -rbac: - create: true - pspEnabled: true - +## Affinity for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} +# podAntiAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# - labelSelector: +# matchExpressions: +# - key: app +# operator: In +# values: +# - loki +# topologyKey: "kubernetes.io/hostname" + +## Deployment annotations +annotations: {} + +config: + auth_enabled: false + ingester: + lifecycler: + ring: + store: inmemory + replication_factor: 1 + schema_configs: + - from: 0 + store: boltdb + object_store: filesystem + schema: v9 + index: + prefix: index_ + period: 168h + storage_config: + boltdb: + directory: /data/loki/index + filesystem: + directory: /data/loki/chunks + +deploymentStrategy: RollingUpdate + +image: + repository: grafana/loki + tag: latest + pullPolicy: Always # Always pull while in BETA + +livenessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + +minReadySeconds: 0 + +## Enable persistence using Persistent Volume Claims networkPolicy: enabled: false -serviceAccount: - # if has deployed loki or promtail, must set false - create: true - name: - -loki: - enabled: true - - replicas: 1 - minReadySeconds: 0 - terminationGracePeriodSeconds: 30 - deploymentStrategy: RollingUpdate - port: 3100 - - image: - repository: grafana/loki - tag: latest - pullPolicy: Always # Always pull while in BETA - - service: - port: 3100 - annotations: {} - # prometheus.io/scrape: "true" - # prometheus.io/port: "http-metrics" - labels: {} - - readinessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 - - livenessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 - - resources: {} - # limits: - # cpu: 200m - # memory: 256Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: - fsGroup: 10001 - runAsGroup: 10001 - runAsNonRoot: true - runAsUser: 10001 +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} - ## Pod Annotations - podAnnotations: {} - - ## Deployment annotations +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## If you set enabled as "True", you need : +## - create a pv which above 10Gi and has same namespace with loki +## - keep storageClassName same with below setting +persistence: + enabled: false + accessModes: + - ReadWriteOnce + size: 10Gi + storageClassName: default annotations: {} + # subPath: "" + # existingClaim: - ## Assign a PriorityClassName to pods if set - # priorityClassName: - - ## Node labels for pod assignment - ## ref: https://kubernetes.io/docs/user-guide/node-selection/ - # - nodeSelector: {} - - ## Tolerations for pod assignment - ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - ## - tolerations: [] - - ## Affinity for pod assignment - ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity - ## - affinity: {} - # podAntiAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # - labelSelector: - # matchExpressions: - # - key: app - # operator: In - # values: - # - loki - # topologyKey: "kubernetes.io/hostname" +## Pod Annotations +podAnnotations: {} +# prometheus.io/scrape: "true" +# prometheus.io/port: "http-metrics" - ## Enable persistence using Persistent Volume Claims - ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ - ## If you set enabled as "True", you need : - ## - create a pv which above 10Gi and has same namespace with loki - ## - keep storageClassName same with below setting - persistence: - enabled: false - accessModes: - - ReadWriteOnce - size: 10Gi - storageClassName: default - annotations: {} - # subPath: "" - # existingClaim: +port: 3100 - config: - auth_enabled: false - ingester: - lifecycler: - ring: - store: inmemory - replication_factor: 1 - schema_configs: - - from: 0 - store: boltdb - object_store: filesystem - schema: v9 - index: - prefix: index_ - period: 168h - storage_config: - boltdb: - directory: /data/loki/index - filesystem: - directory: /data/loki/chunks +## Assign a PriorityClassName to pods if set +# priorityClassName: promtail: enabled: true - nameOverride: promtail - deploymentStrategy: RollingUpdate - - entryParser: docker - - image: - repository: grafana/promtail - tag: latest - pullPolicy: Always # Always pull while in BETA - - # This should match promtail.config.server.http_listen_port - port: 3101 - - # Extra volumes to scrape logs from - volumes: - - name: pods - hostPath: - path: /var/log/pods - - name: docker - hostPath: - path: /var/lib/docker/containers - - volumeMounts: - - name: pods - mountPath: /var/log/pods - readOnly: true - - name: docker - mountPath: /var/lib/docker/containers - readOnly: true - - readinessProbe: - failureThreshold: 5 - httpGet: - path: /metrics - port: http-metrics - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - - livenessProbe: - failureThreshold: 5 - httpGet: - path: /metrics - port: http-metrics - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - - resources: {} - # limits: - # cpu: 200m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: - readOnlyRootFilesystem: true - runAsGroup: 0 - runAsUser: 0 - - ## Pod Annotations - podAnnotations: {} - # prometheus.io/scrape: "true" - # prometheus.io/port: "http-metrics" +rbac: + create: true + pspEnabled: true - ## Deployment annotations +readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + +replicas: 1 + +resources: {} +# limits: +# cpu: 200m +# memory: 256Mi +# requests: +# cpu: 100m +# memory: 128Mi + +securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + +service: + port: 3100 annotations: {} + labels: {} - ## Assign a PriorityClassName to pods if set - # priorityClassName: - - ## Node labels for pod assignment - ## ref: https://kubernetes.io/docs/user-guide/node-selection/ - # - nodeSelector: {} - - ## Tolerations for pod assignment - ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - ## - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - ## Affinity for pod assignment - ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity - ## - affinity: {} - - config: - client: - # Maximum wait period before sending batch - batchwait: 1s - # Maximum batch size to accrue before sending, unit is byte - batchsize: 102400 - - # Maximum time to wait for server to respond to a request - timeout: 10s +serviceAccount: + create: true + name: - backoff_config: - # Initial backoff time between retries - minbackoff: 100ms - # Maximum backoff time between retries - maxbackoff: 5s - # Maximum number of retires when sending batches, 0 means infinite retries - maxretries: 5 +terminationGracePeriodSeconds: 30 - # The labels to add to any time series or alerts when communicating with loki - external_labels: {} +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] - server: - http_listen_port: 3101 - positions: - filename: /run/promtail/positions.yaml - scrape_configs: - - entry_parser: '{{ .Values.promtail.entryParser }}' - job_name: kubernetes-pods-name - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: - - __meta_kubernetes_pod_label_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - entry_parser: '{{ .Values.promtail.entryParser }}' - job_name: kubernetes-pods-app - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - source_labels: - - __meta_kubernetes_pod_label_name - - source_labels: - - __meta_kubernetes_pod_label_app - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - entry_parser: '{{ .Values.promtail.entryParser }}' - job_name: kubernetes-pods-direct-controllers - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - separator: '' - source_labels: - - __meta_kubernetes_pod_label_name - - __meta_kubernetes_pod_label_app - - action: drop - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - - source_labels: - - __meta_kubernetes_pod_controller_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - entry_parser: '{{ .Values.promtail.entryParser }}' - job_name: kubernetes-pods-indirect-controller - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - separator: '' - source_labels: - - __meta_kubernetes_pod_label_name - - __meta_kubernetes_pod_label_app - - action: keep - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - - action: replace - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml new file mode 100644 index 0000000000..94f7963e14 --- /dev/null +++ b/charts/promtail/Chart.yaml @@ -0,0 +1,13 @@ +name: promtail +version: 0.6.0 +appVersion: 0.0.1 +kubeVersion: "^1.10.0-0" +description: "Responsible for gathering logs and sending them to Loki" +home: https://grafana.com/loki +icon: https://github.com/grafana/loki/raw/master/docs/logo.png +sources: +- https://github.com/grafana/loki +maintainers: +- name: Loki Maintainers + email: lokiproject@googlegroups.com +engine: gotpl diff --git a/charts/promtail/templates/NOTES.txt b/charts/promtail/templates/NOTES.txt new file mode 100644 index 0000000000..0f46731b6f --- /dev/null +++ b/charts/promtail/templates/NOTES.txt @@ -0,0 +1,3 @@ +Verify the application is working by running these commands: + kubectl port-forward daemonset/{{ include "promtail.fullname" . }} {{ .Values.port }} + curl http://127.0.0.1:{{ .Values.port }}/metrics diff --git a/charts/loki/templates/promtail/_helpers.tpl b/charts/promtail/templates/_helpers.tpl similarity index 64% rename from charts/loki/templates/promtail/_helpers.tpl rename to charts/promtail/templates/_helpers.tpl index 92f1d483ba..43a1c27542 100644 --- a/charts/loki/templates/promtail/_helpers.tpl +++ b/charts/promtail/templates/_helpers.tpl @@ -3,7 +3,7 @@ Expand the name of the chart. */}} {{- define "promtail.name" -}} -{{- default .Chart.Name .Values.promtail.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* @@ -12,10 +12,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this If release name contains chart name it will be used as a full name. */}} {{- define "promtail.fullname" -}} -{{- if .Values.promtail.fullnameOverride -}} -{{- .Values.promtail.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} -{{- $name := default .Chart.Name .Values.promtail.nameOverride -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} {{- if contains $name .Release.Name -}} {{- .Release.Name | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -36,8 +36,17 @@ Create the name of the service account */}} {{- define "promtail.serviceAccountName" -}} {{- if .Values.serviceAccount.create -}} - {{ default (include "loki.fullname" .) .Values.serviceAccount.name }} + {{ default (include "promtail.fullname" .) .Values.serviceAccount.name }} {{- else -}} {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +The service name to connect to Loki. Defaults to release-loki if not set +*/}} +{{- define "loki.serviceName" -}} +{{- $service := .Values.loki.serviceName | default (printf "%s-loki" .Release.Name) -}} +{{- $service | trunc 63 | trimSuffix "-" -}} +{{- end -}} + diff --git a/charts/loki/templates/promtail/clusterrole.yaml b/charts/promtail/templates/clusterrole.yaml similarity index 80% rename from charts/loki/templates/promtail/clusterrole.yaml rename to charts/promtail/templates/clusterrole.yaml index b9d3709013..b872041b7a 100644 --- a/charts/loki/templates/promtail/clusterrole.yaml +++ b/charts/promtail/templates/clusterrole.yaml @@ -1,4 +1,3 @@ -{{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -8,8 +7,6 @@ metadata: chart: {{ template "promtail.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - annotations: - {{- toYaml .Values.promtail.annotations | nindent 4 }} name: {{ template "promtail.fullname" . }}-clusterrole rules: - apiGroups: [""] # "" indicates the core API group @@ -21,4 +18,3 @@ rules: - pods verbs: ["get", "watch", "list"] {{- end }} -{{- end }} diff --git a/charts/loki/templates/promtail/clusterrolebinding.yaml b/charts/promtail/templates/clusterrolebinding.yaml similarity index 83% rename from charts/loki/templates/promtail/clusterrolebinding.yaml rename to charts/promtail/templates/clusterrolebinding.yaml index a6d9984325..2b73945722 100644 --- a/charts/loki/templates/promtail/clusterrolebinding.yaml +++ b/charts/promtail/templates/clusterrolebinding.yaml @@ -1,4 +1,3 @@ -{{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -9,8 +8,6 @@ metadata: chart: {{ template "promtail.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - annotations: - {{- toYaml .Values.promtail.annotations | nindent 4 }} subjects: - kind: ServiceAccount name: {{ template "promtail.serviceAccountName" . }} @@ -20,4 +17,3 @@ roleRef: name: {{ template "promtail.fullname" . }}-clusterrole apiGroup: rbac.authorization.k8s.io {{- end }} -{{- end }} diff --git a/charts/loki/templates/promtail/configmap.yaml b/charts/promtail/templates/configmap.yaml similarity index 71% rename from charts/loki/templates/promtail/configmap.yaml rename to charts/promtail/templates/configmap.yaml index 25fdc366b6..3072048aa5 100644 --- a/charts/loki/templates/promtail/configmap.yaml +++ b/charts/promtail/templates/configmap.yaml @@ -1,4 +1,3 @@ -{{- if .Values.promtail.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -10,5 +9,4 @@ metadata: heritage: {{ .Release.Service }} data: promtail.yaml: | - {{- tpl (toYaml .Values.promtail.config | nindent 4 ) . }} -{{- end }} + {{- tpl (toYaml .Values.config | nindent 4 ) . }} diff --git a/charts/loki/templates/promtail/daemonset.yaml b/charts/promtail/templates/daemonset.yaml similarity index 53% rename from charts/loki/templates/promtail/daemonset.yaml rename to charts/promtail/templates/daemonset.yaml index ca88143355..4a04c713fc 100644 --- a/charts/loki/templates/promtail/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -1,4 +1,3 @@ -{{- if .Values.promtail.enabled }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -9,15 +8,15 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} annotations: - {{- toYaml .Values.promtail.annotations | nindent 4 }} + {{- toYaml .Values.annotations | nindent 4 }} spec: selector: matchLabels: app: {{ template "promtail.name" . }} release: {{ .Release.Name }} updateStrategy: - type: {{ .Values.promtail.deploymentStrategy }} - {{- if ne .Values.promtail.deploymentStrategy "RollingUpdate" }} + type: {{ .Values.deploymentStrategy }} + {{- if ne .Values.deploymentStrategy "RollingUpdate" }} rollingUpdate: null {{- end }} template: @@ -26,25 +25,25 @@ spec: app: {{ template "promtail.name" . }} release: {{ .Release.Name }} annotations: - {{ toYaml .Values.promtail.podAnnotations | nindent 8 }} + {{ toYaml .Values.podAnnotations | nindent 8 }} spec: serviceAccountName: {{ template "promtail.serviceAccountName" . }} - {{- if .Values.promtail.priorityClassName }} - priorityClassName: {{ .Values.promtail.priorityClassName }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} {{- end }} containers: - name: promtail - image: "{{ .Values.promtail.image.repository }}:{{ .Values.promtail.image.tag }}" - imagePullPolicy: {{ .Values.promtail.image.pullPolicy }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "-config.file=/etc/promtail/promtail.yaml" - - "-client.url=http://{{ template "loki.fullname" . }}:{{ .Values.loki.service.port }}/api/prom/push" + - "-client.url=http://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/api/prom/push" volumeMounts: - name: config mountPath: /etc/promtail - name: run mountPath: /run/promtail - {{- with .Values.promtail.volumeMounts }} + {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} env: @@ -53,26 +52,26 @@ spec: fieldRef: fieldPath: spec.nodeName ports: - - containerPort: {{ .Values.promtail.port }} + - containerPort: {{ .Values.port }} name: http-metrics securityContext: - {{- toYaml .Values.promtail.securityContext | nindent 12 }} - {{- if .Values.promtail.livenessProbe }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.livenessProbe }} livenessProbe: - {{- toYaml .Values.promtail.livenessProbe | nindent 12 }} + {{- toYaml .Values.livenessProbe | nindent 12 }} {{- end }} - {{- if .Values.promtail.readinessProbe }} + {{- if .Values.readinessProbe }} readinessProbe: - {{- toYaml .Values.promtail.readinessProbe | nindent 12 }} + {{- toYaml .Values.readinessProbe | nindent 12 }} {{- end }} resources: - {{- toYaml .Values.promtail.resources | nindent 12 }} + {{- toYaml .Values.resources | nindent 12 }} nodeSelector: - {{- toYaml .Values.promtail.nodeSelector | nindent 8 }} + {{- toYaml .Values.nodeSelector | nindent 8 }} affinity: - {{- toYaml .Values.promtail.affinity | nindent 8 }} + {{- toYaml .Values.affinity | nindent 8 }} tolerations: - {{- toYaml .Values.promtail.tolerations | nindent 8 }} + {{- toYaml .Values.tolerations | nindent 8 }} volumes: - name: config configMap: @@ -80,8 +79,7 @@ spec: - name: run hostPath: path: /run/promtail - {{- with .Values.promtail.volumes }} + {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} -{{- end }} diff --git a/charts/loki/templates/promtail/podsecuritypolicy.yaml b/charts/promtail/templates/podsecuritypolicy.yaml similarity index 93% rename from charts/loki/templates/promtail/podsecuritypolicy.yaml rename to charts/promtail/templates/podsecuritypolicy.yaml index 96d761caa8..a38595eeef 100644 --- a/charts/loki/templates/promtail/podsecuritypolicy.yaml +++ b/charts/promtail/templates/podsecuritypolicy.yaml @@ -1,4 +1,3 @@ -{{- if .Values.promtail.enabled }} {{- if .Values.rbac.pspEnabled }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -31,4 +30,3 @@ spec: requiredDropCapabilities: - ALL {{- end }} -{{- end }} diff --git a/charts/loki/templates/promtail/role.yaml b/charts/promtail/templates/role.yaml similarity index 91% rename from charts/loki/templates/promtail/role.yaml rename to charts/promtail/templates/role.yaml index 96ead979c7..97969e1a7a 100644 --- a/charts/loki/templates/promtail/role.yaml +++ b/charts/promtail/templates/role.yaml @@ -1,4 +1,3 @@ -{{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -17,4 +16,3 @@ rules: resourceNames: [{{ template "promtail.fullname" . }}] {{- end }} {{- end }} -{{- end }} diff --git a/charts/loki/templates/promtail/rolebinding.yaml b/charts/promtail/templates/rolebinding.yaml similarity index 91% rename from charts/loki/templates/promtail/rolebinding.yaml rename to charts/promtail/templates/rolebinding.yaml index 9f77e754c1..ff99f2a923 100644 --- a/charts/loki/templates/promtail/rolebinding.yaml +++ b/charts/promtail/templates/rolebinding.yaml @@ -1,4 +1,3 @@ -{{- if .Values.promtail.enabled }} {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -17,4 +16,4 @@ subjects: - kind: ServiceAccount name: {{ template "promtail.serviceAccountName" . }} {{- end }} -{{- end }} + diff --git a/charts/promtail/templates/serviceaccount.yaml b/charts/promtail/templates/serviceaccount.yaml new file mode 100644 index 0000000000..8165e08dff --- /dev/null +++ b/charts/promtail/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "promtail.name" . }} + chart: {{ template "promtail.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "promtail.serviceAccountName" . }} +{{- end }} + diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml new file mode 100644 index 0000000000..8d82183d01 --- /dev/null +++ b/charts/promtail/values.yaml @@ -0,0 +1,313 @@ +## Affinity for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} + +annotations: {} + +deploymentStrategy: RollingUpdate + +entryParser: docker + +image: + repository: grafana/promtail + tag: latest + pullPolicy: Always # Always pull while in BETA + +livenessProbe: + failureThreshold: 5 + httpGet: + path: /metrics + port: http-metrics + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +loki: + serviceName: "" # Defaults to "${RELEASE}-loki" if not set + servicePort: 3100 + +nameOverride: promtail + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +podAnnotations: {} +# prometheus.io/scrape: "true" +# prometheus.io/port: "http-metrics" + +# This should match config.server.http_listen_port +port: 3101 + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +rbac: + create: true + pspEnabled: true + +readinessProbe: + failureThreshold: 5 + httpGet: + path: /metrics + port: http-metrics + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +resources: {} +# limits: +# cpu: 200m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +securityContext: + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsUser: 0 + +serviceAccount: + create: true + name: + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: +- key: node-role.kubernetes.io/master + effect: NoSchedule + +# Extra volumes to scrape logs from +volumes: +- name: pods + hostPath: + path: /var/log/pods + +volumeMounts: +- name: pods + mountPath: /var/log/pods + readOnly: true + +config: + client: + # Maximum wait period before sending batch + batchwait: 1s + # Maximum batch size to accrue before sending, unit is byte + batchsize: 102400 + + # Maximum time to wait for server to respond to a request + timeout: 10s + + backoff_config: + # Initial backoff time between retries + minbackoff: 100ms + # Maximum backoff time between retries + maxbackoff: 5s + # Maximum number of retires when sending batches, 0 means infinite retries + maxretries: 5 + + # The labels to add to any time series or alerts when communicating with loki + external_labels: {} + + server: + http_listen_port: 3101 + positions: + filename: /run/promtail/positions.yaml + scrape_configs: + - entry_parser: '{{ .Values.entryParser }}' + job_name: kubernetes-pods-name + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - entry_parser: '{{ .Values.entryParser }}' + job_name: kubernetes-pods-app + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + source_labels: + - __meta_kubernetes_pod_label_name + - source_labels: + - __meta_kubernetes_pod_label_app + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - entry_parser: '{{ .Values.entryParser }}' + job_name: kubernetes-pods-direct-controllers + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: drop + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - entry_parser: '{{ .Values.entryParser }}' + job_name: kubernetes-pods-indirect-controller + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: keep + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - action: replace + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ From eb80f2c8baa4bf738404a6c9828cc549d0844d40 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Mon, 8 Apr 2019 13:04:04 -0500 Subject: [PATCH 050/202] Add loki-stack helm chart Signed-off-by: Steven Sheehy --- charts/{loki => }/README.md | 6 ++--- charts/loki-stack/.helmignore | 22 +++++++++++++++++++ charts/loki-stack/Chart.yaml | 13 +++++++++++ charts/{loki => loki-stack}/requirements.yaml | 4 ++++ charts/loki-stack/templates/NOTES.txt | 3 +++ charts/loki-stack/values.yaml | 6 +++++ charts/loki/.helmignore | 22 +++++++++++++++++++ charts/loki/grafana.yaml | 5 ----- charts/loki/values.yaml | 3 --- charts/promtail/.helmignore | 22 +++++++++++++++++++ charts/promtail/templates/_helpers.tpl | 16 +++++++++++--- 11 files changed, 108 insertions(+), 14 deletions(-) rename charts/{loki => }/README.md (85%) create mode 100644 charts/loki-stack/.helmignore create mode 100644 charts/loki-stack/Chart.yaml rename charts/{loki => loki-stack}/requirements.yaml (56%) create mode 100644 charts/loki-stack/templates/NOTES.txt create mode 100644 charts/loki-stack/values.yaml create mode 100644 charts/loki/.helmignore delete mode 100644 charts/loki/grafana.yaml create mode 100644 charts/promtail/.helmignore diff --git a/charts/loki/README.md b/charts/README.md similarity index 85% rename from charts/loki/README.md rename to charts/README.md index 9d19a306a4..3a851e4055 100644 --- a/charts/loki/README.md +++ b/charts/README.md @@ -13,13 +13,13 @@ $ helm repo add loki https://grafana.github.io/loki/charts ## Deploy Loki and Promtail to your cluster ```bash -$ helm upgrade --install loki loki/loki +$ helm upgrade --install loki loki/loki-stack ``` ## Deploy Loki only ```bash -$ helm upgrade --install loki loki/loki --set 'promtail.enabled=false' +$ helm upgrade --install loki loki/loki --set "loki.serviceName=my-loki" ``` ## Deploy Promtail only @@ -33,7 +33,7 @@ $ helm upgrade --install promtail loki/promtail To install Grafana on your cluster with helm, use the following command: ```bash -$ helm install stable/grafana -n loki-grafana -f grafana.yaml --namespace +$ helm install stable/grafana -n loki-grafana ``` To get the admin password for the Grafana pod, run the following command: diff --git a/charts/loki-stack/.helmignore b/charts/loki-stack/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/charts/loki-stack/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml new file mode 100644 index 0000000000..0c9c3790f2 --- /dev/null +++ b/charts/loki-stack/Chart.yaml @@ -0,0 +1,13 @@ +name: loki-stack +version: 0.6.0 +appVersion: 0.0.1 +kubeVersion: "^1.10.0-0" +description: "Loki: like Prometheus, but for logs." +home: https://grafana.com/loki +icon: https://github.com/grafana/loki/raw/master/docs/logo.png +sources: +- https://github.com/grafana/loki +maintainers: +- name: Loki Maintainers + email: lokiproject@googlegroups.com +engine: gotpl diff --git a/charts/loki/requirements.yaml b/charts/loki-stack/requirements.yaml similarity index 56% rename from charts/loki/requirements.yaml rename to charts/loki-stack/requirements.yaml index 426a6970a7..3587624cb6 100644 --- a/charts/loki/requirements.yaml +++ b/charts/loki-stack/requirements.yaml @@ -1,4 +1,8 @@ dependencies: +- name: "loki" + condition: loki.enabled + repository: "file://../loki" + version: "^0.6.0" - name: "promtail" condition: promtail.enabled repository: "file://../promtail" diff --git a/charts/loki-stack/templates/NOTES.txt b/charts/loki-stack/templates/NOTES.txt new file mode 100644 index 0000000000..d9cdccbeed --- /dev/null +++ b/charts/loki-stack/templates/NOTES.txt @@ -0,0 +1,3 @@ +The Loki stack has been deployed to your cluster. Loki can now be added as a datasource in Grafana. + +See http://docs.grafana.org/features/datasources/loki/ for more detail. diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml new file mode 100644 index 0000000000..06b64946a2 --- /dev/null +++ b/charts/loki-stack/values.yaml @@ -0,0 +1,6 @@ +loki: + enabled: true + +promtail: + enabled: true + diff --git a/charts/loki/.helmignore b/charts/loki/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/charts/loki/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/loki/grafana.yaml b/charts/loki/grafana.yaml deleted file mode 100644 index a80753d353..0000000000 --- a/charts/loki/grafana.yaml +++ /dev/null @@ -1,5 +0,0 @@ -image: - tag: latest - -env: - GF_EXPLORE_ENABLED: true \ No newline at end of file diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index b31874069a..801d4e9594 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -81,9 +81,6 @@ port: 3100 ## Assign a PriorityClassName to pods if set # priorityClassName: -promtail: - enabled: true - rbac: create: true pspEnabled: true diff --git a/charts/promtail/.helmignore b/charts/promtail/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/charts/promtail/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/promtail/templates/_helpers.tpl b/charts/promtail/templates/_helpers.tpl index 43a1c27542..c270b0b281 100644 --- a/charts/promtail/templates/_helpers.tpl +++ b/charts/promtail/templates/_helpers.tpl @@ -43,10 +43,20 @@ Create the name of the service account {{- end -}} {{/* -The service name to connect to Loki. Defaults to release-loki if not set +The service name to connect to Loki. Defaults to the same logic as "loki.fullname" */}} {{- define "loki.serviceName" -}} -{{- $service := .Values.loki.serviceName | default (printf "%s-loki" .Release.Name) -}} -{{- $service | trunc 63 | trimSuffix "-" -}} +{{- if .Values.loki.serviceName -}} +{{- .Values.loki.serviceName -}} +{{- else if .Values.loki.fullnameOverride -}} +{{- .Values.loki.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "loki" .Values.loki.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} {{- end -}} From 7190bc486826980de57ff36571a3e00101a0f1da Mon Sep 17 00:00:00 2001 From: Sverre Boschman Date: Mon, 1 Apr 2019 14:41:06 +0200 Subject: [PATCH 051/202] include all lifecycler ring config --- charts/loki/templates/_helpers.tpl | 3 +-- charts/loki/values.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/charts/loki/templates/_helpers.tpl b/charts/loki/templates/_helpers.tpl index 9187b96add..22a3f2711c 100644 --- a/charts/loki/templates/_helpers.tpl +++ b/charts/loki/templates/_helpers.tpl @@ -54,8 +54,7 @@ limits_config: ingester: lifecycler: ring: - store: {{ .Values.config.ingester.lifecycler.ring.store }} - replication_factor: {{ .Values.config.ingester.lifecycler.ring.replication_factor }} +{{ toYaml .Values.config.ingester.lifecycler.ring | indent 6 }} chunk_idle_period: 15m {{- if .Values.config.schema_configs }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 801d4e9594..28b210f8f0 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -21,6 +21,16 @@ config: ring: store: inmemory replication_factor: 1 + + ## Different ring configs can be used. E.g. Consul + # ring: + # store: consul + # replication_factor: 1 + # consul: + # host: "consul:8500" + # prefix: "" + # httpclienttimeout: "20s" + # consistentreads: true schema_configs: - from: 0 store: boltdb From 3cb048af1807105f7ec9de93b32008cf1749f88a Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Wed, 10 Apr 2019 10:42:45 -0500 Subject: [PATCH 052/202] Fix missing volume after refactor Signed-off-by: Steven Sheehy --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 0c9c3790f2..14761a85eb 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.6.0 +version: 0.6.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 94f7963e14..1554c7e4ea 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.6.0 +version: 0.6.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 8d82183d01..e9f1539b79 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -82,11 +82,17 @@ tolerations: # Extra volumes to scrape logs from volumes: +- name: docker + hostPath: + path: /var/lib/docker/containers - name: pods hostPath: path: /var/log/pods volumeMounts: +- name: docker + mountPath: /var/lib/docker/containers + readOnly: true - name: pods mountPath: /var/log/pods readOnly: true From e24836c4c6c1e0c7e75004612e55427deb4e74ef Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 11 Apr 2019 09:56:30 +0800 Subject: [PATCH 053/202] Update helm docs Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/README.md b/charts/README.md index 3a851e4055..3c6f81e930 100644 --- a/charts/README.md +++ b/charts/README.md @@ -10,6 +10,12 @@ Loki's chart repository to Helm: $ helm repo add loki https://grafana.github.io/loki/charts ``` +If you have added it, need to update in case miss any sub-repo: + +```bash +$ helm repo update +``` + ## Deploy Loki and Promtail to your cluster ```bash From 5e0d934a4768c0e4e0f1456d6ae051236b502295 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 11 Apr 2019 19:35:21 +0800 Subject: [PATCH 054/202] Remove useless config option port need match http_listen_port, keep this option only increase possibility of wrong configuration, remove it. Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/promtail/templates/NOTES.txt | 4 ++-- charts/promtail/templates/daemonset.yaml | 2 +- charts/promtail/values.yaml | 3 --- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/charts/promtail/templates/NOTES.txt b/charts/promtail/templates/NOTES.txt index 0f46731b6f..e31aa8616c 100644 --- a/charts/promtail/templates/NOTES.txt +++ b/charts/promtail/templates/NOTES.txt @@ -1,3 +1,3 @@ Verify the application is working by running these commands: - kubectl port-forward daemonset/{{ include "promtail.fullname" . }} {{ .Values.port }} - curl http://127.0.0.1:{{ .Values.port }}/metrics + kubectl port-forward daemonset/{{ include "promtail.fullname" . }} {{ .Values.config.server.http_listen_port }} + curl http://127.0.0.1:{{ .Values.config.server.http_listen_port }}/metrics diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 4a04c713fc..d008d0803b 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -52,7 +52,7 @@ spec: fieldRef: fieldPath: spec.nodeName ports: - - containerPort: {{ .Values.port }} + - containerPort: {{ .Values.config.server.http_listen_port }} name: http-metrics securityContext: {{- toYaml .Values.securityContext | nindent 12 }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index e9f1539b79..6bfff18f2a 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -37,9 +37,6 @@ podAnnotations: {} # prometheus.io/scrape: "true" # prometheus.io/port: "http-metrics" -# This should match config.server.http_listen_port -port: 3101 - ## Assign a PriorityClassName to pods if set # priorityClassName: From d3e45c9f61574efc6be9f7889e0fbb2515d26ae9 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Sat, 6 Apr 2019 01:30:25 -0500 Subject: [PATCH 055/202] Add a /ready endpoint to promtail Signed-off-by: Steven Sheehy --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 14761a85eb..b824c0ad52 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.6.1 +version: 0.6.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 1554c7e4ea..bc60af652d 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.6.1 +version: 0.6.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index e9f1539b79..54ac463f91 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -16,7 +16,7 @@ image: livenessProbe: failureThreshold: 5 httpGet: - path: /metrics + path: /ready port: http-metrics initialDelaySeconds: 10 periodSeconds: 10 @@ -50,7 +50,7 @@ rbac: readinessProbe: failureThreshold: 5 httpGet: - path: /metrics + path: /ready port: http-metrics initialDelaySeconds: 10 periodSeconds: 10 From 5f44082181ec59cd930b7809f32e976d5e1df522 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Thu, 11 Apr 2019 13:56:53 -0500 Subject: [PATCH 056/202] Remove liveness probes from promtail Signed-off-by: Steven Sheehy --- charts/promtail/values.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 54ac463f91..e6cb8f620f 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -13,15 +13,7 @@ image: tag: latest pullPolicy: Always # Always pull while in BETA -livenessProbe: - failureThreshold: 5 - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 +livenessProbe: {} loki: serviceName: "" # Defaults to "${RELEASE}-loki" if not set From 2229932b692f4912c30c655bbe8c19718afc6b9b Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Thu, 11 Apr 2019 15:11:28 -0500 Subject: [PATCH 057/202] Helm: Allow custom loki config Signed-off-by: Steven Sheehy --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/_helpers.tpl | 35 --------------------------- charts/loki/templates/deployment.yaml | 2 +- charts/loki/templates/secret.yaml | 2 +- charts/loki/values.yaml | 24 ++++++++++-------- 6 files changed, 18 insertions(+), 49 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index b824c0ad52..b0c5c29176 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.6.2 +version: 0.7.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index a6b3fbb8ae..e6cc11d368 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.6.0 +version: 0.7.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/_helpers.tpl b/charts/loki/templates/_helpers.tpl index 22a3f2711c..608753377e 100644 --- a/charts/loki/templates/_helpers.tpl +++ b/charts/loki/templates/_helpers.tpl @@ -42,38 +42,3 @@ Create the name of the service account {{- end -}} {{- end -}} -{{- define "configSecret" }} -auth_enabled: {{ .Values.config.auth_enabled }} - -server: - http_listen_port: {{ .Values.port }} - -limits_config: - enforce_metric_name: false - -ingester: - lifecycler: - ring: -{{ toYaml .Values.config.ingester.lifecycler.ring | indent 6 }} - chunk_idle_period: 15m - -{{- if .Values.config.schema_configs }} -schema_config: - configs: -{{- range .Values.config.schema_configs }} - - from: {{ .from }} - store: {{ .store }} - object_store: {{ .object_store }} - schema: {{ .schema }} - index: - prefix: {{ .index.prefix }} - period: {{ .index.period }} -{{- end -}} -{{- end -}} - -{{- with .Values.config.storage_config }} -storage_config: -{{ toYaml . | indent 2 }} -{{- end }} - -{{- end}} diff --git a/charts/loki/templates/deployment.yaml b/charts/loki/templates/deployment.yaml index c6fd7b67ea..915cb17545 100644 --- a/charts/loki/templates/deployment.yaml +++ b/charts/loki/templates/deployment.yaml @@ -53,7 +53,7 @@ spec: subPath: {{ .Values.persistence.subPath }} ports: - name: http-metrics - containerPort: {{ .Values.port }} + containerPort: {{ .Values.config.server.http_listen_port }} protocol: TCP livenessProbe: {{- toYaml .Values.livenessProbe | nindent 12 }} diff --git a/charts/loki/templates/secret.yaml b/charts/loki/templates/secret.yaml index 66819067e5..95b16fad4b 100644 --- a/charts/loki/templates/secret.yaml +++ b/charts/loki/templates/secret.yaml @@ -8,4 +8,4 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - loki.yaml: {{ include "configSecret" . | b64enc}} \ No newline at end of file + loki.yaml: {{ tpl (toYaml .Values.config) . | b64enc}} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 28b210f8f0..43a09f28dd 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -17,6 +17,7 @@ annotations: {} config: auth_enabled: false ingester: + chunk_idle_period: 15m lifecycler: ring: store: inmemory @@ -31,14 +32,19 @@ config: # prefix: "" # httpclienttimeout: "20s" # consistentreads: true - schema_configs: - - from: 0 - store: boltdb - object_store: filesystem - schema: v9 - index: - prefix: index_ - period: 168h + limits_config: + enforce_metric_name: false + schema_config: + configs: + - from: 0 + store: boltdb + object_store: filesystem + schema: v9 + index: + prefix: index_ + period: 168h + server: + http_listen_port: 3100 storage_config: boltdb: directory: /data/loki/index @@ -86,8 +92,6 @@ podAnnotations: {} # prometheus.io/scrape: "true" # prometheus.io/port: "http-metrics" -port: 3100 - ## Assign a PriorityClassName to pods if set # priorityClassName: From cd7efe1e3ef859f7b176324fdb4e9ad2c7918af9 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 12 Apr 2019 09:54:38 +0800 Subject: [PATCH 058/202] update expression Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index 3c6f81e930..833a1325ee 100644 --- a/charts/README.md +++ b/charts/README.md @@ -10,7 +10,7 @@ Loki's chart repository to Helm: $ helm repo add loki https://grafana.github.io/loki/charts ``` -If you have added it, need to update in case miss any sub-repo: +You can update the chart repository by running: ```bash $ helm repo update From 4ecbdde33d6b6730dc461c6e149a828283fba083 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 12 Apr 2019 09:56:21 +0800 Subject: [PATCH 059/202] Update chart version Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 14761a85eb..b824c0ad52 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.6.1 +version: 0.6.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 1554c7e4ea..8f3ea952a3 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.6.1 +version: 0.6.3 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" From d914c20495166e43b6c1e58918ff5ba83c845eb3 Mon Sep 17 00:00:00 2001 From: Juan Manuel Date: Fri, 12 Apr 2019 09:10:58 +0200 Subject: [PATCH 060/202] Added option that loki service run as nodeport --- charts/loki/templates/service.yaml | 12 +++++++++--- charts/loki/values.yaml | 13 ++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/charts/loki/templates/service.yaml b/charts/loki/templates/service.yaml index a72c4128f4..9a8c1da585 100644 --- a/charts/loki/templates/service.yaml +++ b/charts/loki/templates/service.yaml @@ -13,12 +13,18 @@ metadata: annotations: {{- toYaml .Values.service.annotations | nindent 4 }} spec: + type: {{ .Values.service.type }} +{{- if (and (eq .Values.service.type "ClusterIP") (not (empty .Values.service.clusterIP))) }} + clusterIP: {{ .Values.service.clusterIP }} +{{- end }} ports: - - name: http-metrics - port: {{ .Values.service.port }} + - port: {{ .Values.service.port }} protocol: TCP + name: http-metrics targetPort: http-metrics +{{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} +{{- end }} selector: app: {{ template "loki.name" . }} release: {{ .Release.Name }} - diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 801d4e9594..6f8e25a77a 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -21,6 +21,16 @@ config: ring: store: inmemory replication_factor: 1 + + ## Different ring configs can be used. E.g. Consul + # ring: + # store: consul + # replication_factor: 1 + # consul: + # host: "consul:8500" + # prefix: "" + # httpclienttimeout: "20s" + # consistentreads: true schema_configs: - from: 0 store: boltdb @@ -108,6 +118,8 @@ securityContext: runAsUser: 10001 service: + type: ClusterIP + nodePort: port: 3100 annotations: {} labels: {} @@ -121,4 +133,3 @@ terminationGracePeriodSeconds: 30 ## Tolerations for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ tolerations: [] - From 1159f5ce88970fa1de52d98f0b08cb3f33063bd7 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 17 Apr 2019 15:49:42 +0200 Subject: [PATCH 061/202] Bumped chart versions --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index b824c0ad52..2080257105 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.6.2 +version: 0.7.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 8f3ea952a3..97c5129a87 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.6.3 +version: 0.7.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" From 2baf4ac4a6e91417694390549b0721d589771fdc Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 18 Apr 2019 14:08:00 +0800 Subject: [PATCH 062/202] Comment how to contribute helm chart Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/charts/README.md b/charts/README.md index 833a1325ee..a039405009 100644 --- a/charts/README.md +++ b/charts/README.md @@ -56,3 +56,15 @@ $ kubectl port-forward --namespace service/loki-grafana 3000:80 Navigate to http://localhost:3000 and login with `admin` and the password output above. Then follow the [instructions for adding the loki datasource](/docs/usage.md), using the URL `http://loki:3100/`. + +## How to contribute + +If you want to add any feature to helm chart, you can follow as below: + +```bash +$ cd production/helm +$ # do some changes to loki/promtail in corresponding directory +$ helm dependency update loki-stack +$ helm install ./loki-stack --dry-run --debug # to see changes format as expected +$ helm install ./loki-stack # to see changes work as expected +``` From 1c90baaf09bd267bd136276ebf7b75caa1bea78a Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 18 Apr 2019 14:10:12 +0800 Subject: [PATCH 063/202] Bump chart version Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/README.md b/charts/README.md index a039405009..84b7920dc5 100644 --- a/charts/README.md +++ b/charts/README.md @@ -68,3 +68,11 @@ $ helm dependency update loki-stack $ helm install ./loki-stack --dry-run --debug # to see changes format as expected $ helm install ./loki-stack # to see changes work as expected ``` + +After verify changes, need to bump chart version. +For example, if you update loki chart, you need bump version as following: + +```bash +$ # update version loki/Chart.yaml +$ # update version loki-stack/Chart.yaml +``` From a555ec9acc3394e0a3a11f3f8c79dad9be2733d6 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Thu, 18 Apr 2019 14:15:19 +0800 Subject: [PATCH 064/202] Add comment Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index 84b7920dc5..4c25a31b02 100644 --- a/charts/README.md +++ b/charts/README.md @@ -64,7 +64,7 @@ If you want to add any feature to helm chart, you can follow as below: ```bash $ cd production/helm $ # do some changes to loki/promtail in corresponding directory -$ helm dependency update loki-stack +$ helm dependency update loki-stack # run after do any changes $ helm install ./loki-stack --dry-run --debug # to see changes format as expected $ helm install ./loki-stack # to see changes work as expected ``` From 56e167f47cef833ce7c6b58c4eac30ec4a216680 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 19 Apr 2019 09:32:42 +0800 Subject: [PATCH 065/202] Update language Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/charts/README.md b/charts/README.md index 4c25a31b02..3e9f2a9a7e 100644 --- a/charts/README.md +++ b/charts/README.md @@ -62,15 +62,13 @@ Then follow the [instructions for adding the loki datasource](/docs/usage.md), u If you want to add any feature to helm chart, you can follow as below: ```bash -$ cd production/helm -$ # do some changes to loki/promtail in corresponding directory -$ helm dependency update loki-stack # run after do any changes -$ helm install ./loki-stack --dry-run --debug # to see changes format as expected -$ helm install ./loki-stack # to see changes work as expected +$ # do some changes to loki/promtail in the corresponding directory +$ make helm +$ helm upgrade --install loki ./loki-stack-*.tgz ``` After verify changes, need to bump chart version. -For example, if you update loki chart, you need bump version as following: +For example, if you update the loki chart, you need to bump the version as following: ```bash $ # update version loki/Chart.yaml From 9cd668e80dc003df01dd3fbf0aabae734b2dbd8b Mon Sep 17 00:00:00 2001 From: Sverre Boschman Date: Tue, 23 Apr 2019 20:45:20 +0200 Subject: [PATCH 066/202] allow additional container args for Loki (#503) --- charts/loki/templates/deployment.yaml | 3 +++ charts/loki/values.yaml | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/charts/loki/templates/deployment.yaml b/charts/loki/templates/deployment.yaml index 915cb17545..86e46c014a 100644 --- a/charts/loki/templates/deployment.yaml +++ b/charts/loki/templates/deployment.yaml @@ -45,6 +45,9 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "-config.file=/etc/loki/loki.yaml" + {{- range $key, $value := .Values.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} volumeMounts: - name: config mountPath: /etc/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 2625783582..94e3d7e294 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -58,6 +58,10 @@ image: tag: latest pullPolicy: Always # Always pull while in BETA +## Additional Loki container arguments, e.g. log level (debug, info, warn, error) +extraArgs: {} + # log.level: debug + livenessProbe: httpGet: path: /ready From be47aed6fb139f73b1cfedf793419f8733ce3a5b Mon Sep 17 00:00:00 2001 From: Sverre Boschman Date: Wed, 24 Apr 2019 16:24:55 +0200 Subject: [PATCH 067/202] allow additional pod labels (#509) --- charts/loki/templates/deployment.yaml | 3 +++ charts/loki/values.yaml | 3 +++ charts/promtail/templates/daemonset.yaml | 3 +++ charts/promtail/values.yaml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/charts/loki/templates/deployment.yaml b/charts/loki/templates/deployment.yaml index 86e46c014a..80eaa6f759 100644 --- a/charts/loki/templates/deployment.yaml +++ b/charts/loki/templates/deployment.yaml @@ -27,6 +27,9 @@ spec: app: {{ template "loki.name" . }} name: {{ template "loki.name" . }} release: {{ .Release.Name }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 94e3d7e294..c445040273 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -91,6 +91,9 @@ persistence: # subPath: "" # existingClaim: +## Pod Labels +podLabels: {} + ## Pod Annotations podAnnotations: {} # prometheus.io/scrape: "true" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 4a04c713fc..8664ce8e84 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -24,6 +24,9 @@ spec: labels: app: {{ template "promtail.name" . }} release: {{ .Release.Name }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} spec: diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index e6cb8f620f..c58c23ac4a 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -25,6 +25,9 @@ nameOverride: promtail ## ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} +## Pod Labels +podLabels: {} + podAnnotations: {} # prometheus.io/scrape: "true" # prometheus.io/port: "http-metrics" From 99c53c6b17ef4cf3a3938559cadf2e5269189fae Mon Sep 17 00:00:00 2001 From: Arjen van der Have Date: Thu, 25 Apr 2019 21:21:58 +0200 Subject: [PATCH 068/202] Configurable Loki scheme in helm file, with optional basic auth (#494) * Make Loki url scheme configurable and add optional authentication in helm chart * Added help text in README and changed propery names based on code review --- charts/README.md | 38 ++++++++++++++++++++++++ charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 6 +++- charts/promtail/values.yaml | 3 ++ 5 files changed, 48 insertions(+), 3 deletions(-) diff --git a/charts/README.md b/charts/README.md index 833a1325ee..59e5b39b12 100644 --- a/charts/README.md +++ b/charts/README.md @@ -56,3 +56,41 @@ $ kubectl port-forward --namespace service/loki-grafana 3000:80 Navigate to http://localhost:3000 and login with `admin` and the password output above. Then follow the [instructions for adding the loki datasource](/docs/usage.md), using the URL `http://loki:3100/`. + +## Run Loki behind https ingress + +If Loki and Promtail are deployed on different clusters you can add an Ingress in front of Loki. +By adding a certificate you create an https endpoint. For extra security enable basic authentication on the Ingress. + +In promtail set the following values to communicate with https and basic auth + +``` +loki: + serviceScheme: https + user: user + password: pass +``` + +Sample helm template for ingress: +``` +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: +annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.class }} + ingress.kubernetes.io/auth-type: "basic" + ingress.kubernetes.io/auth-secret: {{ .Values.ingress.basic.secret }} +name: loki +spec: +rules: +- host: {{ .Values.ingress.host }} + http: + paths: + - backend: + serviceName: loki + servicePort: 3100 +tls: +- secretName: {{ .Values.ingress.cert }} + hosts: + - {{ .Values.ingress.host }} +``` diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index b0c5c29176..2080257105 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.7.0 +version: 0.7.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index bc60af652d..8f3ea952a3 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.6.2 +version: 0.6.3 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 8664ce8e84..f800bc3e8e 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -40,7 +40,11 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "-config.file=/etc/promtail/promtail.yaml" - - "-client.url=http://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/api/prom/push" + {{- if and .Values.loki.user .Values.loki.password }} + - "-client.url={{ .Values.loki.serviceScheme }}://{{ .Values.loki.user }}:{{ .Values.loki.password }}@{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/api/prom/push" + {{- else }} + - "-client.url={{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/api/prom/push" + {{- end }} volumeMounts: - name: config mountPath: /etc/promtail diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index c58c23ac4a..8c77f41ceb 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -18,6 +18,9 @@ livenessProbe: {} loki: serviceName: "" # Defaults to "${RELEASE}-loki" if not set servicePort: 3100 + serviceScheme: http + # user: user + # password: pass nameOverride: promtail From bcba097c3c3cdf21ae0d4aa9824b96b582bfa2cf Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 26 Apr 2019 11:16:10 +0800 Subject: [PATCH 069/202] Add custom help info Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/README.md b/charts/README.md index 59e5b39b12..7932cbfa18 100644 --- a/charts/README.md +++ b/charts/README.md @@ -18,10 +18,18 @@ $ helm repo update ## Deploy Loki and Promtail to your cluster +### Deploy with default config + ```bash $ helm upgrade --install loki loki/loki-stack ``` +### Deploy with custom config + +```bash +$ helm upgrade --install loki loki/loki --set "key1=val1,key2=val2,..." +``` + ## Deploy Loki only ```bash From d4977d7aef1db504f5b4a638a3d3d8b8ca1a91b3 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Fri, 26 Apr 2019 17:52:46 +0530 Subject: [PATCH 070/202] =?UTF-8?q?Make=20sure=20the=20default=20for=20Enf?= =?UTF-8?q?orceMetricName=20is=20=E2=9C=85=20(#518)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Goutham Veeramachaneni --- charts/loki/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index c445040273..93739f3e72 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -32,8 +32,6 @@ config: # prefix: "" # httpclienttimeout: "20s" # consistentreads: true - limits_config: - enforce_metric_name: false schema_config: configs: - from: 0 From d8c0feb02455be2ae56a40bd835e42ee9ba31aa3 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 26 Apr 2019 21:32:38 +0800 Subject: [PATCH 071/202] Enable tracing loki in helm chart (#496) * Enable tracing loki in helm chart Tracing support from #328, need to support config in helm chart Signed-off-by: Xiang Dai <764524258@qq.com> * Use camel case for variables Signed-off-by: Xiang Dai <764524258@qq.com> * update condition Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/loki/Chart.yaml | 2 +- charts/loki/templates/deployment.yaml | 3 +++ charts/loki/values.yaml | 7 ++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index e6cc11d368..842628d995 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.7.0 +version: 0.7.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/deployment.yaml b/charts/loki/templates/deployment.yaml index 80eaa6f759..8f3257eff5 100644 --- a/charts/loki/templates/deployment.yaml +++ b/charts/loki/templates/deployment.yaml @@ -69,6 +69,9 @@ spec: {{- toYaml .Values.resources | nindent 12 }} securityContext: readOnlyRootFilesystem: true + env: + - name: JAEGER_AGENT_HOST + value: "{{ .Values.tracing.jaegerAgentHost }}" nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }} affinity: diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 93739f3e72..a03ca21f51 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -14,8 +14,13 @@ affinity: {} ## Deployment annotations annotations: {} +# enable tracing for debug, need install jaeger and specify right jaeger_agent_host +tracing: + jaegerAgentHost: + config: auth_enabled: false + ingester: chunk_idle_period: 15m lifecycler: @@ -128,7 +133,7 @@ securityContext: service: type: ClusterIP - nodePort: + nodePort: port: 3100 annotations: {} labels: {} From 5c737e5dbd639d8c1c5b42af80bf81058e4751ca Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Fri, 26 Apr 2019 19:55:17 +0530 Subject: [PATCH 072/202] Take 2 for fix the limit settings (#519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "Make sure the default for EnforceMetricName is ✅ (#518)" This reverts commit d4977d7aef1db504f5b4a638a3d3d8b8ca1a91b3. * Fix overrides unmarshalling properly Signed-off-by: Goutham Veeramachaneni --- charts/loki/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index a03ca21f51..560a374062 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -37,6 +37,8 @@ config: # prefix: "" # httpclienttimeout: "20s" # consistentreads: true + limits_config: + enforce_metric_name: false schema_config: configs: - from: 0 From c305f66c69da42ce6103a58d301580a7c3c49f9b Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 26 Apr 2019 23:16:12 +0800 Subject: [PATCH 073/202] Add target config (#486) * Add target config Support costomize target config. Signed-off-by: Xiang Dai <764524258@qq.com> * Update default value Signed-off-by: Xiang Dai <764524258@qq.com> * bump helm version Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/promtail/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 8c77f41ceb..2f5c0dcc46 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -120,6 +120,9 @@ config: http_listen_port: 3101 positions: filename: /run/promtail/positions.yaml + target_config: + # Period to resync directories being watched and files being tailed + sync_period: 10s scrape_configs: - entry_parser: '{{ .Values.entryParser }}' job_name: kubernetes-pods-name From fe36262dc711029710073b39d9742b621b9b644c Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Sun, 28 Apr 2019 09:40:52 +0800 Subject: [PATCH 074/202] bump up chart version Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/loki-stack/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 2080257105..86789f10a6 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.7.1 +version: 0.7.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." From f56cb64498483e7e7d77d204c764aeacc8e6569e Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 30 Apr 2019 15:40:16 +0800 Subject: [PATCH 075/202] Add option (#528) * Add chunk_block_size option Introduce by #464. Signed-off-by: Xiang Dai <764524258@qq.com> * fix white noise Signed-off-by: Xiang Dai <764524258@qq.com> * Bump up helm version Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 2080257105..86789f10a6 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.7.1 +version: 0.7.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 842628d995..7c7753f7f5 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.7.1 +version: 0.7.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 560a374062..32ed62c55a 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -23,6 +23,7 @@ config: ingester: chunk_idle_period: 15m + chunk_block_size: 262144 lifecycler: ring: store: inmemory @@ -63,7 +64,7 @@ image: tag: latest pullPolicy: Always # Always pull while in BETA -## Additional Loki container arguments, e.g. log level (debug, info, warn, error) +## Additional Loki container arguments, e.g. log level (debug, info, warn, error) extraArgs: {} # log.level: debug From 8e67e52aa676d0ae27fc23a566819102cf7cd64c Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Sat, 20 Apr 2019 01:15:27 -0500 Subject: [PATCH 076/202] Add scrape config for control plane static pods Signed-off-by: Steven Sheehy --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 5 ++- charts/promtail/values.yaml | 48 ++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 86789f10a6..777922dc2b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.7.2 +version: 0.8.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 8f3ea952a3..806b12091f 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.6.3 +version: 0.7.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index f800bc3e8e..2d98b3fc62 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -28,7 +28,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{ toYaml .Values.podAnnotations | nindent 8 }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ template "promtail.serviceAccountName" . }} {{- if .Values.priorityClassName }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 2f5c0dcc46..5d31ea2c12 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -318,3 +318,51 @@ config: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ + - entry_parser: '{{ .Values.entryParser }}' + job_name: kubernetes-pods-static + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: ^$ + source_labels: + - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror + - action: replace + source_labels: + - __meta_kubernetes_pod_label_component + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror + - __meta_kubernetes_pod_container_name + target_label: __path__ + From ddabd7aa1c7045fa6c90d350bb7ae90ab0cff7ba Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Tue, 7 May 2019 15:04:25 -0500 Subject: [PATCH 077/202] Support CRI 1.14+ directory change Signed-off-by: Steven Sheehy --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 11 +++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 777922dc2b..09803be592 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.8.0 +version: 0.8.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 806b12091f..97c5129a87 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.7.0 +version: 0.7.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 5d31ea2c12..b2020dbd48 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -160,7 +160,7 @@ config: target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log + - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid @@ -206,7 +206,7 @@ config: target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log + - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid @@ -258,7 +258,7 @@ config: target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log + - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid @@ -312,7 +312,7 @@ config: target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log + - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid @@ -359,10 +359,9 @@ config: target_label: container_name - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/$1/*.log + - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror - __meta_kubernetes_pod_container_name target_label: __path__ - From 4884ad2899d1065d159cb21447850b9c9b8060da Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Wed, 8 May 2019 11:26:21 -0400 Subject: [PATCH 078/202] Updating helm/ksonnet to use new pipeline config Updating docs Rebase fixes --- charts/promtail/values.yaml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index b2020dbd48..285dbc6401 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -6,8 +6,6 @@ annotations: {} deploymentStrategy: RollingUpdate -entryParser: docker - image: repository: grafana/promtail tag: latest @@ -124,8 +122,9 @@ config: # Period to resync directories being watched and files being tailed sync_period: 10s scrape_configs: - - entry_parser: '{{ .Values.entryParser }}' - job_name: kubernetes-pods-name + - job_name: kubernetes-pods-name + pipeline_stages: + - docker: {} kubernetes_sd_configs: - role: pod relabel_configs: @@ -166,8 +165,9 @@ config: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - - entry_parser: '{{ .Values.entryParser }}' - job_name: kubernetes-pods-app + - job_name: kubernetes-pods-app + pipeline_stages: + - docker: {} kubernetes_sd_configs: - role: pod relabel_configs: @@ -212,8 +212,9 @@ config: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - - entry_parser: '{{ .Values.entryParser }}' - job_name: kubernetes-pods-direct-controllers + - job_name: kubernetes-pods-direct-controllers + pipeline_stages: + - docker: {} kubernetes_sd_configs: - role: pod relabel_configs: @@ -264,8 +265,9 @@ config: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - - entry_parser: '{{ .Values.entryParser }}' - job_name: kubernetes-pods-indirect-controller + - job_name: kubernetes-pods-indirect-controller + pipeline_stages: + - docker: {} kubernetes_sd_configs: - role: pod relabel_configs: @@ -318,8 +320,9 @@ config: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - - entry_parser: '{{ .Values.entryParser }}' - job_name: kubernetes-pods-static + - job_name: kubernetes-pods-static + pipeline_stages: + - docker: {} kubernetes_sd_configs: - role: pod relabel_configs: From 09d633b0481d87909676634cdca21388a50972a5 Mon Sep 17 00:00:00 2001 From: Victoria Date: Thu, 9 May 2019 13:37:22 -0500 Subject: [PATCH 079/202] add optional PodDisruptionBudget to helm chart (#515) * add optional PodDisruptionBudget to helm chart * address comments * add back new line - no change to helpers template * fix silly typo in loki-stack version --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/_helpers.tpl | 1 - charts/loki/templates/pdb.yaml | 15 +++++++++++++++ charts/loki/values.yaml | 6 ++++++ 5 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 charts/loki/templates/pdb.yaml diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 09803be592..30d3301d13 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.8.1 +version: 0.9.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 7c7753f7f5..734fb87593 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.7.2 +version: 0.8.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/_helpers.tpl b/charts/loki/templates/_helpers.tpl index 608753377e..2e333aae60 100644 --- a/charts/loki/templates/_helpers.tpl +++ b/charts/loki/templates/_helpers.tpl @@ -41,4 +41,3 @@ Create the name of the service account {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} - diff --git a/charts/loki/templates/pdb.yaml b/charts/loki/templates/pdb.yaml new file mode 100644 index 0000000000..795733619d --- /dev/null +++ b/charts/loki/templates/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "loki.fullname" . }} + app: {{ template "loki.name" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + chart: {{ template "loki.chart" . }} +spec: + selector: + matchLabels: + app: {{ template "loki.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} \ No newline at end of file diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 32ed62c55a..84c06fc67f 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -150,3 +150,9 @@ terminationGracePeriodSeconds: 30 ## Tolerations for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ tolerations: [] + +# The values to set in the PodDisruptionBudget spec +# If not set then a PodDisruptionBudget will not be created +podDisruptionBudget: {} +# minAvailable: 1 +# maxUnavailable: 1 From 1b0d55bea3e1a0d5876ae2cb3cbc0d158a6e6ca2 Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Wed, 15 May 2019 12:46:54 +0530 Subject: [PATCH 080/202] Limiting query start time with config (#572) * Limiting query start time with config * Fixed maxLookBackPeriod check, added it to existing configs, some other fixes * gofmted file * Changed maxLookBackPeriod to 4 weeks in helm chart * Setting maxLookBackPeriod in helm chart to 0 and bumping up the helm chart version * changed maxLookBackPeriod in yaml in cmd to 0 and some nit --- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 734fb87593..1bd95d5868 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.8.0 +version: 0.8.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 84c06fc67f..10842ddabd 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -56,6 +56,8 @@ config: directory: /data/loki/index filesystem: directory: /data/loki/chunks + querier: + max_look_back_period: 0 deploymentStrategy: RollingUpdate From 06ca1fbb3b10d766ded8dce206af1a4cffb5f400 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Wed, 15 May 2019 08:35:13 -0500 Subject: [PATCH 081/202] Helm: Allow custom pipeline stages (#580) Signed-off-by: Steven Sheehy --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/templates/deployment.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/configmap.yaml | 253 +++++++++++++++++++++- charts/promtail/values.yaml | 254 +---------------------- 5 files changed, 262 insertions(+), 251 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 30d3301d13..5d92b3fa64 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.9.0 +version: 0.9.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/deployment.yaml b/charts/loki/templates/deployment.yaml index 8f3257eff5..9bf0617044 100644 --- a/charts/loki/templates/deployment.yaml +++ b/charts/loki/templates/deployment.yaml @@ -39,9 +39,9 @@ spec: serviceAccountName: {{ template "loki.serviceAccountName" . }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} + {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 8 }} - {{- end }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 97c5129a87..47897514e1 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.7.1 +version: 0.7.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/configmap.yaml b/charts/promtail/templates/configmap.yaml index 3072048aa5..68dad71d39 100644 --- a/charts/promtail/templates/configmap.yaml +++ b/charts/promtail/templates/configmap.yaml @@ -9,4 +9,255 @@ metadata: heritage: {{ .Release.Service }} data: promtail.yaml: | - {{- tpl (toYaml .Values.config | nindent 4 ) . }} + {{- toYaml .Values.config | nindent 4 }} + scrape_configs: + {{- if .Values.scrapeConfigs }} + {{- toYaml .Values.scrapeConfigs | nindent 4 }} + {{- else }} + - job_name: kubernetes-pods-name + pipeline_stages: + {{- toYaml .Values.pipelineStages | nindent 8 }} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-app + pipeline_stages: + {{- toYaml .Values.pipelineStages | nindent 8 }} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + source_labels: + - __meta_kubernetes_pod_label_name + - source_labels: + - __meta_kubernetes_pod_label_app + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-direct-controllers + pipeline_stages: + {{- toYaml .Values.pipelineStages | nindent 8 }} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: drop + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-indirect-controller + pipeline_stages: + {{- toYaml .Values.pipelineStages | nindent 8 }} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: keep + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - action: replace + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-static + pipeline_stages: + {{- toYaml .Values.pipelineStages | nindent 8 }} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: ^$ + source_labels: + - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror + - action: replace + source_labels: + - __meta_kubernetes_pod_label_component + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror + - __meta_kubernetes_pod_container_name + target_label: __path__ + {{- end }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 285dbc6401..46943240ba 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -26,6 +26,9 @@ nameOverride: promtail ## ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} +pipelineStages: +- docker: {} + ## Pod Labels podLabels: {} @@ -61,6 +64,9 @@ resources: {} # cpu: 100m # memory: 128Mi +# Custom scrape_configs to override the default ones in the configmap +scrapeConfigs: [] + securityContext: readOnlyRootFilesystem: true runAsGroup: 0 @@ -121,250 +127,4 @@ config: target_config: # Period to resync directories being watched and files being tailed sync_period: 10s - scrape_configs: - - job_name: kubernetes-pods-name - pipeline_stages: - - docker: {} - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: - - __meta_kubernetes_pod_label_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - job_name: kubernetes-pods-app - pipeline_stages: - - docker: {} - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - source_labels: - - __meta_kubernetes_pod_label_name - - source_labels: - - __meta_kubernetes_pod_label_app - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - job_name: kubernetes-pods-direct-controllers - pipeline_stages: - - docker: {} - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - separator: '' - source_labels: - - __meta_kubernetes_pod_label_name - - __meta_kubernetes_pod_label_app - - action: drop - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - - source_labels: - - __meta_kubernetes_pod_controller_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - job_name: kubernetes-pods-indirect-controller - pipeline_stages: - - docker: {} - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - separator: '' - source_labels: - - __meta_kubernetes_pod_label_name - - __meta_kubernetes_pod_label_app - - action: keep - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - - action: replace - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ - source_labels: - - __meta_kubernetes_pod_controller_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ - - job_name: kubernetes-pods-static - pipeline_stages: - - docker: {} - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: ^$ - source_labels: - - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror - - action: replace - source_labels: - - __meta_kubernetes_pod_label_component - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: ^$ - source_labels: - - __service__ - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: instance - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror - - __meta_kubernetes_pod_container_name - target_label: __path__ + From 8c192b57864575bdfed045d03f07496165453390 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 17 May 2019 13:38:24 +0800 Subject: [PATCH 082/202] bump up promtail chart version Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/promtail/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 3431acd326..4132054d31 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.6.4 +version: 0.7.3 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" From d69610b0efa18d91eedbc95be434dd1bfb4918ee Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Fri, 17 May 2019 17:07:07 +0530 Subject: [PATCH 083/202] Fix local config: Use DayTime (#598) * Fix local config: Use DayTime Psst, the date is the first commit date for loki ;) Signed-off-by: Goutham Veeramachaneni * Fix the config in ksonnet and helm too Signed-off-by: Goutham Veeramachaneni --- charts/loki/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 10842ddabd..f5b79ddcf6 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -42,7 +42,7 @@ config: enforce_metric_name: false schema_config: configs: - - from: 0 + - from: 2018-04-15 store: boltdb object_store: filesystem schema: v9 From 19c4d3f461425160300f3285e60803fdc819be0c Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Tue, 21 May 2019 14:04:45 -0400 Subject: [PATCH 084/202] need to bump helm chart after PR #598 --- charts/loki/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 1bd95d5868..d785a68c19 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.8.1 +version: 0.8.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." From 016a78edeb0371a4e06117f3e7bc509da32cdad1 Mon Sep 17 00:00:00 2001 From: Ed Date: Thu, 23 May 2019 10:01:43 -0400 Subject: [PATCH 085/202] Also bumping lok-stack version which needs to be done if the loki or promtail chart versions are changed (#612) --- charts/loki-stack/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 5d92b3fa64..e7221b79ab 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.9.1 +version: 0.9.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." From 481929f68cbddfd1a41615e35dcbdfb30b6ea5c1 Mon Sep 17 00:00:00 2001 From: Love Billingskog Nyberg Date: Fri, 24 May 2019 14:37:14 +0200 Subject: [PATCH 086/202] Added namespace to helm install notes (#617) --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/NOTES.txt | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/NOTES.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index e7221b79ab..2264efc7b1 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.9.2 +version: 0.9.3 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index d785a68c19..a19a980521 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.8.2 +version: 0.8.3 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/NOTES.txt b/charts/loki/templates/NOTES.txt index 095580bc4d..abe023a700 100644 --- a/charts/loki/templates/NOTES.txt +++ b/charts/loki/templates/NOTES.txt @@ -1,3 +1,3 @@ Verify the application is working by running these commands: - kubectl port-forward service/{{ include "loki.fullname" . }} {{ .Values.service.port }} + kubectl --namespace {{ .Release.Namespace }} port-forward service/{{ include "loki.fullname" . }} {{ .Values.service.port }} curl http://127.0.0.1:{{ .Values.service.port }}/api/prom/label diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 47897514e1..4132054d31 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.7.2 +version: 0.7.3 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/NOTES.txt b/charts/promtail/templates/NOTES.txt index 0f46731b6f..1da0c9d891 100644 --- a/charts/promtail/templates/NOTES.txt +++ b/charts/promtail/templates/NOTES.txt @@ -1,3 +1,3 @@ Verify the application is working by running these commands: - kubectl port-forward daemonset/{{ include "promtail.fullname" . }} {{ .Values.port }} + kubectl --namespace {{ .Release.Namespace }} port-forward daemonset/{{ include "promtail.fullname" . }} {{ .Values.port }} curl http://127.0.0.1:{{ .Values.port }}/metrics From fb034ddc9db309c96866d6e4ad894d8a154dffc3 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Fri, 24 May 2019 08:53:17 -0400 Subject: [PATCH 087/202] This add make target to deploy a dev version using helm (#586) * add a dev target to deploy the current image in k8s * impr/clients: Handle TLS config and MTLS for logcli and promtail (#540) * impr/clients: Handle TLS config and MTLS for logcli and promtail * fix/tls: Please gofmt... * impr/clients: use prometheus HTTPClientConfig for logcli and promtail * fix/promtail: Set proper Client config name * impr/promtail: Use prometheus HTTPClientConfig configuration * adapt with master * address review * fix conflicts * address requested changes * remove file * add helm dev targets * adding back assets * fix review comments * Review feedback --- charts/README.md | 3 ++ charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/requirements.yaml | 9 +++++- charts/loki-stack/templates/_helpers.tpl | 32 ++++++++++++++++++++ charts/loki-stack/templates/datasources.yaml | 30 ++++++++++++++++++ charts/loki-stack/values.yaml | 10 ++++++ charts/loki/values.yaml | 6 ++-- charts/promtail/values.yaml | 6 ++-- 8 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 charts/loki-stack/templates/_helpers.tpl create mode 100644 charts/loki-stack/templates/datasources.yaml diff --git a/charts/README.md b/charts/README.md index 18553dd134..0ba9fabe8d 100644 --- a/charts/README.md +++ b/charts/README.md @@ -119,3 +119,6 @@ For example, if you update the loki chart, you need to bump the version as follo ```bash $ # update version loki/Chart.yaml $ # update version loki-stack/Chart.yaml +``` + +You can use the `make helm-debug` to test and print out all chart templates. If you want to install helm (tiller) in your cluster use `make helm-install`, to install the current build in your Kubernetes cluster run `make helm-upgrade`. \ No newline at end of file diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 2264efc7b1..750056bc70 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.9.3 +version: 0.9.4 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/requirements.yaml b/charts/loki-stack/requirements.yaml index 3587624cb6..639032a174 100644 --- a/charts/loki-stack/requirements.yaml +++ b/charts/loki-stack/requirements.yaml @@ -7,4 +7,11 @@ dependencies: condition: promtail.enabled repository: "file://../promtail" version: "^0.6.0" - +- name: "grafana" + condition: grafana.enabled + version: "~3.3.7" + repository: "https://kubernetes-charts.storage.googleapis.com/" +- name: "prometheus" + condition: prometheus.enabled + version: "~8.11.2" + repository: "https://kubernetes-charts.storage.googleapis.com/" \ No newline at end of file diff --git a/charts/loki-stack/templates/_helpers.tpl b/charts/loki-stack/templates/_helpers.tpl new file mode 100644 index 0000000000..9c9e517686 --- /dev/null +++ b/charts/loki-stack/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "loki-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "loki-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "loki-stack.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/loki-stack/templates/datasources.yaml b/charts/loki-stack/templates/datasources.yaml new file mode 100644 index 0000000000..581bd58906 --- /dev/null +++ b/charts/loki-stack/templates/datasources.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "loki-stack.fullname" . }} + labels: + app: {{ template "loki-stack.name" . }} + chart: {{ template "loki-stack.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + grafana_datasource: "1" +data: + loki-stack-datasource.yaml: |- + apiVersion: 1 + datasources: +{{- if .Values.loki.enabled }} + - name: Loki + type: loki + access: proxy + url: http://{{(include "loki.serviceName" .)}}:{{ .Values.loki.service.port }} + version: 1 +{{- end }} +{{- if .Values.prometheus.enabled }} + - name: Prometheus + type: prometheus + access: proxy + url: http://{{ .Values.prometheus.server.fullnameOverride }}:{{ .Values.prometheus.server.service.servicePort }} + version: 1 +{{- end }} +{{- end }} diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index 06b64946a2..3da8751e61 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -4,3 +4,13 @@ loki: promtail: enabled: true +grafana: + enabled: false + sidecar: + datasources: + enabled: true + +prometheus: + enabled: false + server: + fullnameOverride: prometheus-server diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index f5b79ddcf6..7e30a32917 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -103,9 +103,9 @@ persistence: podLabels: {} ## Pod Annotations -podAnnotations: {} -# prometheus.io/scrape: "true" -# prometheus.io/port: "http-metrics" +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "http-metrics" ## Assign a PriorityClassName to pods if set # priorityClassName: diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 46943240ba..abbd990f4e 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -32,9 +32,9 @@ pipelineStages: ## Pod Labels podLabels: {} -podAnnotations: {} -# prometheus.io/scrape: "true" -# prometheus.io/port: "http-metrics" +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "http-metrics" # This should match config.server.http_listen_port port: 3101 From 20b3f1bfcdd04acb660976f47dc7a0ca41893515 Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Fri, 24 May 2019 18:24:09 +0530 Subject: [PATCH 088/202] Update cortex vendor (#610) * Update cortex vendor Use query max look back from cortex Update config changes from cortex Fixed breaking code due to updating cortex vendor * fixed linter error --- charts/loki/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 7e30a32917..1cbfff557c 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -26,7 +26,8 @@ config: chunk_block_size: 262144 lifecycler: ring: - store: inmemory + kvstore: + store: inmemory replication_factor: 1 ## Different ring configs can be used. E.g. Consul @@ -56,7 +57,7 @@ config: directory: /data/loki/index filesystem: directory: /data/loki/chunks - querier: + chunk_store_config: max_look_back_period: 0 deploymentStrategy: RollingUpdate From ae69b96862b4def32bd4676b89c750ff5bd2a5e0 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 24 May 2019 15:50:02 +0200 Subject: [PATCH 089/202] Update readme (loki-stack) --- charts/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/README.md b/charts/README.md index 0ba9fabe8d..19cadd3017 100644 --- a/charts/README.md +++ b/charts/README.md @@ -27,7 +27,7 @@ $ helm upgrade --install loki loki/loki-stack ### Deploy with custom config ```bash -$ helm upgrade --install loki loki/loki --set "key1=val1,key2=val2,..." +$ helm upgrade --install loki loki/loki-stack --set "key1=val1,key2=val2,..." ``` ## Deploy Loki only @@ -121,4 +121,4 @@ $ # update version loki/Chart.yaml $ # update version loki-stack/Chart.yaml ``` -You can use the `make helm-debug` to test and print out all chart templates. If you want to install helm (tiller) in your cluster use `make helm-install`, to install the current build in your Kubernetes cluster run `make helm-upgrade`. \ No newline at end of file +You can use the `make helm-debug` to test and print out all chart templates. If you want to install helm (tiller) in your cluster use `make helm-install`, to install the current build in your Kubernetes cluster run `make helm-upgrade`. From d535827fb78ff984cfa0066666f42ece705c6870 Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Mon, 27 May 2019 18:55:18 +0530 Subject: [PATCH 090/202] ksonnet changes for running loki in single binary (#622) * ksonnet changes for running loki in single bianry Added retention config with default values * Fixed indentation --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 750056bc70..1f5b69491b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.9.4 +version: 0.9.5 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index a19a980521..13795ee04d 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.8.3 +version: 0.8.4 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 1cbfff557c..55eed62791 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -59,6 +59,9 @@ config: directory: /data/loki/chunks chunk_store_config: max_look_back_period: 0 + table_manager: + retention_deletes_enabled: false + retention_period: 0 deploymentStrategy: RollingUpdate From 8c0383d2c5642254b932d4a4f8f1d883a1e57a06 Mon Sep 17 00:00:00 2001 From: Jon Davies Date: Wed, 29 May 2019 20:03:51 +0100 Subject: [PATCH 091/202] Helm chart tracing variable fix (#621) * loki chart deployment.yaml: Only set JAEGER_AGENT_HOST if set. * docs/troubleshooting.md: Fixed variable name for chart option. * Bumped loki chart version for variable fix. --- charts/loki/Chart.yaml | 2 +- charts/loki/templates/deployment.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 13795ee04d..cf5aab6f2d 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.8.4 +version: 0.8.5 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/deployment.yaml b/charts/loki/templates/deployment.yaml index 9bf0617044..e8fe9be095 100644 --- a/charts/loki/templates/deployment.yaml +++ b/charts/loki/templates/deployment.yaml @@ -70,8 +70,10 @@ spec: securityContext: readOnlyRootFilesystem: true env: + {{- if .Values.tracing.jaegerAgentHost }} - name: JAEGER_AGENT_HOST value: "{{ .Values.tracing.jaegerAgentHost }}" + {{- end }} nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }} affinity: From 5db6cfd439b910504b715999030449558efbbf61 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Wed, 29 May 2019 14:23:06 -0500 Subject: [PATCH 092/202] Switch Loki to StatefulSet (#585) Signed-off-by: Steven Sheehy --- charts/README.md | 13 +++--- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/pvc.yaml | 22 ---------- charts/loki/templates/service-headless.yaml | 19 +++++++++ .../{deployment.yaml => statefulset.yaml} | 40 ++++++++++++------- charts/loki/values.yaml | 12 +++--- 7 files changed, 58 insertions(+), 52 deletions(-) delete mode 100644 charts/loki/templates/pvc.yaml create mode 100644 charts/loki/templates/service-headless.yaml rename charts/loki/templates/{deployment.yaml => statefulset.yaml} (77%) diff --git a/charts/README.md b/charts/README.md index 19cadd3017..8a4cd09866 100644 --- a/charts/README.md +++ b/charts/README.md @@ -105,20 +105,17 @@ tls: ## How to contribute -If you want to add any feature to helm chart, you can follow as below: +After adding your new feature to the appropriate chart, you can build and deploy it locally to test: ```bash -$ # do some changes to loki/promtail in the corresponding directory $ make helm $ helm upgrade --install loki ./loki-stack-*.tgz ``` -After verify changes, need to bump chart version. -For example, if you update the loki chart, you need to bump the version as following: +After verifying your changes, you need to bump the chart version following [semantic versioning](https://semver.org) rules. +For example, if you update the loki chart, you need to bump the versions as follows: -```bash -$ # update version loki/Chart.yaml -$ # update version loki-stack/Chart.yaml -``` +- Update version loki/Chart.yaml +- Update version loki-stack/Chart.yaml You can use the `make helm-debug` to test and print out all chart templates. If you want to install helm (tiller) in your cluster use `make helm-install`, to install the current build in your Kubernetes cluster run `make helm-upgrade`. diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 1f5b69491b..1209a6ad8b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.9.5 +version: 0.10.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index cf5aab6f2d..d0ef9035a0 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.8.5 +version: 0.9.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/pvc.yaml b/charts/loki/templates/pvc.yaml deleted file mode 100644 index 350c04b120..0000000000 --- a/charts/loki/templates/pvc.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ template "loki.fullname" . }} - labels: - app: {{ template "loki.name" . }} - chart: {{ template "loki.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - annotations: - {{- toYaml .Values.persistence.annotations | nindent 4 }} -spec: - accessModes: - {{- range .Values.persistence.accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.persistence.size | quote }} - storageClassName: {{ .Values.persistence.storageClassName }} -{{- end }} diff --git a/charts/loki/templates/service-headless.yaml b/charts/loki/templates/service-headless.yaml new file mode 100644 index 0000000000..dbc127b3be --- /dev/null +++ b/charts/loki/templates/service-headless.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "loki.fullname" . }}-headless + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + clusterIP: None + ports: + - port: {{ .Values.service.port }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + selector: + app: {{ template "loki.name" . }} + release: {{ .Release.Name }} diff --git a/charts/loki/templates/deployment.yaml b/charts/loki/templates/statefulset.yaml similarity index 77% rename from charts/loki/templates/deployment.yaml rename to charts/loki/templates/statefulset.yaml index e8fe9be095..a1813b130a 100644 --- a/charts/loki/templates/deployment.yaml +++ b/charts/loki/templates/statefulset.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ template "loki.fullname" . }} labels: @@ -10,17 +10,15 @@ metadata: annotations: {{- toYaml .Values.annotations | nindent 4 }} spec: + podManagementPolicy: {{ .Values.podManagementPolicy }} replicas: {{ .Values.replicas }} - minReadySeconds: {{ .Values.minReadySeconds }} selector: matchLabels: app: {{ template "loki.name" . }} release: {{ .Release.Name }} - strategy: - type: {{ .Values.deploymentStrategy }} - {{- if ne .Values.deploymentStrategy "RollingUpdate" }} - rollingUpdate: null - {{- end }} + serviceName: {{ template "loki.fullname" . }}-headless + updateStrategy: + {{- toYaml .Values.updateStrategy | nindent 4 }} template: metadata: labels: @@ -29,7 +27,7 @@ spec: release: {{ .Release.Name }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} annotations: checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} @@ -50,7 +48,7 @@ spec: - "-config.file=/etc/loki/loki.yaml" {{- range $key, $value := .Values.extraArgs }} - "-{{ $key }}={{ $value }}" - {{- end }} + {{- end }} volumeMounts: - name: config mountPath: /etc/loki @@ -85,11 +83,25 @@ spec: - name: config secret: secretName: {{ template "loki.fullname" . }} + {{- if not .Values.persistence.enabled }} - name: storage - {{- if .Values.persistence.enabled }} - persistentVolumeClaim: - claimName: {{ .Values.persistence.existingClaim | default (include "loki.fullname" .) }} - {{- else }} emptyDir: {} - {{- end }} + {{- else if .Values.persistence.existingClaim }} + - name: storage + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: storage + annotations: + {{- toYaml .Values.persistence.annotations | nindent 8 }} + spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 8 }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + storageClassName: {{ .Values.persistence.storageClassName }} + {{- end }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 55eed62791..c1825e85b9 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -11,7 +11,7 @@ affinity: {} # - loki # topologyKey: "kubernetes.io/hostname" -## Deployment annotations +## StatefulSet annotations annotations: {} # enable tracing for debug, need install jaeger and specify right jaeger_agent_host @@ -20,7 +20,6 @@ tracing: config: auth_enabled: false - ingester: chunk_idle_period: 15m chunk_block_size: 262144 @@ -63,8 +62,6 @@ config: retention_deletes_enabled: false retention_period: 0 -deploymentStrategy: RollingUpdate - image: repository: grafana/loki tag: latest @@ -80,8 +77,6 @@ livenessProbe: port: http-metrics initialDelaySeconds: 45 -minReadySeconds: 0 - ## Enable persistence using Persistent Volume Claims networkPolicy: enabled: false @@ -111,6 +106,8 @@ podAnnotations: prometheus.io/scrape: "true" prometheus.io/port: "http-metrics" +podManagementPolicy: OrderedReady + ## Assign a PriorityClassName to pods if set # priorityClassName: @@ -162,3 +159,6 @@ tolerations: [] podDisruptionBudget: {} # minAvailable: 1 # maxUnavailable: 1 + +updateStrategy: + type: RollingUpdate From d5e22518acd6cb2107bcf7ad94a3b5b2a7c7e6f2 Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Thu, 30 May 2019 17:31:34 +0530 Subject: [PATCH 093/202] Limits: Reject entries based on age set in limits (#631) * Reject entries based on age set in limits --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 1209a6ad8b..b4df444c68 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.10.0 +version: 0.10.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index d0ef9035a0..22ce5789e6 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.9.0 +version: 0.9.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index c1825e85b9..e871de7ab1 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -40,6 +40,8 @@ config: # consistentreads: true limits_config: enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h schema_config: configs: - from: 2018-04-15 From 264884a557d2c15914559d6193285f445b6f466f Mon Sep 17 00:00:00 2001 From: Andrey Sidorov Date: Sat, 1 Jun 2019 19:38:54 +0300 Subject: [PATCH 094/202] fix labels for PodDisruptionBudget on helm (#623) --- charts/loki/Chart.yaml | 2 +- charts/loki/templates/pdb.yaml | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 22ce5789e6..0727864f69 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.9.1 +version: 0.9.2 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/pdb.yaml b/charts/loki/templates/pdb.yaml index 795733619d..27093f5d8e 100644 --- a/charts/loki/templates/pdb.yaml +++ b/charts/loki/templates/pdb.yaml @@ -3,13 +3,14 @@ apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: name: {{ template "loki.fullname" . }} - app: {{ template "loki.name" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - chart: {{ template "loki.chart" . }} + labels: + app: {{ template "loki.name" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + chart: {{ template "loki.chart" . }} spec: selector: matchLabels: app: {{ template "loki.name" . }} {{ toYaml .Values.podDisruptionBudget | indent 2 }} -{{- end -}} \ No newline at end of file +{{- end -}} From f69a74a583aeddaa65ed7ff1228876f6246bba2d Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Mon, 3 Jun 2019 16:35:56 -0400 Subject: [PATCH 095/202] change the helm pullPolicy since we will be using releases now --- charts/loki/values.yaml | 2 +- charts/promtail/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index e871de7ab1..1cd3bf1b47 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -67,7 +67,7 @@ config: image: repository: grafana/loki tag: latest - pullPolicy: Always # Always pull while in BETA + pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) extraArgs: {} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index abbd990f4e..4e8c7d7373 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -9,7 +9,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail tag: latest - pullPolicy: Always # Always pull while in BETA + pullPolicy: IfNotPresent livenessProbe: {} From 81c54b79c8b620f14c90fca3c1a18ff8d5e7bd69 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Mon, 3 Jun 2019 16:47:33 -0400 Subject: [PATCH 096/202] updating versions for loki v0.1.0 --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index b4df444c68..5ebe33ef3a 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.10.1 +version: 0.11.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 0727864f69..8703a5fff7 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.9.2 +version: 0.10.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 1cd3bf1b47..7ac6129ae6 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -66,7 +66,7 @@ config: image: repository: grafana/loki - tag: latest + tag: v0.1.0 pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 4132054d31..efbfc8b55d 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.7.3 +version: 0.8.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 4e8c7d7373..9848bf3640 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -8,7 +8,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail - tag: latest + tag: v0.1.0 pullPolicy: IfNotPresent livenessProbe: {} From caa5f9f8c37644f79d0f6aab6f343a432addbf11 Mon Sep 17 00:00:00 2001 From: Steven Sheehy Date: Wed, 12 Jun 2019 15:20:02 -0500 Subject: [PATCH 097/202] Helm: Integration testing (#641) * Helm chart integration testing Signed-off-by: Steven Sheehy * Don't check helm version on every push Signed-off-by: Steven Sheehy * Remove chart upgrade testing Signed-off-by: Steven Sheehy --- .../templates/tests/loki-test-configmap.yaml | 42 +++++++++++++++++++ .../templates/tests/loki-test-pod.yaml | 30 +++++++++++++ charts/loki/values.yaml | 4 +- charts/promtail/values.yaml | 9 ++-- 4 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 charts/loki-stack/templates/tests/loki-test-configmap.yaml create mode 100644 charts/loki-stack/templates/tests/loki-test-pod.yaml diff --git a/charts/loki-stack/templates/tests/loki-test-configmap.yaml b/charts/loki-stack/templates/tests/loki-test-configmap.yaml new file mode 100644 index 0000000000..205fd774c7 --- /dev/null +++ b/charts/loki-stack/templates/tests/loki-test-configmap.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "loki-stack.fullname" . }}-test + labels: + app: {{ template "loki-stack.name" . }} + chart: {{ template "loki-stack.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + test.sh: | + #!/usr/bin/env bash + + LOKI_URI="http://${LOKI_SERVICE}:${LOKI_PORT}" + + function setup() { + apk add -u curl jq + until (curl -s ${LOKI_URI}/api/prom/label/app/values | jq -e '.values[] | select(. == "loki")'); do + sleep 1 + done + } + + @test "Has labels" { + curl -s ${LOKI_URI}/api/prom/label | \ + jq -e '.values[] | select(. == "app")' + } + + @test "Query log entry" { + curl -sG ${LOKI_URI}/api/prom/query?limit=10 --data-urlencode 'query={app="loki"}' | \ + jq -e '.streams[].entries | length >= 1' + } + + @test "Push log entry" { + local timestamp=$(date -Iseconds -u | sed 's/UTC/.000000000+00:00/') + local data=$(jq -n --arg timestamp "${timestamp}" '{"streams": [{"labels": "{app=\"loki-test\"}", "entries": [{"ts": $timestamp, "line": "foobar"}]}]}') + + curl -s -X POST -H "Content-Type: application/json" ${LOKI_URI}/api/prom/push -d "${data}" + + curl -sG ${LOKI_URI}/api/prom/query?limit=1 --data-urlencode 'query={app="loki-test"}' | \ + jq -e '.streams[].entries[].line == "foobar"' + } + diff --git a/charts/loki-stack/templates/tests/loki-test-pod.yaml b/charts/loki-stack/templates/tests/loki-test-pod.yaml new file mode 100644 index 0000000000..a153a05df9 --- /dev/null +++ b/charts/loki-stack/templates/tests/loki-test-pod.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + "helm.sh/hook": test-success + labels: + app: {{ template "loki-stack.name" . }} + chart: {{ template "loki-stack.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "loki-stack.fullname" . }}-test +spec: + containers: + - name: test + image: bats/bats:v1.1.0 + args: + - /var/lib/loki/test.sh + env: + - name: LOKI_SERVICE + value: {{ template "loki.serviceName" . }} + - name: LOKI_PORT + value: "{{ .Values.loki.service.port }}" + volumeMounts: + - name: tests + mountPath: /var/lib/loki + restartPolicy: Never + volumes: + - name: tests + configMap: + name: {{ template "loki-stack.fullname" . }}-test diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 7ac6129ae6..c6725a5f8b 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -105,8 +105,8 @@ podLabels: {} ## Pod Annotations podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/port: "http-metrics" + prometheus.io/scrape: "true" + prometheus.io/port: "http-metrics" podManagementPolicy: OrderedReady diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 9848bf3640..ae75dfccab 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -14,7 +14,7 @@ image: livenessProbe: {} loki: - serviceName: "" # Defaults to "${RELEASE}-loki" if not set + serviceName: "" # Defaults to "${RELEASE}-loki" if not set servicePort: 3100 serviceScheme: http # user: user @@ -32,9 +32,9 @@ pipelineStages: ## Pod Labels podLabels: {} -podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/port: "http-metrics" +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "http-metrics" # This should match config.server.http_listen_port port: 3101 @@ -127,4 +127,3 @@ config: target_config: # Period to resync directories being watched and files being tailed sync_period: 10s - From 60e28396fa42ca8b1bdbc4bfb9e90b42ce286782 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Wed, 19 Jun 2019 09:32:47 +0800 Subject: [PATCH 098/202] fix helm lint issue Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/loki-stack/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 58be496c52..02301b1bc4 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -10,4 +10,4 @@ sources: maintainers: - name: Loki Maintainers email: lokiproject@googlegroups.com -engine: gotpl \ No newline at end of file +engine: gotpl From 3b82ef3428a154a040fd5adb7c1bdb7aa923141c Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Fri, 12 Apr 2019 09:56:21 +0800 Subject: [PATCH 099/202] Update chart version Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 5ebe33ef3a..02301b1bc4 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.11.0 +version: 0.11.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index efbfc8b55d..f5892a283b 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.8.0 +version: 0.8.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" From ab8a166ae1ab1baec05cd1e37b261a609f4ef8f4 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Fri, 5 Jul 2019 09:48:45 -0400 Subject: [PATCH 100/202] update grafana to fix dashboard provider (#674) --- charts/loki-stack/requirements.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loki-stack/requirements.yaml b/charts/loki-stack/requirements.yaml index 639032a174..2554a0ee58 100644 --- a/charts/loki-stack/requirements.yaml +++ b/charts/loki-stack/requirements.yaml @@ -9,7 +9,7 @@ dependencies: version: "^0.6.0" - name: "grafana" condition: grafana.enabled - version: "~3.3.7" + version: "~3.4.3" repository: "https://kubernetes-charts.storage.googleapis.com/" - name: "prometheus" condition: prometheus.enabled From 39ee3e236ee3df17d8dab09a376fb0001a7356fa Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Tue, 9 Jul 2019 14:41:45 +0200 Subject: [PATCH 101/202] Typo on values.yaml (#728) --- charts/promtail/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 884e729858..508f833858 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -111,7 +111,7 @@ config: minbackoff: 100ms # Maximum backoff time between retries maxbackoff: 5s - # Maximum number of retires when sending batches, 0 means infinite retries + # Maximum number of retries when sending batches, 0 means infinite retries maxretries: 5 # The labels to add to any time series or alerts when communicating with loki From 1132db8f1046ab31f20031665a566035b99ec051 Mon Sep 17 00:00:00 2001 From: Daniel Ulman Date: Tue, 16 Jul 2019 18:47:06 +0200 Subject: [PATCH 102/202] fixed helm installation instructions (#761) --- charts/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/README.md b/charts/README.md index 8a4cd09866..46938e78a7 100644 --- a/charts/README.md +++ b/charts/README.md @@ -33,13 +33,13 @@ $ helm upgrade --install loki loki/loki-stack --set "key1=val1,key2=val2,..." ## Deploy Loki only ```bash -$ helm upgrade --install loki loki/loki --set "loki.serviceName=my-loki" +$ helm upgrade --install loki loki/loki ``` ## Deploy Promtail only ```bash -$ helm upgrade --install promtail loki/promtail +$ helm upgrade --install promtail loki/promtail --set "loki.serviceName=loki" ``` ## Deploy Grafana to your cluster From 424f36ed997c3b3b791b6989f231ecd68fb2a3f5 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 1 Aug 2019 15:44:37 +0200 Subject: [PATCH 103/202] Helm: Be able to customize the Namespace (#830) * Add release namespace for loki * Add Release namespace for promtail chart * Add Release namespace for loki-stack chart * Add doumentation * Increase chart version --- charts/README.md | 6 ++++++ charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/templates/datasources.yaml | 1 + charts/loki/Chart.yaml | 2 +- charts/loki/templates/networkpolicy.yaml | 1 + charts/loki/templates/pdb.yaml | 1 + charts/loki/templates/podsecuritypolicy.yaml | 1 + charts/loki/templates/role.yaml | 1 + charts/loki/templates/rolebinding.yaml | 1 + charts/loki/templates/secret.yaml | 1 + charts/loki/templates/service-headless.yaml | 1 + charts/loki/templates/service.yaml | 1 + charts/loki/templates/serviceaccount.yaml | 1 + charts/loki/templates/statefulset.yaml | 1 + charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/clusterrole.yaml | 1 + charts/promtail/templates/configmap.yaml | 1 + charts/promtail/templates/daemonset.yaml | 1 + charts/promtail/templates/podsecuritypolicy.yaml | 1 + charts/promtail/templates/role.yaml | 1 + charts/promtail/templates/rolebinding.yaml | 1 + charts/promtail/templates/serviceaccount.yaml | 1 + 22 files changed, 27 insertions(+), 3 deletions(-) diff --git a/charts/README.md b/charts/README.md index 46938e78a7..4444271886 100644 --- a/charts/README.md +++ b/charts/README.md @@ -24,6 +24,12 @@ $ helm repo update $ helm upgrade --install loki loki/loki-stack ``` +### Deploy in a custom namespace + +```bash +$ helm upgrade --install loki --namespace=loki-stack loki/loki-stack +``` + ### Deploy with custom config ```bash diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 02301b1bc4..103ebdd732 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.11.1 +version: 0.12.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/templates/datasources.yaml b/charts/loki-stack/templates/datasources.yaml index 581bd58906..cec67f19bf 100644 --- a/charts/loki-stack/templates/datasources.yaml +++ b/charts/loki-stack/templates/datasources.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ template "loki-stack.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki-stack.name" . }} chart: {{ template "loki-stack.chart" . }} diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 8703a5fff7..4be7c928c8 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.10.0 +version: 0.11.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/networkpolicy.yaml b/charts/loki/templates/networkpolicy.yaml index 5becbcdb59..dea85abcaf 100644 --- a/charts/loki/templates/networkpolicy.yaml +++ b/charts/loki/templates/networkpolicy.yaml @@ -3,6 +3,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: {{ template "loki.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/loki/templates/pdb.yaml b/charts/loki/templates/pdb.yaml index 27093f5d8e..91c8d21271 100644 --- a/charts/loki/templates/pdb.yaml +++ b/charts/loki/templates/pdb.yaml @@ -3,6 +3,7 @@ apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: name: {{ template "loki.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} heritage: {{ .Release.Service }} diff --git a/charts/loki/templates/podsecuritypolicy.yaml b/charts/loki/templates/podsecuritypolicy.yaml index 9968b4844b..6a6444e3fa 100644 --- a/charts/loki/templates/podsecuritypolicy.yaml +++ b/charts/loki/templates/podsecuritypolicy.yaml @@ -3,6 +3,7 @@ apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: {{ template "loki.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/loki/templates/role.yaml b/charts/loki/templates/role.yaml index 756c42b57e..b7bfb29d68 100644 --- a/charts/loki/templates/role.yaml +++ b/charts/loki/templates/role.yaml @@ -3,6 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ template "loki.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/loki/templates/rolebinding.yaml b/charts/loki/templates/rolebinding.yaml index de1990b77b..41fc5039fe 100644 --- a/charts/loki/templates/rolebinding.yaml +++ b/charts/loki/templates/rolebinding.yaml @@ -3,6 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ template "loki.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/loki/templates/secret.yaml b/charts/loki/templates/secret.yaml index 95b16fad4b..40d50c8ee8 100644 --- a/charts/loki/templates/secret.yaml +++ b/charts/loki/templates/secret.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ template "loki.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/loki/templates/service-headless.yaml b/charts/loki/templates/service-headless.yaml index dbc127b3be..5536c39840 100644 --- a/charts/loki/templates/service-headless.yaml +++ b/charts/loki/templates/service-headless.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ template "loki.fullname" . }}-headless + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/loki/templates/service.yaml b/charts/loki/templates/service.yaml index 9a8c1da585..987da1989a 100644 --- a/charts/loki/templates/service.yaml +++ b/charts/loki/templates/service.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ template "loki.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/loki/templates/serviceaccount.yaml b/charts/loki/templates/serviceaccount.yaml index f223f0a1c2..a4315ced37 100644 --- a/charts/loki/templates/serviceaccount.yaml +++ b/charts/loki/templates/serviceaccount.yaml @@ -8,5 +8,6 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} name: {{ template "loki.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} {{- end }} diff --git a/charts/loki/templates/statefulset.yaml b/charts/loki/templates/statefulset.yaml index a1813b130a..ac2661cdfa 100644 --- a/charts/loki/templates/statefulset.yaml +++ b/charts/loki/templates/statefulset.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ template "loki.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index f5892a283b..fdeb252de0 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.8.1 +version: 0.9.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/clusterrole.yaml b/charts/promtail/templates/clusterrole.yaml index b872041b7a..7260d2e107 100644 --- a/charts/promtail/templates/clusterrole.yaml +++ b/charts/promtail/templates/clusterrole.yaml @@ -8,6 +8,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} name: {{ template "promtail.fullname" . }}-clusterrole + namespace: {{ .Release.Namespace }} rules: - apiGroups: [""] # "" indicates the core API group resources: diff --git a/charts/promtail/templates/configmap.yaml b/charts/promtail/templates/configmap.yaml index 68dad71d39..0f8b2c0faf 100644 --- a/charts/promtail/templates/configmap.yaml +++ b/charts/promtail/templates/configmap.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ template "promtail.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "promtail.name" . }} chart: {{ template "promtail.chart" . }} diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index e5923b1e61..1fee17590e 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: {{ template "promtail.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "promtail.name" . }} chart: {{ template "promtail.chart" . }} diff --git a/charts/promtail/templates/podsecuritypolicy.yaml b/charts/promtail/templates/podsecuritypolicy.yaml index a38595eeef..abaca89bf1 100644 --- a/charts/promtail/templates/podsecuritypolicy.yaml +++ b/charts/promtail/templates/podsecuritypolicy.yaml @@ -3,6 +3,7 @@ apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: {{ template "promtail.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "promtail.name" . }} chart: {{ template "promtail.chart" . }} diff --git a/charts/promtail/templates/role.yaml b/charts/promtail/templates/role.yaml index 97969e1a7a..1f55b1deae 100644 --- a/charts/promtail/templates/role.yaml +++ b/charts/promtail/templates/role.yaml @@ -3,6 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ template "promtail.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "promtail.name" . }} chart: {{ template "promtail.chart" . }} diff --git a/charts/promtail/templates/rolebinding.yaml b/charts/promtail/templates/rolebinding.yaml index ff99f2a923..a035c19f8d 100644 --- a/charts/promtail/templates/rolebinding.yaml +++ b/charts/promtail/templates/rolebinding.yaml @@ -3,6 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ template "promtail.fullname" . }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "promtail.name" . }} chart: {{ template "promtail.chart" . }} diff --git a/charts/promtail/templates/serviceaccount.yaml b/charts/promtail/templates/serviceaccount.yaml index 8165e08dff..9181d98a25 100644 --- a/charts/promtail/templates/serviceaccount.yaml +++ b/charts/promtail/templates/serviceaccount.yaml @@ -8,5 +8,6 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} name: {{ template "promtail.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} {{- end }} From 5a9c666c9e7b07d8113c01675a9a2d7303fd17e5 Mon Sep 17 00:00:00 2001 From: Alessio Martorelli <9568357+amartorelli@users.noreply.github.com> Date: Thu, 1 Aug 2019 17:17:13 +0300 Subject: [PATCH 104/202] Helm chart: Added service monitor (#824) * Helm chart: Added service monitor * Bumped loki-stack chart version * Bumped loki chart version --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/servicemonitor.yaml | 24 +++++++++++++++++++++++ charts/loki/values.yaml | 4 ++++ 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 charts/loki/templates/servicemonitor.yaml diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 103ebdd732..8259886280 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.12.0 +version: 0.13.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 4be7c928c8..26552809fc 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.11.0 +version: 0.12.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/servicemonitor.yaml b/charts/loki/templates/servicemonitor.yaml new file mode 100644 index 0000000000..191c08527b --- /dev/null +++ b/charts/loki/templates/servicemonitor.yaml @@ -0,0 +1,24 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "loki.fullname" . }} + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ template "loki.name" . }} + release: {{ .Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + endpoints: + - port: http-metrics + {{- if .Values.serviceMonitor.interval }} + interval: {{ .Values.serviceMonitor.interval }} + {{- end }} +{{- end }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index c6725a5f8b..3f25cffd0c 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -164,3 +164,7 @@ podDisruptionBudget: {} updateStrategy: type: RollingUpdate + +serviceMonitor: + enabled: false + interval: "" From 336df4174319467384ec1b06e7b2b5d2c60d2d8b Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 1 Aug 2019 15:15:11 -0400 Subject: [PATCH 105/202] fix grafana image tag by pinning it (#835) --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/values.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 8259886280..e671115d92 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.13.0 +version: 0.13.1 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index 3da8751e61..8d6fc56a80 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -9,6 +9,8 @@ grafana: sidecar: datasources: enabled: true + image: + tag: 6.3.0-beta2 prometheus: enabled: false From 61c077fe99efafcbbc0de119415a7a072e152b82 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Fri, 2 Aug 2019 15:04:42 -0400 Subject: [PATCH 106/202] release: update ksonnet and helm charts to version v0.2.0 --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index e671115d92..5077369c97 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.13.1 +version: 0.14.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 26552809fc..21e164747d 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.12.0 +version: 0.13.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 3f25cffd0c..264514140b 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -66,7 +66,7 @@ config: image: repository: grafana/loki - tag: v0.1.0 + tag: v0.2.0 pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index fdeb252de0..19a6d76ee1 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.9.0 +version: 0.10.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 508f833858..8c74282d30 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -8,7 +8,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail - tag: v0.1.0 + tag: v0.2.0 pullPolicy: IfNotPresent livenessProbe: {} From f4238a2bd1e746b38044ae196dc0efd5c7c5b6b5 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Tue, 6 Aug 2019 11:33:24 -0400 Subject: [PATCH 107/202] Reorder relabeling rules to prevent pod label from overwriting config defined labels --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/configmap.yaml | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 5077369c97..c1c6cf5bd7 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.14.0 +version: 0.15.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 19a6d76ee1..1b2b63487a 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.10.0 +version: 0.11.0 appVersion: 0.0.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/configmap.yaml b/charts/promtail/templates/configmap.yaml index 0f8b2c0faf..234aa2b43f 100644 --- a/charts/promtail/templates/configmap.yaml +++ b/charts/promtail/templates/configmap.yaml @@ -31,6 +31,8 @@ data: regex: ^$ source_labels: - __service__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / @@ -50,8 +52,6 @@ data: source_labels: - __meta_kubernetes_pod_container_name target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - replacement: /var/log/pods/*$1/*.log separator: / source_labels: @@ -78,6 +78,8 @@ data: regex: ^$ source_labels: - __service__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / @@ -97,8 +99,6 @@ data: source_labels: - __meta_kubernetes_pod_container_name target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - replacement: /var/log/pods/*$1/*.log separator: / source_labels: @@ -131,6 +131,8 @@ data: regex: ^$ source_labels: - __service__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / @@ -150,8 +152,6 @@ data: source_labels: - __meta_kubernetes_pod_container_name target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - replacement: /var/log/pods/*$1/*.log separator: / source_labels: @@ -186,6 +186,8 @@ data: regex: ^$ source_labels: - __service__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / @@ -205,8 +207,6 @@ data: source_labels: - __meta_kubernetes_pod_container_name target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - replacement: /var/log/pods/*$1/*.log separator: / source_labels: @@ -234,6 +234,8 @@ data: regex: ^$ source_labels: - __service__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / @@ -253,8 +255,6 @@ data: source_labels: - __meta_kubernetes_pod_container_name target_label: container_name - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - replacement: /var/log/pods/*$1/*.log separator: / source_labels: From 2d2aee4830fd889a0859696f7751666e6be32c31 Mon Sep 17 00:00:00 2001 From: Martin Baillie Date: Thu, 8 Aug 2019 01:23:47 +1000 Subject: [PATCH 108/202] Add promtail ServiceMonitor and headless Service (#856) These are optional objects guarded by `serviceMonitor.enabled`. Chart minor version also receives a bump. --- .../promtail/templates/service-headless.yaml | 22 +++++++++++++++++ charts/promtail/templates/servicemonitor.yaml | 24 +++++++++++++++++++ charts/promtail/values.yaml | 4 ++++ 3 files changed, 50 insertions(+) create mode 100644 charts/promtail/templates/service-headless.yaml create mode 100644 charts/promtail/templates/servicemonitor.yaml diff --git a/charts/promtail/templates/service-headless.yaml b/charts/promtail/templates/service-headless.yaml new file mode 100644 index 0000000000..f5f52365aa --- /dev/null +++ b/charts/promtail/templates/service-headless.yaml @@ -0,0 +1,22 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "promtail.fullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "promtail.name" . }} + chart: {{ template "promtail.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + clusterIP: None + ports: + - port: {{ .Values.config.server.http_listen_port }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + selector: + app: {{ template "promtail.name" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/charts/promtail/templates/servicemonitor.yaml b/charts/promtail/templates/servicemonitor.yaml new file mode 100644 index 0000000000..cd3fe02715 --- /dev/null +++ b/charts/promtail/templates/servicemonitor.yaml @@ -0,0 +1,24 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "promtail.fullname" . }} + labels: + app: {{ template "promtail.name" . }} + chart: {{ template "promtail.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ template "promtail.name" . }} + release: {{ .Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + endpoints: + - port: http-metrics + {{- if .Values.serviceMonitor.interval }} + interval: {{ .Values.serviceMonitor.interval }} + {{- end }} +{{- end }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 8c74282d30..2b0065f19d 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -124,3 +124,7 @@ config: target_config: # Period to resync directories being watched and files being tailed sync_period: 10s + +serviceMonitor: + enabled: false + interval: "" From 4fca8ec6cf07ff34a35530977bfd586d5efb3800 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Sun, 18 Aug 2019 08:56:17 -0400 Subject: [PATCH 109/202] update helm and ksonnet to 0.3.0 --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index c1c6cf5bd7..4c8fa7187b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ name: loki-stack -version: 0.15.0 -appVersion: 0.0.1 +version: 0.16.0 +appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 21e164747d..a0eb66d523 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ name: loki -version: 0.13.0 -appVersion: 0.0.1 +version: 0.14.0 +appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 264514140b..4a721643d4 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -66,7 +66,7 @@ config: image: repository: grafana/loki - tag: v0.2.0 + tag: v0.3.0 pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 1b2b63487a..cbd5bb2106 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ name: promtail -version: 0.11.0 -appVersion: 0.0.1 +version: 0.12.0 +appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 2b0065f19d..52373505a8 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -8,7 +8,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail - tag: v0.2.0 + tag: v0.3.0 pullPolicy: IfNotPresent livenessProbe: {} From 915ddbf55a00f8e9609ca14b0054419406e46223 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 17 Jun 2019 15:46:06 -0400 Subject: [PATCH 110/202] Improve helm default values for ingesters. --- charts/loki/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 4a721643d4..603bac5083 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -21,8 +21,9 @@ tracing: config: auth_enabled: false ingester: - chunk_idle_period: 15m + chunk_idle_period: 3m chunk_block_size: 262144 + chunk_retain_period: 1m lifecycler: ring: kvstore: From b40eb43523694154405002bf78c458eb45661521 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 17 Jun 2019 15:47:00 -0400 Subject: [PATCH 111/202] bump chart --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 4c8fa7187b..660b4f46f6 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.16.0 +version: 0.16.1 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index a0eb66d523..622f0f33ed 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.14.0 +version: 0.14.1 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." From cf68f7ca6eaa6713f0a3f83aa6d443e66295d829 Mon Sep 17 00:00:00 2001 From: Minh Danh Date: Mon, 26 Aug 2019 11:39:53 +0700 Subject: [PATCH 112/202] Add support for additional labels and scrapeTimeout for serviceMonitors --- charts/loki/templates/servicemonitor.yaml | 6 ++++++ charts/loki/values.yaml | 2 ++ charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/servicemonitor.yaml | 6 ++++++ charts/promtail/values.yaml | 2 ++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/charts/loki/templates/servicemonitor.yaml b/charts/loki/templates/servicemonitor.yaml index 191c08527b..4435832718 100644 --- a/charts/loki/templates/servicemonitor.yaml +++ b/charts/loki/templates/servicemonitor.yaml @@ -8,6 +8,9 @@ metadata: chart: {{ template "loki.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} + {{- if .Values.serviceMonitor.additionalLabels }} +{{ toYaml .Values.serviceMonitor.additionalLabels | indent 4 }} + {{- end }} spec: selector: matchLabels: @@ -21,4 +24,7 @@ spec: {{- if .Values.serviceMonitor.interval }} interval: {{ .Values.serviceMonitor.interval }} {{- end }} + {{- if .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + {{- end }} {{- end }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 603bac5083..826e5ca793 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -169,3 +169,5 @@ updateStrategy: serviceMonitor: enabled: false interval: "" + additionalLabels: {} + # scrapeTimeout: 10s diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index cbd5bb2106..e0d24bea65 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.12.0 +version: 0.12.1 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/servicemonitor.yaml b/charts/promtail/templates/servicemonitor.yaml index cd3fe02715..43f8aac8c2 100644 --- a/charts/promtail/templates/servicemonitor.yaml +++ b/charts/promtail/templates/servicemonitor.yaml @@ -8,6 +8,9 @@ metadata: chart: {{ template "promtail.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} + {{- if .Values.serviceMonitor.additionalLabels }} +{{ toYaml .Values.serviceMonitor.additionalLabels | indent 4 }} + {{- end }} spec: selector: matchLabels: @@ -21,4 +24,7 @@ spec: {{- if .Values.serviceMonitor.interval }} interval: {{ .Values.serviceMonitor.interval }} {{- end }} + {{- if .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + {{- end }} {{- end }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 52373505a8..43559f1400 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -128,3 +128,5 @@ config: serviceMonitor: enabled: false interval: "" + additionalLabels: {} + # scrapeTimeout: 10s From 6e47577f713c23b38878a8748dad50ab7d932e76 Mon Sep 17 00:00:00 2001 From: Minh Danh Date: Sat, 7 Sep 2019 08:51:51 +0700 Subject: [PATCH 113/202] Bump chart versions --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 660b4f46f6..d9fba6f47a 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.16.1 +version: 0.16.2 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 622f0f33ed..f0de1c0f9b 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.14.1 +version: 0.14.2 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." From 39e7f252bf9b333d373126d22990b8e153fc398b Mon Sep 17 00:00:00 2001 From: Alexander Danilov Date: Wed, 11 Sep 2019 15:55:05 +0300 Subject: [PATCH 114/202] Fix links to correct doc (#1005) Co-authored-by: Alexander Danilov --- charts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index 4444271886..cf77f9aac9 100644 --- a/charts/README.md +++ b/charts/README.md @@ -69,7 +69,7 @@ $ kubectl port-forward --namespace service/loki-grafana 3000:80 ``` Navigate to http://localhost:3000 and login with `admin` and the password output above. -Then follow the [instructions for adding the loki datasource](/docs/usage.md), using the URL `http://loki:3100/`. +Then follow the [instructions for adding the loki datasource](/docs/querying.md#grafana), using the URL `http://loki:3100/`. ## Run Loki behind https ingress From c6b0658e7735f61a9fe1c4a3b39405693ec83545 Mon Sep 17 00:00:00 2001 From: makocchi Date: Tue, 24 Sep 2019 22:39:09 +0900 Subject: [PATCH 115/202] helm chart: Add missing operator to promtail (#967) * Add missing operator * Bump chart versions * Bump chart versions --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index d9fba6f47a..687f2668ab 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.16.2 +version: 0.16.3 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index e0d24bea65..cf53859624 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.12.1 +version: 0.12.2 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 43559f1400..edfbf521fd 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -77,6 +77,7 @@ serviceAccount: ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ tolerations: - key: node-role.kubernetes.io/master + operator: Exists effect: NoSchedule # Extra volumes to scrape logs from From fce1d990044a25256f314379eb9dd8df3ca3aae1 Mon Sep 17 00:00:00 2001 From: polar3130 <47264004+polar3130@users.noreply.github.com> Date: Thu, 26 Sep 2019 22:03:56 +0900 Subject: [PATCH 116/202] Helm: Remove default value of storageClassName in loki/loki helm chart (#1058) * delete default name of storageClassName "default" * bump loki stack --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 687f2668ab..ed11e9e5f2 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.16.3 +version: 0.16.4 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index f0de1c0f9b..3936cc566c 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.14.2 +version: 0.14.3 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 826e5ca793..c82c68e794 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -96,7 +96,6 @@ persistence: accessModes: - ReadWriteOnce size: 10Gi - storageClassName: default annotations: {} # subPath: "" # existingClaim: From 1a7a35e9407b97925dd9766966b790e05b20a39e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 27 Sep 2019 06:10:10 +0900 Subject: [PATCH 117/202] fluent-bit shared object go plugin (#847) * Import fluent-bit-go-loki plugin sources * Add fluent-bit-go package constraint * Ensure dependencies with dep ``` $ dep ensure ``` * Update .gitignore * Move fluent-bit-go-loki to fluent-bit/fluent-bit-go-loki directory * Add documentation for fluent-bit-go-loki * Fix for lint * Fix for goimports errors * Display fluent-bit-go-loki plugin version * Use more descriptive description for fluent-bit Loki plugin * Remove needless newlines * Rectify fluent-bit-go-loki makefile Remove needless parts and rename meaningless target * Use more descriptive description in README * refactor(fluent-bit): move make targets into root Makefile * Organize imports * Reorder imports with goimports $ goimports -w out_loki.go * Remove needless LICENSE file Because it is the same for Loki repository. * Rely on Loki client retry mechanism * Make descriptions more clearly * Change default unit * BatchWait: msec -> sec * BatchSize: KiB -> Byte * Use logql to parse Loki native labels format * Use logger instead of fmt.Printf * Update documentation to reflect recent changes * Use prometheus version module to print version * Dump error message with logger * Use DYN_GO_FLAGS instead of custom FLAGS for fluent-bit-plugin * Support RemoveKeys parameter * use logql.ParseMatchers() * refactor fluent-bit go plugin * add ci step for docker image * fix lint issues * fix a failing url test * fix a failing url test * add build images for drone ci * Adding labelmap file to complex record to labels Signed-off-by: Cyril Tovena * Add helm chart and fix a bug with labelmap * New push path * dep sync * Move fluent-bit folder and use jsonnet for drone * fix build * Adds more documentation and include a basic config in the container Signed-off-by: Cyril Tovena * remove binaries * Some readme grammar fixes * docs: really nitpicky grammar change in clients README * Add helm documentation * fixes bad config * bump chart * fixes .gitignore --- charts/README.md | 16 +++ charts/fluent-bit/.helmignore | 22 ++++ charts/fluent-bit/Chart.yaml | 13 +++ charts/fluent-bit/README.md | 104 ++++++++++++++++++ charts/fluent-bit/templates/NOTES.txt | 3 + charts/fluent-bit/templates/_helpers.tpl | 66 +++++++++++ charts/fluent-bit/templates/clusterrole.yaml | 18 +++ .../templates/clusterrolebinding.yaml | 19 ++++ charts/fluent-bit/templates/configmap.yaml | 52 +++++++++ charts/fluent-bit/templates/daemonset.yaml | 74 +++++++++++++ .../templates/service-headless.yaml | 22 ++++ .../fluent-bit/templates/serviceaccount.yaml | 12 ++ .../fluent-bit/templates/servicemonitor.yaml | 30 +++++ charts/fluent-bit/values.yaml | 96 ++++++++++++++++ charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/requirements.yaml | 6 +- charts/loki-stack/values.yaml | 3 + 17 files changed, 556 insertions(+), 2 deletions(-) create mode 100644 charts/fluent-bit/.helmignore create mode 100644 charts/fluent-bit/Chart.yaml create mode 100644 charts/fluent-bit/README.md create mode 100644 charts/fluent-bit/templates/NOTES.txt create mode 100644 charts/fluent-bit/templates/_helpers.tpl create mode 100644 charts/fluent-bit/templates/clusterrole.yaml create mode 100644 charts/fluent-bit/templates/clusterrolebinding.yaml create mode 100644 charts/fluent-bit/templates/configmap.yaml create mode 100644 charts/fluent-bit/templates/daemonset.yaml create mode 100644 charts/fluent-bit/templates/service-headless.yaml create mode 100644 charts/fluent-bit/templates/serviceaccount.yaml create mode 100644 charts/fluent-bit/templates/servicemonitor.yaml create mode 100644 charts/fluent-bit/values.yaml diff --git a/charts/README.md b/charts/README.md index cf77f9aac9..5f850bc767 100644 --- a/charts/README.md +++ b/charts/README.md @@ -48,6 +48,20 @@ $ helm upgrade --install loki loki/loki $ helm upgrade --install promtail loki/promtail --set "loki.serviceName=loki" ``` +## Deploy Loki and fluent-bit to your cluster + +```bash +$ helm upgrade --install loki loki/loki-stack \ + --set fluent-bit.enabled=true,promtail.enabled=false +``` + +## Deploy fluent-bit only + +```bash +$ helm upgrade --install fluent-bit loki/fluent-bit \ + --set "loki.serviceName=loki.svc.cluster.local" +``` + ## Deploy Grafana to your cluster To install Grafana on your cluster with helm, use the following command: @@ -56,6 +70,8 @@ To install Grafana on your cluster with helm, use the following command: $ helm install stable/grafana -n loki-grafana ``` +> The chart loki-stack contains a pre-configured Grafana, simply use `--set grafana.enabled=true` + To get the admin password for the Grafana pod, run the following command: ```bash diff --git a/charts/fluent-bit/.helmignore b/charts/fluent-bit/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/charts/fluent-bit/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml new file mode 100644 index 0000000000..ae9b2f8d35 --- /dev/null +++ b/charts/fluent-bit/Chart.yaml @@ -0,0 +1,13 @@ +name: fluent-bit +version: 0.0.1 +appVersion: v0.0.1 +kubeVersion: "^1.10.0-0" +description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" +home: https://grafana.com/loki +icon: https://github.com/grafana/loki/raw/master/docs/logo.png +sources: +- https://github.com/grafana/loki +maintainers: +- name: Loki Maintainers + email: lokiproject@googlegroups.com +engine: gotpl diff --git a/charts/fluent-bit/README.md b/charts/fluent-bit/README.md new file mode 100644 index 0000000000..c85935a971 --- /dev/null +++ b/charts/fluent-bit/README.md @@ -0,0 +1,104 @@ +# Fluent-bit Loki chart + +This chart install the fluent-bit application to ship logs to Loki. It defines daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Installing the Chart + +> If you don't have `Helm` installed locally, or `Tiller` installed in your Kubernetes cluster, read the [Using Helm](https://docs.helm.sh/using_helm/) documentation to get started. +To install the chart with the release name `my-release` using our helm repository: + +```bash +helm repo add loki https://grafana.github.io/loki/charts +helm upgrade --install my-release loki/fluent-bit \ + --set loki.serviceName=loki.svc.cluster.local +``` + +The command deploys fluent-bit on the Kubernetes cluster with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +To configure the chart to send to [Grafana Cloud](https://grafana.com/products/cloud) use: + +```bash +helm upgrade --install my-release loki/fluent-bit \ + --set loki.serviceName=logs-us-west1.grafana.net,loki.servicePort=80,loki.serviceScheme=https \ + --set loki.user=2830,loki.password=1234 +``` + +> **Tip**: List all releases using `helm list` + +To install a custom tag use the following command: + +```bash +helm upgrade --install my-release loki/fluent-bit \ + --set image.tag= +``` + +The full list of available tags on [docker hub](https://cloud.docker.com/u/grafana/repository/docker/grafana/fluent-bit-plugin-loki). + +Alternatively you can install the full [Loki stack](../loki-stack) (Loki + fluent-bit) using: + +```bash +helm upgrade --install my-release loki/loki-stack \ + --set fluent-bit.enabled=true,promtail.enabled=false +``` + +This will automatically configured the `loki.serviceName` configuration field to the newly created Loki instance. + +## RBAC + +By default, `rbac.create` is set to true. This enable RBAC support in fluent-bit and must be true if RBAC is enabled in your cluster. + +The chart will take care of creating the required service accounts and roles for fluent-bit. + +If you have RBAC disabled, or to put it another way, ABAC enabled, you should set this value to `false`. + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```bash +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following tables lists the configurable parameters of the fluent-bit chart and their default values. + +For more details, read the [fluent-bit documentation](../../../cmd/fluent-bit/README.md) + +| Parameter | Description | Default | +|--------------------------|----------------------------------------------------------------------------------------------------|----------------------------------| +| `loki.serviceName` | The address of the Loki service. | `"${RELEASE}-loki"` | +| `loki.servicePort` | The port of the Loki service. | `3100` | +| `loki.serviceScheme` | The scheme of the Loki service. | `http` | +| `loki.user` | The http basic auth username to access the Loki service. | | +| `loki.password` | The http basic auth password to access the Loki service. | | +| `config.port` | the fluent-bit port to listen. (This is mainly used to serve metrics) | `2020` | +| `config.loglevel` | the fluent-bit log level (debug,info,warn,error). | `warn` | +| `config.lineFormat` | The line format to use to send a record (json/key_value) | `json` | +| `config.removeKeys` | The list of key to remove from each record | `[removeKeys,stream]` | +| `config.labels` | A set of labels to send for every log | `'{job="fluent-bit"}'` | +| `config.labelMap` | Mapping of labels from a record. See [fluent-bit documentation](../../../cmd/fluent-bit/README.md) | | +| `affinity` | [affinity][affinity] settings for pod assignment | `{}` | +| `annotations` | Annotations to add to Kubernetes resources. | `{}` | +| `deploymentStrategy` | The deployment strategy to use with the daemonset | `RollingUpdate` | +| `image.repository` | The fluent-bit docker image repository | `grafana/fluent-bit-plugin-loki` | +| `image.tag` | The fluent-bit docker image tag | `0.1` | +| `image.pullPolicy` | The fluent-bit docker image pull policy | `IfNotPresent` | +| `nodeSelector` | fluent-bit [node labels][nodeSelector] for pod assignment | `{}` | +| `podLabels` | additional fluent-bit pod labels | `{}` | +| `podAnnotations` | additional fluent-bit pod annotations | `Prometheus discovery` | +| `rbac.create` | Activate support for RBAC | `true` | +| `resources` | Resource requests/limit | | +| `tolerations` | [Toleration][toleration] labels for pod assignment | `no schedule on master nodes` | +| `volumes` | [Volume]([volumes]) to mount | `host containers log` | +| `volumeMounts` | Volume mount mapping | | +| `serviceMonitor.enabled` | Create a [Prometheus Operator](operator) serviceMonitor resource for fluent-bit | `false` | + + +[toleration]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +[nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector +[affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +[volumes]: https://kubernetes.io/docs/concepts/storage/volumes/ +[operator]: https://github.com/coreos/prometheus-operator diff --git a/charts/fluent-bit/templates/NOTES.txt b/charts/fluent-bit/templates/NOTES.txt new file mode 100644 index 0000000000..bbb70770b8 --- /dev/null +++ b/charts/fluent-bit/templates/NOTES.txt @@ -0,0 +1,3 @@ +Verify the application is working by running these commands: + kubectl --namespace {{ .Release.Namespace }} port-forward daemonset/{{ include "fluent-bit-loki.fullname" . }} {{ .Values.config.port }} + curl http://127.0.0.1:{{ .Values.config.port }}/api/v1/metrics/prometheus diff --git a/charts/fluent-bit/templates/_helpers.tpl b/charts/fluent-bit/templates/_helpers.tpl new file mode 100644 index 0000000000..ddbbe85a1f --- /dev/null +++ b/charts/fluent-bit/templates/_helpers.tpl @@ -0,0 +1,66 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "fluent-bit-loki.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "fluent-bit-loki.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "fluent-bit-loki.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "fluent-bit-loki.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "fluent-bit-loki.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +The service name to connect to Loki. Defaults to the same logic as "loki.fullname" +*/}} +{{- define "loki.serviceName" -}} +{{- if .Values.loki.serviceName -}} +{{- .Values.loki.serviceName -}} +{{- else if .Values.loki.fullnameOverride -}} +{{- .Values.loki.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "loki" .Values.loki.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "helm-toolkit.utils.joinListWithComma" -}} +{{- $local := dict "first" true -}} +{{- range $k, $v := . -}}{{- if not $local.first -}},{{- end -}}{{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}} +{{- end -}} diff --git a/charts/fluent-bit/templates/clusterrole.yaml b/charts/fluent-bit/templates/clusterrole.yaml new file mode 100644 index 0000000000..ac9239afa0 --- /dev/null +++ b/charts/fluent-bit/templates/clusterrole.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.create }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "fluent-bit-loki.fullname" . }}-clusterrole + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: ["get", "watch", "list"] +{{- end }} diff --git a/charts/fluent-bit/templates/clusterrolebinding.yaml b/charts/fluent-bit/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..f825c97bf6 --- /dev/null +++ b/charts/fluent-bit/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "fluent-bit-loki.fullname" . }}-clusterrolebinding + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ template "fluent-bit-loki.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "fluent-bit-loki.fullname" . }}-clusterrole + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml new file mode 100644 index 0000000000..57c44dcd12 --- /dev/null +++ b/charts/fluent-bit/templates/configmap.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "fluent-bit-loki.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + fluent-bit.conf: |- + [SERVICE] + Flush 1 + Daemon Off + Log_Level {{ .Values.config.loglevel }} + Parsers_File parsers.conf + [INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*.log + Parser docker + DB /run/fluent-bit/flb_kube.db + Mem_Buf_Limit 5MB + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Merge_Log On + [Output] + Name loki + Match * + {{- if and .Values.loki.user .Values.loki.password }} + Url {{ .Values.loki.serviceScheme }}://{{ .Values.loki.user }}:{{ .Values.loki.password }}@{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push + {{- else }} + Url {{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push + {{- end }} + Labels {{ .Values.config.labels }} + RemoveKeys {{ include "helm-toolkit.utils.joinListWithComma" .Values.config.removeKeys }} + LabelMapPath /fluent-bit/etc/labelmap.json + LineFormat {{ .Values.config.lineFormat }} + LogLevel {{ .Values.config.loglevel }} + + parsers.conf: |- + [PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + + labelmap.json: |- + {{- .Values.config.labelMap | toPrettyJson | nindent 4}} diff --git a/charts/fluent-bit/templates/daemonset.yaml b/charts/fluent-bit/templates/daemonset.yaml new file mode 100644 index 0000000000..e57b53232c --- /dev/null +++ b/charts/fluent-bit/templates/daemonset.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "fluent-bit-loki.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + {{- toYaml .Values.annotations | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "fluent-bit-loki.name" . }} + release: {{ .Release.Name }} + updateStrategy: + type: {{ .Values.deploymentStrategy }} + {{- if ne .Values.deploymentStrategy "RollingUpdate" }} + rollingUpdate: null + {{- end }} + template: + metadata: + labels: + app: {{ template "fluent-bit-loki.name" . }} + release: {{ .Release.Name }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "fluent-bit-loki.serviceAccountName" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + containers: + - name: fluent-bit-loki + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: config + mountPath: /fluent-bit/etc + - name: run + mountPath: /run/fluent-bit + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - containerPort: {{ .Values.config.port }} + name: http-metrics + resources: + {{- toYaml .Values.resources | nindent 12 }} + nodeSelector: + {{- toYaml .Values.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.tolerations | nindent 8 }} + terminationGracePeriodSeconds: 10 + volumes: + - name: config + configMap: + name: {{ template "fluent-bit-loki.fullname" . }} + - name: run + hostPath: + path: /run/fluent-bit + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/fluent-bit/templates/service-headless.yaml b/charts/fluent-bit/templates/service-headless.yaml new file mode 100644 index 0000000000..332d1063d1 --- /dev/null +++ b/charts/fluent-bit/templates/service-headless.yaml @@ -0,0 +1,22 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "fluent-bit-loki.fullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + clusterIP: None + ports: + - port: {{ .Values.config.port }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + selector: + app: {{ template "fluent-bit-loki.name" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/charts/fluent-bit/templates/serviceaccount.yaml b/charts/fluent-bit/templates/serviceaccount.yaml new file mode 100644 index 0000000000..7933b61e97 --- /dev/null +++ b/charts/fluent-bit/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "fluent-bit-loki.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/fluent-bit/templates/servicemonitor.yaml b/charts/fluent-bit/templates/servicemonitor.yaml new file mode 100644 index 0000000000..5507bb1087 --- /dev/null +++ b/charts/fluent-bit/templates/servicemonitor.yaml @@ -0,0 +1,30 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "fluent-bit-loki.fullname" . }} + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.serviceMonitor.additionalLabels }} +{{ toYaml .Values.serviceMonitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app: {{ template "fluent-bit-loki.name" . }} + release: {{ .Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + endpoints: + - port: http-metrics + {{- if .Values.serviceMonitor.interval }} + interval: {{ .Values.serviceMonitor.interval }} + {{- end }} + {{- if .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + {{- end }} +{{- end }} diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml new file mode 100644 index 0000000000..1fabc811a3 --- /dev/null +++ b/charts/fluent-bit/values.yaml @@ -0,0 +1,96 @@ +--- +loki: + serviceName: "" # Defaults to "${RELEASE}-loki" if not set + servicePort: 3100 + serviceScheme: http + # user: user + # password: pass +config: + port: 2020 + loglevel: warn + lineFormat: json + removeKeys: + - kubernetes + - stream + labels: '{job="fluent-bit"}' + labelMap: + kubernetes: + namespace_name: namespace + labels: + app: app + release: release + host: node + container_name: container + pod_name: instance + stream: stream + +affinity: {} + +annotations: {} + +deploymentStrategy: RollingUpdate + +image: + repository: grafana/fluent-bit-plugin-loki + tag: 0.1 + pullPolicy: IfNotPresent + +nameOverride: fluent-bit-loki + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Pod Labels +podLabels: {} + +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "2020" + prometheus.io/path: /api/v1/metrics/prometheus + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +rbac: + create: true + pspEnabled: true + +resources: + limits: + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + +serviceAccount: + create: true + name: + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: +- key: node-role.kubernetes.io/master + effect: NoSchedule + +# Extra volumes to scrape logs from +volumes: +- name: varlog + hostPath: + path: /var/log +- name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + +volumeMounts: +- name: varlog + mountPath: /var/log +- name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + +serviceMonitor: + enabled: false + interval: "" + additionalLabels: {} + # scrapeTimeout: 10s diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index ed11e9e5f2..8699507193 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.16.4 +version: 0.16.5 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/requirements.yaml b/charts/loki-stack/requirements.yaml index 2554a0ee58..62807f647b 100644 --- a/charts/loki-stack/requirements.yaml +++ b/charts/loki-stack/requirements.yaml @@ -7,6 +7,10 @@ dependencies: condition: promtail.enabled repository: "file://../promtail" version: "^0.6.0" +- name: "fluent-bit" + condition: fluent-bit.enabled + repository: "file://../fluent-bit" + version: "^0.0.1" - name: "grafana" condition: grafana.enabled version: "~3.4.3" @@ -14,4 +18,4 @@ dependencies: - name: "prometheus" condition: prometheus.enabled version: "~8.11.2" - repository: "https://kubernetes-charts.storage.googleapis.com/" \ No newline at end of file + repository: "https://kubernetes-charts.storage.googleapis.com/" diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index 8d6fc56a80..40c4197187 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -4,6 +4,9 @@ loki: promtail: enabled: true +fluent-bit: + enabled: false + grafana: enabled: false sidecar: From 7fe67cfedf2405e96796d9ca11816c2fc38da1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20Barrera?= Date: Tue, 1 Oct 2019 18:12:43 +0200 Subject: [PATCH 118/202] Feature: Add extra containers to loki helm chart (#909) * Add extra containers feature to loki chart * minor bump version to loki helm charts --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/service-headless.yaml | 3 +++ charts/loki/templates/service.yaml | 3 +++ charts/loki/templates/statefulset.yaml | 6 +++++ charts/loki/values.yaml | 29 +++++++++++++++++++++ 6 files changed, 43 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 8699507193..06558b37fc 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.16.5 +version: 0.17.0 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 3936cc566c..30973e4b98 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.14.3 +version: 0.15.0 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/service-headless.yaml b/charts/loki/templates/service-headless.yaml index 5536c39840..0c0a0eee5b 100644 --- a/charts/loki/templates/service-headless.yaml +++ b/charts/loki/templates/service-headless.yaml @@ -15,6 +15,9 @@ spec: protocol: TCP name: http-metrics targetPort: http-metrics +{{- if .Values.extraPorts }} +{{ toYaml .Values.extraPorts | indent 4}} +{{- end }} selector: app: {{ template "loki.name" . }} release: {{ .Release.Name }} diff --git a/charts/loki/templates/service.yaml b/charts/loki/templates/service.yaml index 987da1989a..0df957409f 100644 --- a/charts/loki/templates/service.yaml +++ b/charts/loki/templates/service.yaml @@ -25,6 +25,9 @@ spec: targetPort: http-metrics {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} nodePort: {{ .Values.service.nodePort }} +{{- end }} +{{- if .Values.extraPorts }} +{{ toYaml .Values.extraPorts | indent 4}} {{- end }} selector: app: {{ template "loki.name" . }} diff --git a/charts/loki/templates/statefulset.yaml b/charts/loki/templates/statefulset.yaml index ac2661cdfa..4337da1d76 100644 --- a/charts/loki/templates/statefulset.yaml +++ b/charts/loki/templates/statefulset.yaml @@ -73,6 +73,9 @@ spec: - name: JAEGER_AGENT_HOST value: "{{ .Values.tracing.jaegerAgentHost }}" {{- end }} +{{- if .Values.extraContainers }} +{{ toYaml .Values.extraContainers | indent 8}} +{{- end }} nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }} affinity: @@ -84,6 +87,9 @@ spec: - name: config secret: secretName: {{ template "loki.fullname" . }} +{{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | indent 8}} +{{- end }} {{- if not .Values.persistence.enabled }} - name: storage emptyDir: {} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index c82c68e794..0b331c41b4 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -170,3 +170,32 @@ serviceMonitor: interval: "" additionalLabels: {} # scrapeTimeout: 10s + +extraContainers: [] +## Additional containers to be added to the loki pod. +# - name: reverse-proxy +# image: angelbarrera92/basic-auth-reverse-proxy:dev +# args: +# - "serve" +# - "--upstream=http://localhost:3100" +# - "--auth-config=/etc/reverse-proxy-conf/authn.yaml" +# ports: +# - name: http +# containerPort: 11811 +# protocol: TCP +# volumeMounts: +# - name: reverse-proxy-auth-config +# mountPath: /etc/reverse-proxy-conf + +extraVolumes: [] +## Additional volumes to the loki pod. +# - name: reverse-proxy-auth-config +# secret: +# secretName: reverse-proxy-auth-config + +extraPorts: [] +## Additional ports to the loki services. Useful to expose extra container ports. +# - port: 11811 +# protocol: TCP +# name: http +# targetPort: http From edcda84a7954549bcf13b7d8adec29433a9312e0 Mon Sep 17 00:00:00 2001 From: Marcos Nils Date: Thu, 3 Oct 2019 14:57:59 +0100 Subject: [PATCH 119/202] Bump grafana image (#1110) --- charts/loki-stack/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index 40c4197187..1e2c923844 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -13,7 +13,7 @@ grafana: datasources: enabled: true image: - tag: 6.3.0-beta2 + tag: 6.4.1 prometheus: enabled: false From 946b55a55a94893949f47f37cb075da69d703ce9 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Mon, 7 Oct 2019 14:24:39 +0200 Subject: [PATCH 120/202] Increased promtail's backoff settings in prod and improved doc (#1083) --- charts/promtail/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index edfbf521fd..b5acc2c097 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -113,7 +113,7 @@ config: # Maximum backoff time between retries maxbackoff: 5s # Maximum number of retries when sending batches, 0 means infinite retries - maxretries: 5 + maxretries: 20 # The labels to add to any time series or alerts when communicating with loki external_labels: {} From da53f331666d654aa611b68ec210ab33aa01b2b0 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Tue, 8 Oct 2019 15:41:41 +0200 Subject: [PATCH 121/202] docs: fix Promtail / Loki capitalization (#1130) * docs: fixed Promtail capitalization * docs: fixed Loki capitalization --- charts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index 5f850bc767..3cc7cc3173 100644 --- a/charts/README.md +++ b/charts/README.md @@ -92,7 +92,7 @@ Then follow the [instructions for adding the loki datasource](/docs/querying.md# If Loki and Promtail are deployed on different clusters you can add an Ingress in front of Loki. By adding a certificate you create an https endpoint. For extra security enable basic authentication on the Ingress. -In promtail set the following values to communicate with https and basic auth +In Promtail set the following values to communicate with https and basic auth ``` loki: From b22f3ea16333ffcbd4c1f84601f4a72a1933236c Mon Sep 17 00:00:00 2001 From: George Adams Date: Wed, 9 Oct 2019 14:49:16 -0400 Subject: [PATCH 122/202] Client recommendations documentation tweaks (#1124) * consistent use of proper noun for Fluent Bit * add notes about metrics extraction to client selection docs --- charts/README.md | 4 ++-- charts/fluent-bit/README.md | 36 ++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/charts/README.md b/charts/README.md index 3cc7cc3173..92499f3152 100644 --- a/charts/README.md +++ b/charts/README.md @@ -48,14 +48,14 @@ $ helm upgrade --install loki loki/loki $ helm upgrade --install promtail loki/promtail --set "loki.serviceName=loki" ``` -## Deploy Loki and fluent-bit to your cluster +## Deploy Loki and Fluent Bit to your cluster ```bash $ helm upgrade --install loki loki/loki-stack \ --set fluent-bit.enabled=true,promtail.enabled=false ``` -## Deploy fluent-bit only +## Deploy Fluent Bit only ```bash $ helm upgrade --install fluent-bit loki/fluent-bit \ diff --git a/charts/fluent-bit/README.md b/charts/fluent-bit/README.md index c85935a971..a9bb505b3e 100644 --- a/charts/fluent-bit/README.md +++ b/charts/fluent-bit/README.md @@ -1,6 +1,6 @@ -# Fluent-bit Loki chart +# Fluent Bit Loki chart -This chart install the fluent-bit application to ship logs to Loki. It defines daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. +This chart install the Fluent Bit application to ship logs to Loki. It defines daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. ## Installing the Chart @@ -13,7 +13,7 @@ helm upgrade --install my-release loki/fluent-bit \ --set loki.serviceName=loki.svc.cluster.local ``` -The command deploys fluent-bit on the Kubernetes cluster with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. +The command deploys Fluent Bit on the Kubernetes cluster with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. To configure the chart to send to [Grafana Cloud](https://grafana.com/products/cloud) use: @@ -34,7 +34,7 @@ helm upgrade --install my-release loki/fluent-bit \ The full list of available tags on [docker hub](https://cloud.docker.com/u/grafana/repository/docker/grafana/fluent-bit-plugin-loki). -Alternatively you can install the full [Loki stack](../loki-stack) (Loki + fluent-bit) using: +Alternatively you can install the full [Loki stack](../loki-stack) (Loki + Fluent Bit) using: ```bash helm upgrade --install my-release loki/loki-stack \ @@ -45,9 +45,9 @@ This will automatically configured the `loki.serviceName` configuration field to ## RBAC -By default, `rbac.create` is set to true. This enable RBAC support in fluent-bit and must be true if RBAC is enabled in your cluster. +By default, `rbac.create` is set to true. This enable RBAC support in Fluent Bit and must be true if RBAC is enabled in your cluster. -The chart will take care of creating the required service accounts and roles for fluent-bit. +The chart will take care of creating the required service accounts and roles for Fluent Bit. If you have RBAC disabled, or to put it another way, ABAC enabled, you should set this value to `false`. @@ -63,9 +63,9 @@ The command removes all the Kubernetes components associated with the chart and ## Configuration -The following tables lists the configurable parameters of the fluent-bit chart and their default values. +The following tables lists the configurable parameters of the Fluent Bit chart and their default values. -For more details, read the [fluent-bit documentation](../../../cmd/fluent-bit/README.md) +For more details, read the [Fluent Bit documentation](../../../cmd/fluent-bit/README.md) | Parameter | Description | Default | |--------------------------|----------------------------------------------------------------------------------------------------|----------------------------------| @@ -74,27 +74,27 @@ For more details, read the [fluent-bit documentation](../../../cmd/fluent-bit/RE | `loki.serviceScheme` | The scheme of the Loki service. | `http` | | `loki.user` | The http basic auth username to access the Loki service. | | | `loki.password` | The http basic auth password to access the Loki service. | | -| `config.port` | the fluent-bit port to listen. (This is mainly used to serve metrics) | `2020` | -| `config.loglevel` | the fluent-bit log level (debug,info,warn,error). | `warn` | +| `config.port` | the Fluent Bit port to listen. (This is mainly used to serve metrics) | `2020` | +| `config.loglevel` | the Fluent Bit log level (debug,info,warn,error). | `warn` | | `config.lineFormat` | The line format to use to send a record (json/key_value) | `json` | | `config.removeKeys` | The list of key to remove from each record | `[removeKeys,stream]` | | `config.labels` | A set of labels to send for every log | `'{job="fluent-bit"}'` | -| `config.labelMap` | Mapping of labels from a record. See [fluent-bit documentation](../../../cmd/fluent-bit/README.md) | | +| `config.labelMap` | Mapping of labels from a record. See [Fluent Bit documentation](../../../cmd/fluent-bit/README.md) | | | `affinity` | [affinity][affinity] settings for pod assignment | `{}` | | `annotations` | Annotations to add to Kubernetes resources. | `{}` | | `deploymentStrategy` | The deployment strategy to use with the daemonset | `RollingUpdate` | -| `image.repository` | The fluent-bit docker image repository | `grafana/fluent-bit-plugin-loki` | -| `image.tag` | The fluent-bit docker image tag | `0.1` | -| `image.pullPolicy` | The fluent-bit docker image pull policy | `IfNotPresent` | -| `nodeSelector` | fluent-bit [node labels][nodeSelector] for pod assignment | `{}` | -| `podLabels` | additional fluent-bit pod labels | `{}` | -| `podAnnotations` | additional fluent-bit pod annotations | `Prometheus discovery` | +| `image.repository` | The Fluent Bit docker image repository | `grafana/fluent-bit-plugin-loki` | +| `image.tag` | The Fluent Bit docker image tag | `0.1` | +| `image.pullPolicy` | The Fluent Bit docker image pull policy | `IfNotPresent` | +| `nodeSelector` | Fluent Bit [node labels][nodeSelector] for pod assignment | `{}` | +| `podLabels` | additional Fluent Bit pod labels | `{}` | +| `podAnnotations` | additional Fluent Bit pod annotations | `Prometheus discovery` | | `rbac.create` | Activate support for RBAC | `true` | | `resources` | Resource requests/limit | | | `tolerations` | [Toleration][toleration] labels for pod assignment | `no schedule on master nodes` | | `volumes` | [Volume]([volumes]) to mount | `host containers log` | | `volumeMounts` | Volume mount mapping | | -| `serviceMonitor.enabled` | Create a [Prometheus Operator](operator) serviceMonitor resource for fluent-bit | `false` | +| `serviceMonitor.enabled` | Create a [Prometheus Operator](operator) serviceMonitor resource for Fluent Bit | `false` | [toleration]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ From 7173bcd2ef086a0a1f127a67b58fa9d9d8c85611 Mon Sep 17 00:00:00 2001 From: Endre Karlson Date: Wed, 9 Oct 2019 21:51:54 +0200 Subject: [PATCH 123/202] helm: Add support for passing arbitrary secrets (#1111) * Add support for adding secrets and mounting * Add possibility to pass arbitrary env * Bump chart to 0.16.0 * bump chart Signed-off-by: Cyril Tovena Co-authored-by: Cyril Tovena --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/statefulset.yaml | 6 ++++++ charts/loki/values.yaml | 7 +++++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 06558b37fc..e4737e0190 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.17.0 +version: 0.17.1 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 30973e4b98..80ed058550 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,5 @@ name: loki -version: 0.15.0 +version: 0.16.0 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/statefulset.yaml b/charts/loki/templates/statefulset.yaml index 4337da1d76..dd486cf3fc 100644 --- a/charts/loki/templates/statefulset.yaml +++ b/charts/loki/templates/statefulset.yaml @@ -51,6 +51,9 @@ spec: - "-{{ $key }}={{ $value }}" {{- end }} volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 12}} + {{- end }} - name: config mountPath: /etc/loki - name: storage @@ -69,6 +72,9 @@ spec: securityContext: readOnlyRootFilesystem: true env: + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} {{- if .Values.tracing.jaegerAgentHost }} - name: JAEGER_AGENT_HOST value: "{{ .Values.tracing.jaegerAgentHost }}" diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 0b331c41b4..7088515eb2 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -187,15 +187,22 @@ extraContainers: [] # - name: reverse-proxy-auth-config # mountPath: /etc/reverse-proxy-conf + extraVolumes: [] ## Additional volumes to the loki pod. # - name: reverse-proxy-auth-config # secret: # secretName: reverse-proxy-auth-config +## Extra volume mounts that will be added to the loki container +extraVolumeMounts: [] + extraPorts: [] ## Additional ports to the loki services. Useful to expose extra container ports. # - port: 11811 # protocol: TCP # name: http # targetPort: http + +# Extra env variables to pass to the loki container +env: [] From e4715c19078879bb5d9c1284e71bef870a0d90ba Mon Sep 17 00:00:00 2001 From: Minh Danh Date: Thu, 10 Oct 2019 21:38:53 +0700 Subject: [PATCH 124/202] Helm chart: Allow additional scrape_configs to be added (#1134) * Allow additional scrape_configs to be added * Bump version of loki-stack chart --- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/configmap.yaml | 3 +++ charts/promtail/values.yaml | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index cf53859624..12fd854ffd 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,5 +1,5 @@ name: promtail -version: 0.12.2 +version: 0.12.3 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/configmap.yaml b/charts/promtail/templates/configmap.yaml index 234aa2b43f..debc1480e9 100644 --- a/charts/promtail/templates/configmap.yaml +++ b/charts/promtail/templates/configmap.yaml @@ -262,3 +262,6 @@ data: - __meta_kubernetes_pod_container_name target_label: __path__ {{- end }} + {{- if .Values.extraScrapeConfigs }} + {{- toYaml .Values.extraScrapeConfigs | nindent 4 }} + {{- end }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index b5acc2c097..959bf5424d 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -64,6 +64,9 @@ resources: {} # Custom scrape_configs to override the default ones in the configmap scrapeConfigs: [] +# Custom scrape_configs together with the default ones in the configmap +extraScrapeConfigs: [] + securityContext: readOnlyRootFilesystem: true runAsGroup: 0 From 3cf5449b82e4d1bd78c1c6d73ec6a155ba634819 Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Tue, 15 Oct 2019 10:36:34 +0800 Subject: [PATCH 125/202] doc(fluent-bit): add missing namespace (#1077) * doc(fluent-bit): add missing namespace Signed-off-by: Xiang Dai <764524258@qq.com> * fix a nit Signed-off-by: Xiang Dai <764524258@qq.com> --- charts/fluent-bit/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/fluent-bit/README.md b/charts/fluent-bit/README.md index a9bb505b3e..007591b5a5 100644 --- a/charts/fluent-bit/README.md +++ b/charts/fluent-bit/README.md @@ -10,9 +10,11 @@ To install the chart with the release name `my-release` using our helm repositor ```bash helm repo add loki https://grafana.github.io/loki/charts helm upgrade --install my-release loki/fluent-bit \ - --set loki.serviceName=loki.svc.cluster.local + --set loki.serviceName=loki.default.svc.cluster.local ``` +If you deploy Loki with a custom namespace or service name, you must change the value above for `loki.serviceName` to the appropriate value. + The command deploys Fluent Bit on the Kubernetes cluster with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. To configure the chart to send to [Grafana Cloud](https://grafana.com/products/cloud) use: From 129d352eab850b716200d7b13b502312fb4ebd97 Mon Sep 17 00:00:00 2001 From: Nic Roland Date: Tue, 15 Oct 2019 05:24:38 -0700 Subject: [PATCH 126/202] docs(helm): fix broken link to grafana datasource (#1152) --- charts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index 92499f3152..d2a3055b0e 100644 --- a/charts/README.md +++ b/charts/README.md @@ -85,7 +85,7 @@ $ kubectl port-forward --namespace service/loki-grafana 3000:80 ``` Navigate to http://localhost:3000 and login with `admin` and the password output above. -Then follow the [instructions for adding the loki datasource](/docs/querying.md#grafana), using the URL `http://loki:3100/`. +Then follow the [instructions for adding the loki datasource](/docs/getting-started/grafana.md), using the URL `http://loki:3100/`. ## Run Loki behind https ingress From d774d0ad7f3014374a6bfcbf4b1c1f75e043ccbc Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 15 Oct 2019 08:28:31 -0400 Subject: [PATCH 127/202] rollback fluent-bit push path until we release 0.4 --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/templates/configmap.yaml | 4 ++-- charts/fluent-bit/values.yaml | 1 + charts/loki-stack/Chart.yaml | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index ae9b2f8d35..909edfd06d 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,5 +1,5 @@ name: fluent-bit -version: 0.0.1 +version: 0.0.2 appVersion: v0.0.1 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml index 57c44dcd12..551e0d9362 100644 --- a/charts/fluent-bit/templates/configmap.yaml +++ b/charts/fluent-bit/templates/configmap.yaml @@ -31,9 +31,9 @@ data: Name loki Match * {{- if and .Values.loki.user .Values.loki.password }} - Url {{ .Values.loki.serviceScheme }}://{{ .Values.loki.user }}:{{ .Values.loki.password }}@{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push + Url {{ .Values.loki.serviceScheme }}://{{ .Values.loki.user }}:{{ .Values.loki.password }}@{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}{{ .Values.loki.servicePath }} {{- else }} - Url {{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push + Url {{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}{{ .Values.loki.servicePath }} {{- end }} Labels {{ .Values.config.labels }} RemoveKeys {{ include "helm-toolkit.utils.joinListWithComma" .Values.config.removeKeys }} diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index 1fabc811a3..7de07d8929 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -3,6 +3,7 @@ loki: serviceName: "" # Defaults to "${RELEASE}-loki" if not set servicePort: 3100 serviceScheme: http + servicePath: /api/prom/push # user: user # password: pass config: diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index e4737e0190..4f006b3cb1 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.17.1 +version: 0.17.2 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." From 47283fb2af6fe5ed15b494c375e2bd9b1a27dd75 Mon Sep 17 00:00:00 2001 From: Jeff Valore Date: Fri, 18 Oct 2019 17:10:40 -0400 Subject: [PATCH 128/202] Add release name to prometheus service name. (#1174) --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/templates/_helpers.tpl | 8 ++++++++ charts/loki-stack/templates/datasources.yaml | 2 +- charts/loki-stack/values.yaml | 2 -- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 4f006b3cb1..eb14e6690a 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,5 @@ name: loki-stack -version: 0.17.2 +version: 0.17.3 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/templates/_helpers.tpl b/charts/loki-stack/templates/_helpers.tpl index 9c9e517686..1543db74a2 100644 --- a/charts/loki-stack/templates/_helpers.tpl +++ b/charts/loki-stack/templates/_helpers.tpl @@ -30,3 +30,11 @@ Create chart name and version as used by the chart label. {{- define "loki-stack.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Override the naming defined by the prometheus chart. +Added as a fix for https://github.com/grafana/loki/issues/1169 +*/}} +{{- define "prometheus.fullname" -}} +{{- printf "%s-%s" .Release.Name "prometheus-server" | trunc 63 | trimSuffix "-" -}} +{{- end -}} \ No newline at end of file diff --git a/charts/loki-stack/templates/datasources.yaml b/charts/loki-stack/templates/datasources.yaml index cec67f19bf..c0e73f03eb 100644 --- a/charts/loki-stack/templates/datasources.yaml +++ b/charts/loki-stack/templates/datasources.yaml @@ -25,7 +25,7 @@ data: - name: Prometheus type: prometheus access: proxy - url: http://{{ .Values.prometheus.server.fullnameOverride }}:{{ .Values.prometheus.server.service.servicePort }} + url: http://{{ include "prometheus.fullname" .}}:{{ .Values.prometheus.server.service.servicePort }} version: 1 {{- end }} {{- end }} diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index 1e2c923844..eae38fe654 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -17,5 +17,3 @@ grafana: prometheus: enabled: false - server: - fullnameOverride: prometheus-server From 92f3a9429a620cee971ac9c19fcf02a40e278b0f Mon Sep 17 00:00:00 2001 From: polar3130 <47264004+polar3130@users.noreply.github.com> Date: Fri, 25 Oct 2019 00:08:00 +0900 Subject: [PATCH 129/202] Helm: Fix the reference error of loki/loki helm chart (#1056) * fix a reference error of loki/loki helm chart add new valuable "client.name" instead of "promtail.name" * bump loki-stack chart * Add missing chart version Co-authored-by: cyriltovena --- charts/fluent-bit/Chart.yaml | 1 + charts/loki-stack/Chart.yaml | 3 ++- charts/loki/Chart.yaml | 3 ++- charts/loki/templates/_helpers.tpl | 18 ++++++++++++++++++ charts/loki/templates/networkpolicy.yaml | 2 +- charts/loki/values.yaml | 4 ++++ charts/promtail/Chart.yaml | 1 + 7 files changed, 29 insertions(+), 3 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index 909edfd06d..cdeff1f45f 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,3 +1,4 @@ +apiVersion: "v1" name: fluent-bit version: 0.0.2 appVersion: v0.0.1 diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index eb14e6690a..3d804f8fd0 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,5 +1,6 @@ +apiVersion: "v1" name: loki-stack -version: 0.17.3 +version: 0.17.4 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 80ed058550..e2576ca10c 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,5 +1,6 @@ +apiVersion: "v1" name: loki -version: 0.16.0 +version: 0.16.1 appVersion: v0.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/_helpers.tpl b/charts/loki/templates/_helpers.tpl index 2e333aae60..5e12a470f1 100644 --- a/charts/loki/templates/_helpers.tpl +++ b/charts/loki/templates/_helpers.tpl @@ -41,3 +41,21 @@ Create the name of the service account {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Create the app name of loki clients. Defaults to the same logic as "loki.fullname", and default client expects "promtail". +*/}} +{{- define "client.name" -}} +{{- if .Values.client.name -}} +{{- .Values.client.name -}} +{{- else if .Values.client.fullnameOverride -}} +{{- .Values.client.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "promtail" .Values.client.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/loki/templates/networkpolicy.yaml b/charts/loki/templates/networkpolicy.yaml index dea85abcaf..31d24d7003 100644 --- a/charts/loki/templates/networkpolicy.yaml +++ b/charts/loki/templates/networkpolicy.yaml @@ -19,7 +19,7 @@ spec: - from: - podSelector: matchLabels: - app: {{ template "promtail.name" . }} + app: {{ template "client.name" . }} release: {{ .Release.Name }} - ports: - port: {{ .Values.service.port }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 7088515eb2..ca4a84cdce 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -84,6 +84,10 @@ livenessProbe: networkPolicy: enabled: false +## The app name of loki clients +client: {} + # name: + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 12fd854ffd..4ecc958a7c 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,3 +1,4 @@ +apiVersion: "v1" name: promtail version: 0.12.3 appVersion: v0.3.0 From c776952fec3bdf46d03d8007d7b1b9a9ce3ef591 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 24 Oct 2019 12:20:49 -0400 Subject: [PATCH 130/202] update versions to new release --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 3d804f8fd0..88d2882fb9 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.17.4 -appVersion: v0.3.0 +version: 0.18.0 +appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index e2576ca10c..3877af4a9f 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.16.1 -appVersion: v0.3.0 +version: 0.17.0 +appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index ca4a84cdce..25ae416e97 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -67,7 +67,7 @@ config: image: repository: grafana/loki - tag: v0.3.0 + tag: v0.4.0 pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 4ecc958a7c..63598ad994 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.12.3 -appVersion: v0.3.0 +version: 0.13.0 +appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 959bf5424d..be0ded75fa 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -8,7 +8,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail - tag: v0.3.0 + tag: v0.4.0 pullPolicy: IfNotPresent livenessProbe: {} From 5003546fce89a42221943eb1a1ea0c8f1be035c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rabenstein?= Date: Mon, 28 Oct 2019 18:21:27 +0100 Subject: [PATCH 131/202] Simplify regexp's (#1202) This is mostly to remove redundant anchoring (as it makes people believe that Prometheus's regexp's aren't anchored by default, which will backfire badly at some point), but it also removes useless grouping (again to avoid confusion). Signed-off-by: beorn7 --- charts/promtail/templates/configmap.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/promtail/templates/configmap.yaml b/charts/promtail/templates/configmap.yaml index debc1480e9..211279538f 100644 --- a/charts/promtail/templates/configmap.yaml +++ b/charts/promtail/templates/configmap.yaml @@ -28,7 +28,7 @@ data: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop - regex: ^$ + regex: '' source_labels: - __service__ - action: labelmap @@ -75,7 +75,7 @@ data: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop - regex: ^$ + regex: '' source_labels: - __service__ - action: labelmap @@ -118,7 +118,7 @@ data: - __meta_kubernetes_pod_label_name - __meta_kubernetes_pod_label_app - action: drop - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + regex: '[0-9a-z-.]+-[0-9a-f]{8,10}' source_labels: - __meta_kubernetes_pod_controller_name - source_labels: @@ -128,7 +128,7 @@ data: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop - regex: ^$ + regex: '' source_labels: - __service__ - action: labelmap @@ -171,11 +171,11 @@ data: - __meta_kubernetes_pod_label_name - __meta_kubernetes_pod_label_app - action: keep - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + regex: '[0-9a-z-.]+-[0-9a-f]{8,10}' source_labels: - __meta_kubernetes_pod_controller_name - action: replace - regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}' source_labels: - __meta_kubernetes_pod_controller_name target_label: __service__ @@ -183,7 +183,7 @@ data: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop - regex: ^$ + regex: '' source_labels: - __service__ - action: labelmap @@ -220,7 +220,7 @@ data: - role: pod relabel_configs: - action: drop - regex: ^$ + regex: '' source_labels: - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror - action: replace @@ -231,7 +231,7 @@ data: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop - regex: ^$ + regex: '' source_labels: - __service__ - action: labelmap From 727fdee8487dd3b554f0e8303cf0b030aad6d464 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 31 Oct 2019 09:33:38 -0400 Subject: [PATCH 132/202] Helm path update (#1061) * Updated helm chart push paths Signed-off-by: Joe Elliott * Updated helm chart versions Signed-off-by: Joe Elliott * Updated test to use new format when pushing Signed-off-by: Joe Elliott --- charts/loki-stack/Chart.yaml | 2 +- .../templates/tests/loki-test-configmap.yaml | 12 +++++++++++- charts/loki/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 4 ++-- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 88d2882fb9..138ff380a2 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.18.0 +version: 0.18.1 appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/templates/tests/loki-test-configmap.yaml b/charts/loki-stack/templates/tests/loki-test-configmap.yaml index 205fd774c7..3e53cfde12 100644 --- a/charts/loki-stack/templates/tests/loki-test-configmap.yaml +++ b/charts/loki-stack/templates/tests/loki-test-configmap.yaml @@ -30,7 +30,7 @@ data: jq -e '.streams[].entries | length >= 1' } - @test "Push log entry" { + @test "Push log entry legacy" { local timestamp=$(date -Iseconds -u | sed 's/UTC/.000000000+00:00/') local data=$(jq -n --arg timestamp "${timestamp}" '{"streams": [{"labels": "{app=\"loki-test\"}", "entries": [{"ts": $timestamp, "line": "foobar"}]}]}') @@ -40,3 +40,13 @@ data: jq -e '.streams[].entries[].line == "foobar"' } + @test "Push log entry" { + local timestamp=$(date +%s000000000) + local data=$(jq -n --arg timestamp "${timestamp}" '{"streams": [{"stream": {"app": "loki-test"}, "values": [[$timestamp, "foobar"]]}]}') + + curl -s -X POST -H "Content-Type: application/json" ${LOKI_URI}/loki/api/v1/push -d "${data}" + + curl -sG ${LOKI_URI}/api/prom/query?limit=1 --data-urlencode 'query={app="loki-test"}' | \ + jq -e '.streams[].entries[].line == "foobar"' + } + diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 3877af4a9f..a1f86419ab 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.17.0 +version: 0.17.1 appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 63598ad994..df4062fca1 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.13.0 +version: 0.13.1 appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 1fee17590e..90ad7418c9 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -45,9 +45,9 @@ spec: args: - "-config.file=/etc/promtail/promtail.yaml" {{- if and .Values.loki.user .Values.loki.password }} - - "-client.url={{ .Values.loki.serviceScheme }}://{{ .Values.loki.user }}:{{ .Values.loki.password }}@{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/api/prom/push" + - "-client.url={{ .Values.loki.serviceScheme }}://{{ .Values.loki.user }}:{{ .Values.loki.password }}@{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push" {{- else }} - - "-client.url={{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/api/prom/push" + - "-client.url={{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push" {{- end }} volumeMounts: - name: config From a1027e113a8c919ddee704f79978dcbfd2f360c4 Mon Sep 17 00:00:00 2001 From: Sam Collins Date: Tue, 5 Nov 2019 17:05:45 +0000 Subject: [PATCH 133/202] Updated Kubernetes docs links (#1219) --- charts/fluent-bit/values.yaml | 2 +- charts/loki/values.yaml | 6 +++--- charts/promtail/values.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index 7de07d8929..dff56ed64a 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -39,7 +39,7 @@ image: nameOverride: fluent-bit-loki ## Node labels for pod assignment -## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} ## Pod Labels diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 25ae416e97..ef9086cc96 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -80,7 +80,7 @@ livenessProbe: port: http-metrics initialDelaySeconds: 45 -## Enable persistence using Persistent Volume Claims +## ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ networkPolicy: enabled: false @@ -88,10 +88,10 @@ networkPolicy: client: {} # name: -## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} -## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ ## If you set enabled as "True", you need : ## - create a pv which above 10Gi and has same namespace with loki ## - keep storageClassName same with below setting diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index be0ded75fa..afd957db31 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -23,7 +23,7 @@ loki: nameOverride: promtail ## Node labels for pod assignment -## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} pipelineStages: From d5093f2dffb22ca625d75c9fd0a300822462298c Mon Sep 17 00:00:00 2001 From: Naoki Oketani Date: Thu, 7 Nov 2019 21:13:48 +0900 Subject: [PATCH 134/202] helm/loki-stack supports k8s 1.16 (#1238) --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/requirements.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 138ff380a2..087f10150b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.18.1 +version: 0.19.0 appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/requirements.yaml b/charts/loki-stack/requirements.yaml index 62807f647b..2a10a61adf 100644 --- a/charts/loki-stack/requirements.yaml +++ b/charts/loki-stack/requirements.yaml @@ -13,9 +13,9 @@ dependencies: version: "^0.0.1" - name: "grafana" condition: grafana.enabled - version: "~3.4.3" + version: "~3.8.15" repository: "https://kubernetes-charts.storage.googleapis.com/" - name: "prometheus" condition: prometheus.enabled - version: "~8.11.2" + version: "~9.3.0" repository: "https://kubernetes-charts.storage.googleapis.com/" From 1b9171d9ff00c6bfe7ff348f5d94564bd6271153 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Wed, 13 Nov 2019 09:56:27 -0500 Subject: [PATCH 135/202] helm: update default terminationGracePeriodSeconds to 4800 (#1257) Users that run Loki using the Helm chart may not modify the value for terminationGracePeriodSeconds that allows Loki enough time to flush all of its data. If Loki is forcefully terminated by Kubernetes during a flush, missing data or an unhealthy ingester in the ring in microservices mode can occur. This commit changes the grace period to 4800 seconds, the value used in the Jsonnet code. --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 087f10150b..431b2766f8 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.19.0 +version: 0.19.1 appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index a1f86419ab..f04deb10c0 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.17.1 +version: 0.17.2 appVersion: v0.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index ef9086cc96..d6c1521cbc 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -154,7 +154,7 @@ serviceAccount: create: true name: -terminationGracePeriodSeconds: 30 +terminationGracePeriodSeconds: 4800 ## Tolerations for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ From 7a639abfb4e9e4e250ff3cf665980e1f62db0a8b Mon Sep 17 00:00:00 2001 From: Ed Welch Date: Wed, 20 Nov 2019 00:48:02 -0500 Subject: [PATCH 136/202] updating helm and ksonnet to version v1.0.0 (#1292) --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 431b2766f8..86e18e6bfc 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.19.1 -appVersion: v0.4.0 +version: 0.20.0 +appVersion: v1.0.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index f04deb10c0..f450638c0e 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.17.2 -appVersion: v0.4.0 +version: 0.18.0 +appVersion: v1.0.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index d6c1521cbc..8e813592c2 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -67,7 +67,7 @@ config: image: repository: grafana/loki - tag: v0.4.0 + tag: v1.0.0 pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index df4062fca1..3334de1a26 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.13.1 -appVersion: v0.4.0 +version: 0.14.0 +appVersion: v1.0.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index afd957db31..617a581e6b 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -8,7 +8,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail - tag: v0.4.0 + tag: v1.0.0 pullPolicy: IfNotPresent livenessProbe: {} From fa5e81d4a8d6ab879a84d6d0472803b140fda308 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 2 Dec 2019 15:56:34 +0000 Subject: [PATCH 137/202] Update loki helm chart to support service account annotations (#1341) * update loki helm chart to support service account annotations for use with EKS IAM roles Signed-off-by: Adam Johnson * Bump minor versions and fix serviceAccount reference for annotations Signed-off-by: Adam Johnson --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/serviceaccount.yaml | 2 ++ charts/loki/values.yaml | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 86e18e6bfc..766c841c7a 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.20.0 +version: 0.21.0 appVersion: v1.0.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index f450638c0e..6c05796ca6 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.18.0 +version: 0.19.0 appVersion: v1.0.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/serviceaccount.yaml b/charts/loki/templates/serviceaccount.yaml index a4315ced37..ccbad8d926 100644 --- a/charts/loki/templates/serviceaccount.yaml +++ b/charts/loki/templates/serviceaccount.yaml @@ -7,6 +7,8 @@ metadata: chart: {{ template "loki.chart" . }} heritage: {{ .Release.Service }} release: {{ .Release.Name }} + annotations: + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} name: {{ template "loki.serviceAccountName" . }} namespace: {{ .Release.Namespace }} {{- end }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 8e813592c2..a740f47c2d 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -153,6 +153,7 @@ service: serviceAccount: create: true name: + annotations: {} terminationGracePeriodSeconds: 4800 From 3563f1c02b69912c0d92fecce852c281be1492c0 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Wed, 4 Dec 2019 13:35:20 -0500 Subject: [PATCH 138/202] Updating version numbers for 1.1.0 release --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 766c841c7a..e74f13aa2b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.21.0 -appVersion: v1.0.0 +version: 0.22.0 +appVersion: v1.1.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 6c05796ca6..cfba450ed1 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.19.0 -appVersion: v1.0.0 +version: 0.20.0 +appVersion: v1.1.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index a740f47c2d..3915ea9b23 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -67,7 +67,7 @@ config: image: repository: grafana/loki - tag: v1.0.0 + tag: v1.1.0 pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 3334de1a26..067e71e28e 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.14.0 -appVersion: v1.0.0 +version: 0.15.0 +appVersion: v1.1.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 617a581e6b..c8e3537424 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -8,7 +8,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail - tag: v1.0.0 + tag: v1.1.0 pullPolicy: IfNotPresent livenessProbe: {} From 53c4f6cb8257421fe6db4054608402e25e741502 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Mon, 9 Dec 2019 11:27:48 -0500 Subject: [PATCH 139/202] v1.2.0: update helm and ksonnet (#1389) --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index e74f13aa2b..3bbe243e8e 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.22.0 -appVersion: v1.1.0 +version: 0.23.0 +appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index cfba450ed1..81f5ea6c1c 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.20.0 -appVersion: v1.1.0 +version: 0.21.0 +appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 3915ea9b23..8bfc435dde 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -67,7 +67,7 @@ config: image: repository: grafana/loki - tag: v1.1.0 + tag: v1.2.0 pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 067e71e28e..c3deb35597 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.15.0 -appVersion: v1.1.0 +version: 0.16.0 +appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index c8e3537424..c66139726b 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -8,7 +8,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail - tag: v1.1.0 + tag: v1.2.0 pullPolicy: IfNotPresent livenessProbe: {} From d367078a427189599c1be76e20c1d2ebbebbf134 Mon Sep 17 00:00:00 2001 From: Terje Sannum Date: Fri, 13 Dec 2019 02:13:17 +0100 Subject: [PATCH 140/202] helm: support adding init containers to the loki pod (#1336) --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/statefulset.yaml | 2 ++ charts/loki/values.yaml | 6 ++++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 3bbe243e8e..c22802469f 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.23.0 +version: 0.24.0 appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 81f5ea6c1c..1c91592417 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.21.0 +version: 0.22.0 appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/statefulset.yaml b/charts/loki/templates/statefulset.yaml index dd486cf3fc..1ca90b7362 100644 --- a/charts/loki/templates/statefulset.yaml +++ b/charts/loki/templates/statefulset.yaml @@ -41,6 +41,8 @@ spec: {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.initContainers | nindent 8 }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 8bfc435dde..b3b99b6e1a 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -176,6 +176,12 @@ serviceMonitor: additionalLabels: {} # scrapeTimeout: 10s +initContainers: [] +## Init containers to be added to the loki pod. +# - name: my-init-container +# image: busybox:latest +# command: ['sh', '-c', 'echo hello'] + extraContainers: [] ## Additional containers to be added to the loki pod. # - name: reverse-proxy From 0d492c2a9976cd19b554f754663c99357179b078 Mon Sep 17 00:00:00 2001 From: Yevhen Stolietnii Date: Thu, 9 Jan 2020 23:36:00 +0200 Subject: [PATCH 141/202] Helm: fix issue with config.clients (#1431) * Add statement for config.clients * Bump up chart version * Update production/helm/promtail/templates/daemonset.yaml Yeap, make sense. Co-Authored-By: Sandeep Sukhani Co-authored-by: Sandeep Sukhani --- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index c3deb35597..6239cab7ec 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.16.0 +version: 0.16.1 appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 90ad7418c9..2ca17e32ea 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -44,11 +44,13 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "-config.file=/etc/promtail/promtail.yaml" + {{- if not (or .Values.config.client.url .Values.config.clients) }} {{- if and .Values.loki.user .Values.loki.password }} - "-client.url={{ .Values.loki.serviceScheme }}://{{ .Values.loki.user }}:{{ .Values.loki.password }}@{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push" {{- else }} - "-client.url={{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push" {{- end }} + {{- end }} volumeMounts: - name: config mountPath: /etc/promtail @@ -93,4 +95,3 @@ spec: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} - From e70549b652ab489199a25f739c2b0b54b511f3fa Mon Sep 17 00:00:00 2001 From: OlivierB Date: Fri, 10 Jan 2020 15:36:29 +0100 Subject: [PATCH 142/202] Allow to define custom parsers to use with fluentbit.io/parser annotation (#1430) Signed-off-by: Olivier Boudet --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/README.md | 2 ++ charts/fluent-bit/templates/configmap.yaml | 8 ++++++++ charts/fluent-bit/values.yaml | 7 +++++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index cdeff1f45f..68d88c1fe1 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.0.2 +version: 0.0.3 appVersion: v0.0.1 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/README.md b/charts/fluent-bit/README.md index 007591b5a5..ae6562737e 100644 --- a/charts/fluent-bit/README.md +++ b/charts/fluent-bit/README.md @@ -79,9 +79,11 @@ For more details, read the [Fluent Bit documentation](../../../cmd/fluent-bit/RE | `config.port` | the Fluent Bit port to listen. (This is mainly used to serve metrics) | `2020` | | `config.loglevel` | the Fluent Bit log level (debug,info,warn,error). | `warn` | | `config.lineFormat` | The line format to use to send a record (json/key_value) | `json` | +| `config.k8sLoggingParser`| Allow Kubernetes Pods to suggest a pre-defined Parser. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/kubernetes#kubernetes-annotations). | `Off` | | `config.removeKeys` | The list of key to remove from each record | `[removeKeys,stream]` | | `config.labels` | A set of labels to send for every log | `'{job="fluent-bit"}'` | | `config.labelMap` | Mapping of labels from a record. See [Fluent Bit documentation](../../../cmd/fluent-bit/README.md) | | +| `config.parsers` | Definition of extras fluent bit parsers. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/parser). The format is a sequence of mappings where each key is the same as the one in the [PARSER] section of parsers.conf file | `[]` | | `affinity` | [affinity][affinity] settings for pod assignment | `{}` | | `annotations` | Annotations to add to Kubernetes resources. | `{}` | | `deploymentStrategy` | The deployment strategy to use with the daemonset | `RollingUpdate` | diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml index 551e0d9362..fda1d4c221 100644 --- a/charts/fluent-bit/templates/configmap.yaml +++ b/charts/fluent-bit/templates/configmap.yaml @@ -27,6 +27,7 @@ data: Match kube.* Kube_URL https://kubernetes.default.svc:443 Merge_Log On + K8S-Logging.Parser {{ .Values.config.k8sLoggingParser }} [Output] Name loki Match * @@ -47,6 +48,13 @@ data: Format json Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%L + {{- range $parser:= .Values.config.parsers }} + [PARSER] + {{- range $key,$value := $parser }} + {{ $key }} = {{ $value }} + {{- end }} + {{- end }} labelmap.json: |- {{- .Values.config.labelMap | toPrettyJson | nindent 4}} + diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index dff56ed64a..2ca9b10aed 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -10,6 +10,7 @@ config: port: 2020 loglevel: warn lineFormat: json + k8sLoggingParser: "Off" removeKeys: - kubernetes - stream @@ -24,6 +25,12 @@ config: container_name: container pod_name: instance stream: stream + # parsers: # Allow to define custom parsers. The key here is the same as the one in the [PARSER] section of parsers.conf file. + # - Name: json + # Format: json + # Time_Key: time + # Time_Format: "%d/%b/%Y:%H:%M:%S %z" + affinity: {} From f4e1e78f1e56209e30ee2c9e3e0ea7af7b385725 Mon Sep 17 00:00:00 2001 From: Terje Sannum Date: Wed, 15 Jan 2020 16:51:50 +0100 Subject: [PATCH 143/202] helm: add podsecuritypolicy for fluent-bit (#1506) --- charts/fluent-bit/Chart.yaml | 2 +- .../templates/podsecuritypolicy.yaml | 33 +++++++++++++++++++ charts/fluent-bit/templates/role.yaml | 19 +++++++++++ charts/fluent-bit/templates/rolebinding.yaml | 19 +++++++++++ charts/loki-stack/Chart.yaml | 2 +- 5 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 charts/fluent-bit/templates/podsecuritypolicy.yaml create mode 100644 charts/fluent-bit/templates/role.yaml create mode 100644 charts/fluent-bit/templates/rolebinding.yaml diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index 68d88c1fe1..b6a091e8c9 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.0.3 +version: 0.0.4 appVersion: v0.0.1 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/templates/podsecuritypolicy.yaml b/charts/fluent-bit/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..c347fdc7c6 --- /dev/null +++ b/charts/fluent-bit/templates/podsecuritypolicy.yaml @@ -0,0 +1,33 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "fluent-bit-loki.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + privileged: false + allowPrivilegeEscalation: false + volumes: + - 'secret' + - 'configMap' + - 'hostPath' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL +{{- end }} diff --git a/charts/fluent-bit/templates/role.yaml b/charts/fluent-bit/templates/role.yaml new file mode 100644 index 0000000000..036c2889a7 --- /dev/null +++ b/charts/fluent-bit/templates/role.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "fluent-bit-loki.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.rbac.pspEnabled }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "fluent-bit-loki.fullname" . }}] +{{- end }} +{{- end }} diff --git a/charts/fluent-bit/templates/rolebinding.yaml b/charts/fluent-bit/templates/rolebinding.yaml new file mode 100644 index 0000000000..90315e85c1 --- /dev/null +++ b/charts/fluent-bit/templates/rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "fluent-bit-loki.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fluent-bit-loki.name" . }} + chart: {{ template "fluent-bit-loki.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "fluent-bit-loki.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "fluent-bit-loki.serviceAccountName" . }} +{{- end }} diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index c22802469f..d7e5f0b2a4 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.24.0 +version: 0.25.0 appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." From 6411263feb36b938f8f905eff08a77858e7b2ecd Mon Sep 17 00:00:00 2001 From: OlivierB Date: Thu, 16 Jan 2020 15:33:42 +0100 Subject: [PATCH 144/202] helm : fix fluent-bit parser configuration syntax (#1534) Signed-off-by: Olivier Boudet --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/templates/configmap.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index b6a091e8c9..a0ca7826c2 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.0.4 +version: 0.0.5 appVersion: v0.0.1 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml index fda1d4c221..21fa06421a 100644 --- a/charts/fluent-bit/templates/configmap.yaml +++ b/charts/fluent-bit/templates/configmap.yaml @@ -51,7 +51,7 @@ data: {{- range $parser:= .Values.config.parsers }} [PARSER] {{- range $key,$value := $parser }} - {{ $key }} = {{ $value }} + {{ $key }} {{ $value }} {{- end }} {{- end }} From e9e3392200449cb93ec97126d62d8f5293dee88e Mon Sep 17 00:00:00 2001 From: Wei Wu Date: Fri, 17 Jan 2020 12:37:54 -0700 Subject: [PATCH 145/202] Allow extra command line args for external labels like hostname (#1530) * Allow extra command line args for external labels like hostname * Bump promtail and loki-stack helm chart version --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index d7e5f0b2a4..900ab631da 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.25.0 +version: 0.25.1 appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 6239cab7ec..9313b35ed3 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.16.1 +version: 0.16.2 appVersion: v1.2.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 2ca17e32ea..66dbac5b3c 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -51,6 +51,11 @@ spec: - "-client.url={{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}/loki/api/v1/push" {{- end }} {{- end }} + {{- if .Values.extraCommandlineArgs }} + {{- range .Values.extraCommandlineArgs }} + - {{ . | quote }} + {{- end }} + {{- end }} volumeMounts: - name: config mountPath: /etc/promtail From 43ca2e1376dc567dc9f276f2e5d44bbc16976ba6 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Wed, 22 Jan 2020 13:09:11 -0500 Subject: [PATCH 146/202] updating to 1.3.0 release Signed-off-by: Edward Welch --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 900ab631da..55bd6ecd9a 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.25.1 -appVersion: v1.2.0 +version: 0.26.0 +appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 1c91592417..2242a5d287 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.22.0 -appVersion: v1.2.0 +version: 0.23.0 +appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index b3b99b6e1a..698f030d56 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -67,7 +67,7 @@ config: image: repository: grafana/loki - tag: v1.2.0 + tag: v1.3.0 pullPolicy: IfNotPresent ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 9313b35ed3..54b914895d 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.16.2 -appVersion: v1.2.0 +version: 0.17.0 +appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index c66139726b..abf2c56f1f 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -8,7 +8,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/promtail - tag: v1.2.0 + tag: v1.3.0 pullPolicy: IfNotPresent livenessProbe: {} From 2faa37751db5e47f79131471f0c3d8729446feaf Mon Sep 17 00:00:00 2001 From: Alejandro Ramirez Date: Fri, 24 Jan 2020 10:31:45 -0600 Subject: [PATCH 147/202] helm: Allow setting 'loadBalancerSourceRanges' for the loki service (#1553) * Allow setting 'loadBalancerSourceRanges' for the loki service * Version bump for setting 'loadBalancerSourceRanges' for the loki service --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/service.yaml | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 55bd6ecd9a..37623721ba 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.26.0 +version: 0.27.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 2242a5d287..9b92fd66fc 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.23.0 +version: 0.24.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/service.yaml b/charts/loki/templates/service.yaml index 0df957409f..10d030d4c5 100644 --- a/charts/loki/templates/service.yaml +++ b/charts/loki/templates/service.yaml @@ -17,6 +17,12 @@ spec: type: {{ .Values.service.type }} {{- if (and (eq .Values.service.type "ClusterIP") (not (empty .Values.service.clusterIP))) }} clusterIP: {{ .Values.service.clusterIP }} +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} {{- end }} ports: - port: {{ .Values.service.port }} From 3e07c29e97d2d4be408c0abd180f8258cb67bf77 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Fri, 24 Jan 2020 15:24:55 -0500 Subject: [PATCH 148/202] Disable transfers in loki-local-config.yaml (#1579) * Disable transfers in loki-local-config.yaml #1425 added the ability to disable transfers by setting the max amount of retries to zero or lower. The PR, however, didn't change the default setting in loki-local-config.yaml. This commit changes the setting in loki-local-config.yaml to disable transfers since they would never succeed with the default settings of an inmemory ring anyway. * Set `max_transfer_retries` to 0 in helm charts --- charts/loki/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 698f030d56..85a94389f3 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -24,6 +24,7 @@ config: chunk_idle_period: 3m chunk_block_size: 262144 chunk_retain_period: 1m + max_transfer_retries: 0 lifecycler: ring: kvstore: From 03129601f19510bfdfc7d2fe94f42e1b0c402bbf Mon Sep 17 00:00:00 2001 From: polar3130 <47264004+polar3130@users.noreply.github.com> Date: Tue, 28 Jan 2020 04:20:51 +0900 Subject: [PATCH 149/202] Helm/loki-stack: add template for the service name to connect to loki (#1587) * add helper for the service name to connect to loki * Bump loki-chart version. Signed-off-by: Cyril Tovena Co-authored-by: Cyril Tovena --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/templates/_helpers.tpl | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 37623721ba..5a0f35f97f 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.27.0 +version: 0.27.1 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/templates/_helpers.tpl b/charts/loki-stack/templates/_helpers.tpl index 1543db74a2..1cb7269e80 100644 --- a/charts/loki-stack/templates/_helpers.tpl +++ b/charts/loki-stack/templates/_helpers.tpl @@ -37,4 +37,22 @@ Added as a fix for https://github.com/grafana/loki/issues/1169 */}} {{- define "prometheus.fullname" -}} {{- printf "%s-%s" .Release.Name "prometheus-server" | trunc 63 | trimSuffix "-" -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{/* +The service name to connect to Loki. Defaults to the same logic as "loki.fullname" +*/}} +{{- define "loki.serviceName" -}} +{{- if .Values.loki.serviceName -}} +{{- .Values.loki.serviceName -}} +{{- else if .Values.loki.fullnameOverride -}} +{{- .Values.loki.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "loki" .Values.loki.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} From d4ca3b3117e10edf7ce6f3563b861bd0806a49b1 Mon Sep 17 00:00:00 2001 From: polar3130 <47264004+polar3130@users.noreply.github.com> Date: Tue, 28 Jan 2020 22:38:04 +0900 Subject: [PATCH 150/202] Helm/loki-stack: refresh default grafana.image.tag to 6.6.0 (#1590) * update grafana.image.tag in loki-stack' helm chart * bump the chart loki-stack version --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 5a0f35f97f..15ad2603fb 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.27.1 +version: 0.28.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index eae38fe654..4cc90cb7f4 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -13,7 +13,7 @@ grafana: datasources: enabled: true image: - tag: 6.4.1 + tag: 6.6.0 prometheus: enabled: false From 0b5c3de952101d47dbf934e2722f713576d04c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Tue, 28 Jan 2020 16:22:18 +0100 Subject: [PATCH 151/202] [loki helm chart] added ingress (#1585) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added ingress template Signed-off-by: André Bauer * raised loki-stack chart version to 0.29.0 Signed-off-by: André Bauer --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/ingress.yaml | 44 ++++++++++++++++++++++++++++++ charts/loki/values.yaml | 23 ++++++++++++---- 4 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 charts/loki/templates/ingress.yaml diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 15ad2603fb..3b250b37f8 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.28.0 +version: 0.29.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 9b92fd66fc..85d50cff64 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.24.0 +version: 0.25.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/ingress.yaml b/charts/loki/templates/ingress.yaml new file mode 100644 index 0000000000..307ed678e6 --- /dev/null +++ b/charts/loki/templates/ingress.yaml @@ -0,0 +1,44 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "loki.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app: {{ template "loki.name" . }} + chart: {{ template "loki.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 85a94389f3..8ed562db11 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -1,3 +1,21 @@ +image: + repository: grafana/loki + tag: v1.3.0 + pullPolicy: IfNotPresent + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + ## Affinity for pod assignment ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} @@ -66,11 +84,6 @@ config: retention_deletes_enabled: false retention_period: 0 -image: - repository: grafana/loki - tag: v1.3.0 - pullPolicy: IfNotPresent - ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) extraArgs: {} # log.level: debug From bd6e0805e0b14e0d5a39fc9b0dada0ee0042ed90 Mon Sep 17 00:00:00 2001 From: Roger Steneteg <36709673+rsteneteg@users.noreply.github.com> Date: Mon, 3 Feb 2020 10:41:22 -0600 Subject: [PATCH 152/202] added ability to configure kernel setting fs.inotify.max_user_instances through init container to resolve issue with too many open files (#1620) Signed-off-by: Roger Steneteg --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 11 +++++++++++ charts/promtail/values.yaml | 4 ++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 3b250b37f8..47a22857a4 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.29.0 +version: 0.30.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 54b914895d..bb6316cbf8 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.17.0 +version: 0.18.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 66dbac5b3c..6a4266a5f3 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -37,6 +37,17 @@ spec: serviceAccountName: {{ template "promtail.serviceAccountName" . }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- if .Values.initContainer.enabled }} + initContainers: + - name: init + image: busybox + command: + - sh + - -c + - sysctl -w fs.inotify.max_user_instances={{ .Values.initContainer.fsInotifyMaxUserInstances }} + securityContext: + privileged: true {{- end }} containers: - name: promtail diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index abf2c56f1f..9f6f942beb 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -6,6 +6,10 @@ annotations: {} deploymentStrategy: RollingUpdate +initContainer: + enabled: false + fsInotifyMaxUserInstances: 128 + image: repository: grafana/promtail tag: v1.3.0 From f7cf65cebecdb524d7f2ecc591180f8c2419f1cb Mon Sep 17 00:00:00 2001 From: Aboubacar Toure Date: Wed, 5 Feb 2020 09:52:08 -0500 Subject: [PATCH 153/202] Promtail Helm Chart: Add support for extra environment variables (#1529) * Make Promtail's Helm chart support passing extra env variables * bump charts versions --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 11 +++++++---- charts/promtail/values.yaml | 3 +++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 47a22857a4..20f43f1aac 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.30.0 +version: 0.31.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index bb6316cbf8..0f4a060bf7 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.18.0 +version: 0.18.1 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 6a4266a5f3..afab81bd5b 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -76,10 +76,13 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} env: - - name: HOSTNAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName ports: - containerPort: {{ .Values.config.server.http_listen_port }} name: http-metrics diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 9f6f942beb..5ea0b7aa9d 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -138,3 +138,6 @@ serviceMonitor: interval: "" additionalLabels: {} # scrapeTimeout: 10s + +# Extra env variables to pass to the promtail container +env: [] From b2c1287c06a168a656e22a8aa9dfeaad9fb58992 Mon Sep 17 00:00:00 2001 From: luki Date: Thu, 6 Feb 2020 17:35:39 +0100 Subject: [PATCH 154/202] Added support for imagePullSecrets in Loki Helm chart (#1632) * Added support for imagePullSecrets in Loki Helm chart * Updating only path version for loki charts --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/statefulset.yaml | 6 ++++++ charts/loki/values.yaml | 7 +++++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 20f43f1aac..2bde7dc500 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.31.0 +version: 0.31.1 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 85d50cff64..0cca46682c 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.25.0 +version: 0.25.1 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/statefulset.yaml b/charts/loki/templates/statefulset.yaml index 1ca90b7362..0b22337125 100644 --- a/charts/loki/templates/statefulset.yaml +++ b/charts/loki/templates/statefulset.yaml @@ -43,6 +43,12 @@ spec: {{- toYaml .Values.securityContext | nindent 8 }} initContainers: {{- toYaml .Values.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 8ed562db11..5079a872fe 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -3,6 +3,13 @@ image: tag: v1.3.0 pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistrKeySecretName + ingress: enabled: false annotations: {} From f5e842a4fce46a856efd4cdf98907527dca66235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barri=C3=A8re?= Date: Thu, 13 Feb 2020 14:24:39 +0100 Subject: [PATCH 155/202] Loki stack helm chart can deploy datasources without Grafana (#1688) --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/templates/datasources.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 2bde7dc500..3e98b4c692 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.31.1 +version: 0.31.2 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/templates/datasources.yaml b/charts/loki-stack/templates/datasources.yaml index c0e73f03eb..bb55a5feb3 100644 --- a/charts/loki-stack/templates/datasources.yaml +++ b/charts/loki-stack/templates/datasources.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled }} +{{- if .Values.grafana.sidecar.datasources.enabled }} apiVersion: v1 kind: ConfigMap metadata: From 23b3ca444a28c86e51cbdd8a5c94596aa77727c9 Mon Sep 17 00:00:00 2001 From: Jeff Billimek Date: Fri, 14 Feb 2020 11:26:27 -0500 Subject: [PATCH 156/202] [promtail helm chart] Enable support for syslog service (#1617) * Enable support for dynamic extra ports This should allow the chart to be configured for extra exposed ports like syslog (port 1514) for example Signed-off-by: Jeff Billimek * making syslog its own optional service Signed-off-by: Jeff Billimek * syslogService config should be at top level Signed-off-by: Jeff Billimek * fixing linting error Signed-off-by: Jeff Billimek * Enhancing chart documentation with syslog example Signed-off-by: Jeff Billimek --- charts/README.md | 24 +++++++++++ charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 4 ++ charts/promtail/templates/service-syslog.yaml | 42 +++++++++++++++++++ charts/promtail/values.yaml | 24 +++++++++++ 6 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 charts/promtail/templates/service-syslog.yaml diff --git a/charts/README.md b/charts/README.md index d2a3055b0e..8bfc136c96 100644 --- a/charts/README.md +++ b/charts/README.md @@ -125,6 +125,30 @@ tls: - {{ .Values.ingress.host }} ``` +## Run promtail with syslog support + +In order to receive and process syslog message into promtail, the following changes will be necessary: + +* Review the [promtail syslog-receiver configuration documentation](/docs/clients/promtail/scraping.md#syslog-receiver) + +* Configure the promtail helm chart with the syslog configuration added to the `extraScrapeConfigs` section and associated service definition to listen for syslog messages. For example: + +```yaml +extraScrapeConfigs: + - job_name: syslog + syslog: + listen_address: 0.0.0.0:1514 + labels: + job: "syslog" + relabel_configs: + - source_labels: ['__syslog_message_hostname'] + target_label: 'host' +syslogService: + enabled: true + type: LoadBalancer + port: 1514 +``` + ## How to contribute After adding your new feature to the appropriate chart, you can build and deploy it locally to test: diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 3e98b4c692..72a295b41d 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.31.2 +version: 0.32.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 0f4a060bf7..41c87a72be 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.18.1 +version: 0.19.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index afab81bd5b..cf5a15091b 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -86,6 +86,10 @@ spec: ports: - containerPort: {{ .Values.config.server.http_listen_port }} name: http-metrics + {{- if .Values.syslogService.enabled }} + - containerPort: {{ .Values.syslogService.port }} + name: syslog + {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} {{- if .Values.livenessProbe }} diff --git a/charts/promtail/templates/service-syslog.yaml b/charts/promtail/templates/service-syslog.yaml new file mode 100644 index 0000000000..6c31ab8ef2 --- /dev/null +++ b/charts/promtail/templates/service-syslog.yaml @@ -0,0 +1,42 @@ +{{- if .Values.syslogService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "promtail.fullname" . }}-syslog + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "promtail.name" . }} + chart: {{ template "promtail.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.syslogService.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.syslogService.annotations | nindent 4 }} +spec: + type: {{ .Values.syslogService.type }} +{{- if (and (eq .Values.syslogService.type "ClusterIP") (not (empty .Values.syslogService.clusterIP))) }} + clusterIP: {{ .Values.syslogService.clusterIP }} +{{- end }} +{{- if .Values.syslogService.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.syslogService.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} + ports: + - port: {{ .Values.syslogService.port }} + protocol: TCP + name: syslog + targetPort: syslog +{{- if (and (eq .Values.syslogService.type "NodePort") (not (empty .Values.syslogService.nodePort))) }} + nodePort: {{ .Values.syslogService.nodePort }} +{{- end }} +{{- if .Values.extraPorts }} +{{ toYaml .Values.extraPorts | indent 4}} +{{- end }} + selector: + app: {{ template "promtail.name" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 5ea0b7aa9d..0fe4a15e92 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -127,6 +127,7 @@ config: server: http_listen_port: 3101 + positions: filename: /run/promtail/positions.yaml target_config: @@ -141,3 +142,26 @@ serviceMonitor: # Extra env variables to pass to the promtail container env: [] + +# enable and configure if using the syslog scrape config +syslogService: + enabled: false + type: ClusterIP + port: 1514 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + labels: {} + ## Use loadBalancerIP to request a specific static IP, + ## otherwise leave blank + ## + loadBalancerIP: + # loadBalancerSourceRanges: [] + ## Set the externalTrafficPolicy in the Service to either Cluster or Local + # externalTrafficPolicy: Cluster From 33350b2d8b097fc91d7ace76f8289f0b69621620 Mon Sep 17 00:00:00 2001 From: Naoki Oketani Date: Sat, 15 Feb 2020 03:14:17 +0900 Subject: [PATCH 157/202] fix promtail/templates/NOTES.txt to show correctly port-forward command (#1697) Co-authored-by: Cyril Tovena --- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/NOTES.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 41c87a72be..619164f3a3 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.19.0 +version: 0.19.1 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/NOTES.txt b/charts/promtail/templates/NOTES.txt index 82df065fe1..4d8a1a50d4 100644 --- a/charts/promtail/templates/NOTES.txt +++ b/charts/promtail/templates/NOTES.txt @@ -1,3 +1,3 @@ Verify the application is working by running these commands: - kubectl --namespace {{ .Release.Namespace }} port-forward daemonset/{{ include "promtail.fullname" . }} {{ .Values.port }} - curl http://127.0.0.1:{{ .Values.port }}/metrics \ No newline at end of file + kubectl --namespace {{ .Release.Namespace }} port-forward daemonset/{{ include "promtail.fullname" . }} {{ .Values.config.server.http_listen_port }} + curl http://127.0.0.1:{{ .Values.config.server.http_listen_port }}/metrics From eabe9ae61a9a3886cea855a20d16c430f3dc9ac7 Mon Sep 17 00:00:00 2001 From: Jeff Billimek Date: Wed, 26 Feb 2020 08:09:03 -0500 Subject: [PATCH 158/202] [promtail helm chart] - Expand promtail syslog svc to support values (#1731) * Expand promtail syslog svc to support values The promtail helm chart syslog service configuration in the values.yaml supports elements not accounted-for in the actual syslog service template. This change modifys the promtail syslog service to account for those missing elements. Signed-off-by: Jeff Billimek * Simplifying service typer selection process Signed-off-by: Jeff Billimek --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/service-syslog.yaml | 16 ++++++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 72a295b41d..c509199098 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.32.0 +version: 0.32.1 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 619164f3a3..8ee8bd0353 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.19.1 +version: 0.19.2 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/service-syslog.yaml b/charts/promtail/templates/service-syslog.yaml index 6c31ab8ef2..a701f71197 100644 --- a/charts/promtail/templates/service-syslog.yaml +++ b/charts/promtail/templates/service-syslog.yaml @@ -16,15 +16,19 @@ metadata: {{- toYaml .Values.syslogService.annotations | nindent 4 }} spec: type: {{ .Values.syslogService.type }} -{{- if (and (eq .Values.syslogService.type "ClusterIP") (not (empty .Values.syslogService.clusterIP))) }} + {{- if .Values.syslogService.clusterIP }} clusterIP: {{ .Values.syslogService.clusterIP }} -{{- end }} -{{- if .Values.syslogService.loadBalancerSourceRanges }} + {{end}} + {{- if .Values.syslogService.loadBalancerIP }} + loadBalancerIP: {{ .Values.syslogService.loadBalancerIP }} + {{- end }} + {{- if .Values.syslogService.loadBalancerSourceRanges }} loadBalancerSourceRanges: - {{- range $cidr := .Values.syslogService.loadBalancerSourceRanges }} - - {{ $cidr }} +{{ toYaml .Values.syslogService.loadBalancerSourceRanges | indent 4 }} + {{- end -}} + {{- if .Values.syslogService.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.syslogService.externalTrafficPolicy }} {{- end }} -{{- end }} ports: - port: {{ .Values.syslogService.port }} protocol: TCP From 2d22ae093ef604f541293c0460311c3ece196e43 Mon Sep 17 00:00:00 2001 From: Bruno Clermont Date: Wed, 18 Mar 2020 21:36:55 +0800 Subject: [PATCH 159/202] Prevent prometheus to scrap both services (#1817) Signed-off-by: Bruno Clermont --- charts/loki/Chart.yaml | 2 +- charts/loki/templates/service-headless.yaml | 1 + charts/loki/templates/servicemonitor.yaml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 0cca46682c..df1ebe64e2 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.25.1 +version: 0.25.2 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/service-headless.yaml b/charts/loki/templates/service-headless.yaml index 0c0a0eee5b..644f527625 100644 --- a/charts/loki/templates/service-headless.yaml +++ b/charts/loki/templates/service-headless.yaml @@ -8,6 +8,7 @@ metadata: chart: {{ template "loki.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} + variant: headless spec: clusterIP: None ports: diff --git a/charts/loki/templates/servicemonitor.yaml b/charts/loki/templates/servicemonitor.yaml index 4435832718..1f38cab142 100644 --- a/charts/loki/templates/servicemonitor.yaml +++ b/charts/loki/templates/servicemonitor.yaml @@ -16,6 +16,7 @@ spec: matchLabels: app: {{ template "loki.name" . }} release: {{ .Release.Name | quote }} + variant: headless namespaceSelector: matchNames: - {{ .Release.Namespace | quote }} From 4b496db75888c9299f4a567afc8eeb01451814fc Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 23 Mar 2020 10:54:51 -0400 Subject: [PATCH 160/202] Move default port for Loki to 3100 everywhere. (#1838) Also fixes default value for the table manager that doesn't support 0 anymore on master. Signed-off-by: Cyril Tovena --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/values.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index c509199098..bf4b8a992f 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.32.1 +version: 0.32.2 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index df1ebe64e2..f6c3a606d9 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.25.2 +version: 0.25.3 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 5079a872fe..2c1e016f0d 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -89,7 +89,7 @@ config: max_look_back_period: 0 table_manager: retention_deletes_enabled: false - retention_period: 0 + retention_period: 0s ## Additional Loki container arguments, e.g. log level (debug, info, warn, error) extraArgs: {} From a7857a8026d5a47e3489cfd1501a8ed55ff9c860 Mon Sep 17 00:00:00 2001 From: polar3130 <47264004+polar3130@users.noreply.github.com> Date: Tue, 24 Mar 2020 00:13:13 +0900 Subject: [PATCH 161/202] Helm/loki-stack: refresh default grafana.image.tag to 6.7.0 (#1825) * update grafana.image.tag in loki-stack' helm chart * bump loki-stack chart Co-authored-by: Cyril Tovena --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index bf4b8a992f..d738984e24 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.32.2 +version: 0.33.0 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index 4cc90cb7f4..f7f2cd8f25 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -13,7 +13,7 @@ grafana: datasources: enabled: true image: - tag: 6.6.0 + tag: 6.7.0 prometheus: enabled: false From 0bd7c1048adb9e867763d841f7b3a2336a3f226a Mon Sep 17 00:00:00 2001 From: Stefan Andres Date: Tue, 31 Mar 2020 17:58:07 +0200 Subject: [PATCH 162/202] Add extraMount, extraVolumeMount to promtail helm chart for systemd-journal (#1871) This commit adds the variables extraMount and extraVolumeMount to the promtail helm chart. It also adds an example on how to use the helm chart with systemd-journal support. --- charts/README.md | 35 ++++++++++++++++++++++++ charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 6 ++++ charts/promtail/values.yaml | 6 ++++ 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/charts/README.md b/charts/README.md index 8bfc136c96..4dafe9cdcc 100644 --- a/charts/README.md +++ b/charts/README.md @@ -149,6 +149,41 @@ syslogService: port: 1514 ``` +## Run promtail with systemd-journal support + +In order to receive and process syslog message into promtail, the following changes will be necessary: + +* Review the [promtail systemd-journal configuration documentation](/docs/clients/promtail/scraping.md#journal-scraping-linux-only) + +* Configure the promtail helm chart with the systemd-journal configuration added to the `extraScrapeConfigs` section and volume mounts for the promtail pods to access the log files. For example: + +```yaml +# Add additional scrape config +extraScrapeConfigs: + - job_name: journal + journal: + path: /var/log/journal + max_age: 12h + labels: + job: systemd-journal + relabel_configs: + - source_labels: ['__journal__systemd_unit'] + target_label: 'unit' + - source_labels: ['__journal__hostname'] + target_label: 'hostname' + +# Mount journal directory into promtail pods +extraVolumes: + - name: journal + hostPath: + path: /var/log/journal + +extraVolumeMounts: + - name: journal + mountPath: /var/log/journal + readOnly: true +``` + ## How to contribute After adding your new feature to the appropriate chart, you can build and deploy it locally to test: diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index d738984e24..39e5db7114 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.33.0 +version: 0.33.1 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 8ee8bd0353..2df694979b 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.19.2 +version: 0.19.3 appVersion: v1.3.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index cf5a15091b..03728c3bd3 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -75,6 +75,9 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} env: {{- with .Values.env }} {{- toYaml . | nindent 12 }} @@ -118,3 +121,6 @@ spec: {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 0fe4a15e92..cbbf0175c9 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -96,6 +96,9 @@ volumes: hostPath: path: /var/log/pods +# Custom volumes together with the default ones +extraVolumes: [] + volumeMounts: - name: docker mountPath: /var/lib/docker/containers @@ -104,6 +107,9 @@ volumeMounts: mountPath: /var/log/pods readOnly: true +# Custom volumeMounts together with the default ones +extraVolumeMounts: [] + config: client: # Maximum wait period before sending batch From b53037c7921762912cb6076b1244d74fd6b0a143 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Wed, 1 Apr 2020 13:34:26 -0400 Subject: [PATCH 163/202] updating helm and ksonnet for 1.4.0 release Signed-off-by: Edward Welch --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 39e5db7114..ed3ebc7ef0 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.33.1 -appVersion: v1.3.0 +version: 0.34.0 +appVersion: v1.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index f6c3a606d9..488e0fcb8e 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.25.3 -appVersion: v1.3.0 +version: 0.26.0 +appVersion: v1.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 2c1e016f0d..a02da6081a 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -1,6 +1,6 @@ image: repository: grafana/loki - tag: v1.3.0 + tag: 1.4.0 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 2df694979b..3dec7c8f36 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.19.3 -appVersion: v1.3.0 +version: 0.20.0 +appVersion: v1.4.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index cbbf0175c9..84286f6520 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -12,7 +12,7 @@ initContainer: image: repository: grafana/promtail - tag: v1.3.0 + tag: 1.4.0 pullPolicy: IfNotPresent livenessProbe: {} From be0343ea77ce19ea8a059475d0008997db050893 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 1 Apr 2020 14:48:16 -0400 Subject: [PATCH 164/202] Sign helm package with GPG. (#1864) * Sign helm package with GPG. Also add GPG to the build image and fixes dlv not pinned (this was causing issue using latest 1.4). Signed-off-by: Cyril Tovena * Regen drone config. Signed-off-by: Cyril Tovena * Testing CI. Signed-off-by: Cyril Tovena * Test CI. Signed-off-by: Cyril Tovena * Include .prov files for charts. Signed-off-by: Cyril Tovena * Revert testing changes. Signed-off-by: Cyril Tovena * Add public GPG key for verifying helm charts. Signed-off-by: Cyril Tovena --- charts/public.pgp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 charts/public.pgp diff --git a/charts/public.pgp b/charts/public.pgp new file mode 100644 index 0000000000..f3d730f097 --- /dev/null +++ b/charts/public.pgp @@ -0,0 +1,31 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBF5/50EBCAC3Pab5BojOdLJT+6aJmfWF3HbVbI4l/TOIxMdvI8jneCA+wBX7 +Pp3x4LoSUWX968fX4Mfqyf/fqgHxQltzm5S+MCakROnVkmy5ecsbm61plcvGA1uU +PiYtsWccXvq+cO22onNieSTV7z/330jAFwOGYRrT/utIVmEOefD64uQx6YXhglEU +PmE2N6OVdajd8mlfKStaq/ef1NCwnax4EgWXBnVMrRfouZX//ouguYOyDaQ7IqcG +kFvEaSVqF5hYyVR2Ot+C7hH4/JNy2q6cf6J2N2JS8PE9aMkc7W6VE7BDi0CTweQ4 +0ejK7Y8oRUrKNZH/fsJK5SpqW8kai0NBKDPDABEBAAG0KUdyYWZhbmEgTG9raSA8 +bG9raS10ZWFtQGdvb2dsZWdyb3Vwcy5jb20+iQFUBBMBCAA+FiEEZzewM0g2wzSd +/y3UcFSpVZ08+wsFAl5/50ECGwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgEC +F4AACgkQcFSpVZ08+wudGAf+OU26FHzudVMd4uMWHlQZGSCAGQWriGeFntYke1Ri +str5cjJo78kLYcMKUXfHGrILBEpO7WYr/VEPMxnKNkW8qC9sV/W+Gz+OpEZ7Zuli +3Y9vRI44kJ18Wo7vDryucy9WLR99GmHn0W7C/PMpuXa8P+X5jEQUs5WAm8Rkd3vC +gS0niAf/k3AwWSPxJV+VEy/rAqSuA61tiPiTXOhaD3OpfsCgCigT4epqr/YBLuYF +K728443pJZbxf1TrT8sBacPPg7KK9UkuafZapC/cQ4+0b8/o4CSbPiOPRbysKYmP +9nwg36lFlR0VSI9y68H6KPHe0dTjfgZISUD7QU1okdTVQbkBDQRef+dBAQgA8+h8 +GneZ6QjLrDSVDndNkukP7q0d2abiSx4hzA65mr6lwfGwg20jHKPeYbuYt0n7wUg8 +e3cIkbN2b/CKojgAesLQMHo/TmH0blY3VWoI6WWyTioOWPII9qc4cKnKZGjA8fG1 +CBse3ZNUbHDtvJP/qBbQd4j6waEjIwODaTCwpWGboKDG0o7Zoemx+YwGN7678g1B +E1f8ckwL1FoHZ8ID+bNsBBEUYO+Iup/gcdhZUVsJ1TdTmDfYdpDNMeQ7fyeRyl6O +kUk+Za/ieJhc6Tv+EdPbXmWiTcQDCwiCb7lrQOqcv68q8zcRRjRaqF03dUz8onOl +dhUUqWpIxGdh1+gXTQARAQABiQE8BBgBCAAmFiEEZzewM0g2wzSd/y3UcFSpVZ08 ++wsFAl5/50ECGwwFCQPCZwAACgkQcFSpVZ08+wssFwgAobGpA0FgW+uJ7N05q9lm +WB4ohlUnPSCUaxvzeLxxDPdw0xlxr5rMDwqD9Zvg546fzVx0xxbHka+l/toIV1oI +yQEPIQSPWNUxt5AYfgi/+uhX1qHhFzFU+ebvwWCxxwCM7WBmItswoAFkPfwFbbu9 +7L3HUaRWZbXneaYfPCrg+mv/20r8BGXD8TMTBmkKPwHqmfO1XhwWFEvRRNeYWUMe +GzLqvocxJoZWulgemrdTj4QNIb9HY+UhZuMxkRk3Lvz34dURK4bg86V/KgR3h0Y5 +wWovQyy1DnH+QryT//MT8lMfbD19bjfsuEwRTCEvEWUptg0Y9EfYNe8cHQADTBHE +Hg== +=pugM +-----END PGP PUBLIC KEY BLOCK----- From f403d40630e0d361ac6d7161e263cfe6c838a629 Mon Sep 17 00:00:00 2001 From: Stefan Andres Date: Mon, 6 Apr 2020 15:08:36 +0200 Subject: [PATCH 165/202] [helm loki/promtail] make UpdateStrategy configurable (#1898) This commit makes UpdateStrategy of the promtail daemonset configurable. On large installations, you may want to increase the value of maxUnavailable. This fixes https://github.com/grafana/loki/issues/1878 --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 5 +---- charts/promtail/values.yaml | 7 ++++++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index ed3ebc7ef0..9150eb9901 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.34.0 +version: 0.34.1 appVersion: v1.4.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 3dec7c8f36..79fec8a1b5 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.20.0 +version: 0.20.1 appVersion: v1.4.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 03728c3bd3..8a9cee4dc2 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -16,10 +16,7 @@ spec: app: {{ template "promtail.name" . }} release: {{ .Release.Name }} updateStrategy: - type: {{ .Values.deploymentStrategy }} - {{- if ne .Values.deploymentStrategy "RollingUpdate" }} - rollingUpdate: null - {{- end }} + {{- toYaml .Values.deploymentStrategy | nindent 4 }} template: metadata: labels: diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 84286f6520..559888645a 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -4,7 +4,12 @@ affinity: {} annotations: {} -deploymentStrategy: RollingUpdate +# The update strategy to apply to the DaemonSet +## +deploymentStrategy: {} +# rollingUpdate: +# maxUnavailable: 1 +# type: RollingUpdate initContainer: enabled: false From 79234cb1490298e26e67e6fb6d860da41315514a Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Mon, 6 Apr 2020 16:48:51 -0400 Subject: [PATCH 166/202] updating helm and ksonnet to 1.4.1 updating Fluent Bit to correct image Signed-off-by: Edward Welch --- charts/fluent-bit/Chart.yaml | 4 ++-- charts/fluent-bit/values.yaml | 2 +- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index a0ca7826c2..7c6908748b 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: fluent-bit -version: 0.0.5 -appVersion: v0.0.1 +version: 0.1.0 +appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index 2ca9b10aed..d9434f25cc 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -40,7 +40,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/fluent-bit-plugin-loki - tag: 0.1 + tag: 1.4.1-amd64 pullPolicy: IfNotPresent nameOverride: fluent-bit-loki diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 9150eb9901..cddb37ec32 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.34.1 -appVersion: v1.4.0 +version: 0.34.2 +appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 488e0fcb8e..6d2df2a6cb 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.26.0 -appVersion: v1.4.0 +version: 0.26.1 +appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index a02da6081a..ea8dd83542 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -1,6 +1,6 @@ image: repository: grafana/loki - tag: 1.4.0 + tag: 1.4.1 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 79fec8a1b5..ba912967d3 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail version: 0.20.1 -appVersion: v1.4.0 +appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 559888645a..0f33c56705 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -17,7 +17,7 @@ initContainer: image: repository: grafana/promtail - tag: 1.4.0 + tag: 1.4.1 pullPolicy: IfNotPresent livenessProbe: {} From 98d95fa90c082a83ce06c01e91f010438a2c551f Mon Sep 17 00:00:00 2001 From: "Nicholas St. Germain" Date: Wed, 8 Apr 2020 08:11:21 -0500 Subject: [PATCH 167/202] Fix fluent-bit metrics (#1915) * Add http server to allow prometheus metrics * add correct prometheus path * version bump --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/templates/configmap.yaml | 3 +++ charts/fluent-bit/templates/servicemonitor.yaml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index 7c6908748b..89f850d1d2 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.1.0 +version: 0.1.1 appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml index 21fa06421a..85f82e0e43 100644 --- a/charts/fluent-bit/templates/configmap.yaml +++ b/charts/fluent-bit/templates/configmap.yaml @@ -11,6 +11,9 @@ metadata: data: fluent-bit.conf: |- [SERVICE] + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_PORT {{ .Values.config.port }} Flush 1 Daemon Off Log_Level {{ .Values.config.loglevel }} diff --git a/charts/fluent-bit/templates/servicemonitor.yaml b/charts/fluent-bit/templates/servicemonitor.yaml index 5507bb1087..1cfb7a6ed9 100644 --- a/charts/fluent-bit/templates/servicemonitor.yaml +++ b/charts/fluent-bit/templates/servicemonitor.yaml @@ -21,6 +21,7 @@ spec: - {{ .Release.Namespace | quote }} endpoints: - port: http-metrics + path: /api/v1/metrics/prometheus {{- if .Values.serviceMonitor.interval }} interval: {{ .Values.serviceMonitor.interval }} {{- end }} From 7b8468dc667d3d666a982231a3860def93797d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Tue, 14 Apr 2020 16:19:56 +0200 Subject: [PATCH 168/202] Update Cortex to latest master (#1869) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update Cortex to master Signed-off-by: Peter Štibraný * Fix compilation errors after Cortex update. Signed-off-by: Peter Štibraný * Update to Cortex v1.0.0-rc.0 Signed-off-by: Peter Štibraný * Added missing vendored file. Signed-off-by: Peter Štibraný * Vendor Cortex 1.0.0 Signed-off-by: Peter Štibraný * Configuration changes. Signed-off-by: Peter Štibraný * Mention Cortex 1.0 changes and link to docs. Signed-off-by: Peter Štibraný * Fix jsonnet files, and mention fields that were removed. Signed-off-by: Peter Štibraný --- charts/loki/values.yaml | 4 ++-- charts/promtail/values.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index ea8dd83542..b530729a67 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -63,8 +63,8 @@ config: # consul: # host: "consul:8500" # prefix: "" - # httpclienttimeout: "20s" - # consistentreads: true + # http_client_timeout: "20s" + # consistent_reads: true limits_config: enforce_metric_name: false reject_old_samples: true diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 0f33c56705..ba4711fd1a 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -127,11 +127,11 @@ config: backoff_config: # Initial backoff time between retries - minbackoff: 100ms + min_period: 100ms # Maximum backoff time between retries - maxbackoff: 5s + max_period: 5s # Maximum number of retries when sending batches, 0 means infinite retries - maxretries: 20 + max_retries: 20 # The labels to add to any time series or alerts when communicating with loki external_labels: {} From dfc0778db4d678fa8446b911bc9eed967bdd6257 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 14 Apr 2020 12:00:23 -0400 Subject: [PATCH 169/202] bumps helm to keepalive-master after cortex vendoring (#1937) * bumps helm to keepalive-master after cortex vendoring * minor helm version bump + keepalive image for promtail --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index cddb37ec32..b44eca7e80 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.34.2 -appVersion: v1.4.1 +version: 0.35.0 +appVersion: v1.4.2 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 6d2df2a6cb..bde167ea42 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.26.1 -appVersion: v1.4.1 +version: 0.27.0 +appVersion: v1.4.2 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index b530729a67..4c87c1e467 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -1,6 +1,6 @@ image: repository: grafana/loki - tag: 1.4.1 + tag: keepalive-master-6d5408c pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index ba912967d3..5df38e18f1 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.20.1 -appVersion: v1.4.1 +version: 0.21.0 +appVersion: v1.4.2 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index ba4711fd1a..d4c3a6c694 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -17,7 +17,7 @@ initContainer: image: repository: grafana/promtail - tag: 1.4.1 + tag: keepalive-master-6d5408c pullPolicy: IfNotPresent livenessProbe: {} From 2f22bdb2d112f7b865da220c448b59d09bddfc34 Mon Sep 17 00:00:00 2001 From: Ed Date: Wed, 15 Apr 2020 11:27:15 -0400 Subject: [PATCH 170/202] Reverting helm changes which were included in Cortex 1.0 vendor upgrade because it resulted in us pushing an unstable helm. --- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 6 +++--- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index b44eca7e80..c080633565 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.35.0 -appVersion: v1.4.2 +version: 0.36.0 +appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index bde167ea42..3970e60c81 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.27.0 -appVersion: v1.4.2 +version: 0.28.0 +appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 4c87c1e467..ea8dd83542 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -1,6 +1,6 @@ image: repository: grafana/loki - tag: keepalive-master-6d5408c + tag: 1.4.1 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. @@ -63,8 +63,8 @@ config: # consul: # host: "consul:8500" # prefix: "" - # http_client_timeout: "20s" - # consistent_reads: true + # httpclienttimeout: "20s" + # consistentreads: true limits_config: enforce_metric_name: false reject_old_samples: true diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 5df38e18f1..a5446537ba 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.21.0 -appVersion: v1.4.2 +version: 0.22.0 +appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index d4c3a6c694..0f33c56705 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -17,7 +17,7 @@ initContainer: image: repository: grafana/promtail - tag: keepalive-master-6d5408c + tag: 1.4.1 pullPolicy: IfNotPresent livenessProbe: {} @@ -127,11 +127,11 @@ config: backoff_config: # Initial backoff time between retries - min_period: 100ms + minbackoff: 100ms # Maximum backoff time between retries - max_period: 5s + maxbackoff: 5s # Maximum number of retries when sending batches, 0 means infinite retries - max_retries: 20 + maxretries: 20 # The labels to add to any time series or alerts when communicating with loki external_labels: {} From c042b22e99ac6cdd63872424313858814a22c3ee Mon Sep 17 00:00:00 2001 From: Marc Bihlmaier <30314040+rdxmb@users.noreply.github.com> Date: Tue, 21 Apr 2020 21:29:58 +0200 Subject: [PATCH 171/202] helm chart: add missing line feed (#1967) * add missing LF in helm charts * increase helm Chart version --- charts/loki/Chart.yaml | 2 +- charts/loki/templates/networkpolicy.yaml | 2 +- charts/loki/templates/pdb.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 3970e60c81..b9ad4c9e91 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.28.0 +version: 0.28.1 appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/networkpolicy.yaml b/charts/loki/templates/networkpolicy.yaml index 31d24d7003..5d73832296 100644 --- a/charts/loki/templates/networkpolicy.yaml +++ b/charts/loki/templates/networkpolicy.yaml @@ -23,4 +23,4 @@ spec: release: {{ .Release.Name }} - ports: - port: {{ .Values.service.port }} -{{- end -}} +{{- end }} diff --git a/charts/loki/templates/pdb.yaml b/charts/loki/templates/pdb.yaml index 91c8d21271..c64ad507ed 100644 --- a/charts/loki/templates/pdb.yaml +++ b/charts/loki/templates/pdb.yaml @@ -14,4 +14,4 @@ spec: matchLabels: app: {{ template "loki.name" . }} {{ toYaml .Values.podDisruptionBudget | indent 2 }} -{{- end -}} +{{- end }} From f0becad90aa5b931a4fe4b7ebb6e0980e6da986a Mon Sep 17 00:00:00 2001 From: Smit Shah Date: Fri, 24 Apr 2020 10:43:53 -0400 Subject: [PATCH 172/202] added extraCommandlineArgs in values file (#1981) * added extraCommandlineArgs in values file * corrected yaml with adding space * upgrade promtail chart version * update loki-stack chart version --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index c080633565..8033f9aea9 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.36.0 +version: 0.36.1 appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index a5446537ba..3f0cd704df 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.22.0 +version: 0.22.1 appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 0f33c56705..09421ae3af 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -115,6 +115,14 @@ volumeMounts: # Custom volumeMounts together with the default ones extraVolumeMounts: [] +# Add extra Commandline args while starting up promtail. +# more info : https://github.com/grafana/loki/pull/1530 + +extraCommandlineArgs: [] +# example: +# extraCommandlineArgs: +# - -client.external-labels=hostname=$(HOSTNAME) + config: client: # Maximum wait period before sending batch From cc1bc312324418c434caa4260c6f2da62cc453b1 Mon Sep 17 00:00:00 2001 From: Etash Singh Date: Thu, 30 Apr 2020 23:36:47 +0530 Subject: [PATCH 173/202] Change image tag from 1.4.1 to 1.4.0 in Helm chart (#2015) * Change image tag from 1.4.1 to 1.4.0 Change image tag from 1.4.1 to 1.4.0 to make the chart work as the image with 1.4.1 runs into errors while deploying * Update chart version value * update loki stack chert version * Changed image tag value from 1.4.1 to 1.4.0 * Update values.yaml * Update Chart.yaml * Update Chart.yaml * Update Chart.yaml * Update loki-stack chart version --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 8033f9aea9..7f76a94ad4 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.36.1 +version: 0.36.2 appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 3f0cd704df..0262bd67b4 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.22.1 +version: 0.22.2 appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 09421ae3af..d4d5a1805b 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -17,7 +17,7 @@ initContainer: image: repository: grafana/promtail - tag: 1.4.1 + tag: 1.4.0 pullPolicy: IfNotPresent livenessProbe: {} From c5e7193dd245fbc2a392e9bf52ec049026bfc9ae Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Thu, 30 Apr 2020 14:12:47 -0400 Subject: [PATCH 174/202] Spelling (#2014) * spelling: access * spelling: actual * spelling: asynchronously * spelling: circleci * spelling: circle * spelling: cognitive * spelling: comment * spelling: computation * spelling: correct * spelling: daemon * spelling: elliott * spelling: enabled * spelling: encbuf * spelling: extracted * spelling: explicitly * spelling: independent * spelling: languages * spelling: loose * spelling: message * spelling: metrics * spelling: mismatch * spelling: nonexistent * spelling: output * spelling: pipeline * spelling: pipe line * spelling: position * spelling: prepend * spelling: query * spelling: receive * spelling: received * spelling: reducible * spelling: registry * spelling: response * spelling: retryable * spelling: scanning * spelling: sourced * spelling: supported * spelling: template * spelling: timeout * spelling: unsupported * spelling: adapter * spelling: deferring * spelling: expirable --- charts/loki/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index ea8dd83542..92607dc72d 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -8,7 +8,7 @@ image: ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## # pullSecrets: - # - myRegistrKeySecretName + # - myRegistryKeySecretName ingress: enabled: false From 4909736e07545e5ac1e3e65906911f3538a3d087 Mon Sep 17 00:00:00 2001 From: avii-ridge <57589565+avii-ridge@users.noreply.github.com> Date: Tue, 5 May 2020 23:04:45 +0300 Subject: [PATCH 175/202] Add extraOutputs variable to support multiple outputs for fluent-bit (#2040) * Adding support for extra outputs * added documentation --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/README.md | 1 + charts/fluent-bit/templates/configmap.yaml | 7 ++++++- charts/fluent-bit/values.yaml | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index 89f850d1d2..ebad2d67bb 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.1.1 +version: 0.1.2 appVersion: v1.4.1 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/README.md b/charts/fluent-bit/README.md index ae6562737e..df26abbc58 100644 --- a/charts/fluent-bit/README.md +++ b/charts/fluent-bit/README.md @@ -84,6 +84,7 @@ For more details, read the [Fluent Bit documentation](../../../cmd/fluent-bit/RE | `config.labels` | A set of labels to send for every log | `'{job="fluent-bit"}'` | | `config.labelMap` | Mapping of labels from a record. See [Fluent Bit documentation](../../../cmd/fluent-bit/README.md) | | | `config.parsers` | Definition of extras fluent bit parsers. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/parser). The format is a sequence of mappings where each key is the same as the one in the [PARSER] section of parsers.conf file | `[]` | +| `config.extraOutputs` | Definition of extras fluent bit outputs. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/pipeline/outputs/). The format is a sequence of mappings where each key is the same as the one in the [OUTPUT] | `[]` | | `affinity` | [affinity][affinity] settings for pod assignment | `{}` | | `annotations` | Annotations to add to Kubernetes resources. | `{}` | | `deploymentStrategy` | The deployment strategy to use with the daemonset | `RollingUpdate` | diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml index 85f82e0e43..6b86b51289 100644 --- a/charts/fluent-bit/templates/configmap.yaml +++ b/charts/fluent-bit/templates/configmap.yaml @@ -44,7 +44,12 @@ data: LabelMapPath /fluent-bit/etc/labelmap.json LineFormat {{ .Values.config.lineFormat }} LogLevel {{ .Values.config.loglevel }} - + {{- range $extraOutput := .Values.config.extraOutputs }} + [OUTPUT] + {{- range $key,$value := $extraOutput }} + {{ $key }} {{ $value }} + {{- end }} + {{- end }} parsers.conf: |- [PARSER] Name docker diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index d9434f25cc..b47c6decdb 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -31,6 +31,10 @@ config: # Time_Key: time # Time_Format: "%d/%b/%Y:%H:%M:%S %z" + # extraOutputs: # Allow to define extra outputs in addition to the one automatically created + # - Name: stdout + # Format: json + # json_date_format: time affinity: {} From 9508bd2b678db66e55ed7a2359cc6c817fce6a53 Mon Sep 17 00:00:00 2001 From: Naoki Oketani Date: Fri, 8 May 2020 23:23:15 +0900 Subject: [PATCH 176/202] Docs: update Loki Helm Chart document to support Helm 3 (#2038) * update Loki Helm Chart document to support Helm 3 * update Loki Helm Chart document to support Helm 2 & 3 --- charts/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/charts/README.md b/charts/README.md index 4dafe9cdcc..b35a174b48 100644 --- a/charts/README.md +++ b/charts/README.md @@ -67,7 +67,11 @@ $ helm upgrade --install fluent-bit loki/fluent-bit \ To install Grafana on your cluster with helm, use the following command: ```bash -$ helm install stable/grafana -n loki-grafana +# with Helm 2 +$ helm install stable/grafana -n loki-grafana --namespace + +# with Helm 3 +$ helm install loki-grafana stable/grafana -n ``` > The chart loki-stack contains a pre-configured Grafana, simply use `--set grafana.enabled=true` @@ -75,7 +79,7 @@ $ helm install stable/grafana -n loki-grafana To get the admin password for the Grafana pod, run the following command: ```bash -$ kubectl get secret --namespace loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo +$ kubectl get secret --namespace loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo ``` To access the Grafana UI, run the following command: From e864f1e42b18c6b2d4dc5d3bbbd40578ab0d3f89 Mon Sep 17 00:00:00 2001 From: Ed Welch Date: Wed, 20 May 2020 12:49:34 -0400 Subject: [PATCH 177/202] updating all the images to 1.5.0 release (#2100) * updating all the images to 1.5.0 release Signed-off-by: Ed Welch * duration must have a unit Signed-off-by: Ed Welch --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/values.yaml | 2 +- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 8 ++++---- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 8 ++++---- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index ebad2d67bb..0620637d0a 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: fluent-bit version: 0.1.2 -appVersion: v1.4.1 +appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index b47c6decdb..6e06046121 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -44,7 +44,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/fluent-bit-plugin-loki - tag: 1.4.1-amd64 + tag: 1.5.0-amd64 pullPolicy: IfNotPresent nameOverride: fluent-bit-loki diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 7f76a94ad4..0c4109214f 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.36.2 -appVersion: v1.4.1 +version: 0.37.0 +appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index b9ad4c9e91..4e737c8ee5 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.28.1 -appVersion: v1.4.1 +version: 0.29.0 +appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 92607dc72d..4b62a9857e 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -1,6 +1,6 @@ image: repository: grafana/loki - tag: 1.4.1 + tag: 1.5.0 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. @@ -63,8 +63,8 @@ config: # consul: # host: "consul:8500" # prefix: "" - # httpclienttimeout: "20s" - # consistentreads: true + # http_client_timeout: "20s" + # consistent_reads: true limits_config: enforce_metric_name: false reject_old_samples: true @@ -86,7 +86,7 @@ config: filesystem: directory: /data/loki/chunks chunk_store_config: - max_look_back_period: 0 + max_look_back_period: 0s table_manager: retention_deletes_enabled: false retention_period: 0s diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 0262bd67b4..c7a2021dd1 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.22.2 -appVersion: v1.4.1 +version: 0.23.0 +appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index d4d5a1805b..eb32111447 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -17,7 +17,7 @@ initContainer: image: repository: grafana/promtail - tag: 1.4.0 + tag: 1.5.0 pullPolicy: IfNotPresent livenessProbe: {} @@ -135,11 +135,11 @@ config: backoff_config: # Initial backoff time between retries - minbackoff: 100ms + min_period: 100ms # Maximum backoff time between retries - maxbackoff: 5s + max_period: 5s # Maximum number of retries when sending batches, 0 means infinite retries - maxretries: 20 + max_retries: 20 # The labels to add to any time series or alerts when communicating with loki external_labels: {} From 8a35174880f783016ea7467bcab22a6d2732aac3 Mon Sep 17 00:00:00 2001 From: Frederik Nordahl Jul Sabroe Date: Tue, 26 May 2020 23:32:56 +0200 Subject: [PATCH 178/202] Allow configuring more options for output configuration (#2089) * Allow configuring more options for output configuration The `TenantID`, `BatchWait`, `BatchSize` and `AutoKubernetesLabels` options in the fluent-bit loki output plugin were not configurable. This change allows configuring those options via the helm chart. * typo in production/helm/fluent-bit/README.md Co-authored-by: Joseph Petersen Co-authored-by: Joseph Petersen Co-authored-by: Cyril Tovena --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/README.md | 4 ++++ charts/fluent-bit/templates/configmap.yaml | 4 ++++ charts/fluent-bit/values.yaml | 4 ++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index 0620637d0a..d800870361 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.1.2 +version: 0.1.3 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/README.md b/charts/fluent-bit/README.md index df26abbc58..858205d948 100644 --- a/charts/fluent-bit/README.md +++ b/charts/fluent-bit/README.md @@ -77,11 +77,15 @@ For more details, read the [Fluent Bit documentation](../../../cmd/fluent-bit/RE | `loki.user` | The http basic auth username to access the Loki service. | | | `loki.password` | The http basic auth password to access the Loki service. | | | `config.port` | the Fluent Bit port to listen. (This is mainly used to serve metrics) | `2020` | +| `config.tenantID` | The tenantID used by default to push logs to Loki | `''` | +| `config.batchWait` | Time to wait before send a log batch to Loki, full or not. (unit: secs) | `1` | +| `config.batchSize` | Log batch size to send a log batch to Loki. (unit: bytes) | `10240` (10KiB) | | `config.loglevel` | the Fluent Bit log level (debug,info,warn,error). | `warn` | | `config.lineFormat` | The line format to use to send a record (json/key_value) | `json` | | `config.k8sLoggingParser`| Allow Kubernetes Pods to suggest a pre-defined Parser. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/kubernetes#kubernetes-annotations). | `Off` | | `config.removeKeys` | The list of key to remove from each record | `[removeKeys,stream]` | | `config.labels` | A set of labels to send for every log | `'{job="fluent-bit"}'` | +| `config.autoKubernetesLabels` | If set to true, it will add all Kubernetes labels to Loki labels | `false` | | `config.labelMap` | Mapping of labels from a record. See [Fluent Bit documentation](../../../cmd/fluent-bit/README.md) | | | `config.parsers` | Definition of extras fluent bit parsers. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/parser). The format is a sequence of mappings where each key is the same as the one in the [PARSER] section of parsers.conf file | `[]` | | `config.extraOutputs` | Definition of extras fluent bit outputs. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/pipeline/outputs/). The format is a sequence of mappings where each key is the same as the one in the [OUTPUT] | `[]` | diff --git a/charts/fluent-bit/templates/configmap.yaml b/charts/fluent-bit/templates/configmap.yaml index 6b86b51289..3022647906 100644 --- a/charts/fluent-bit/templates/configmap.yaml +++ b/charts/fluent-bit/templates/configmap.yaml @@ -39,8 +39,12 @@ data: {{- else }} Url {{ .Values.loki.serviceScheme }}://{{ include "loki.serviceName" . }}:{{ .Values.loki.servicePort }}{{ .Values.loki.servicePath }} {{- end }} + TenantID {{ .Values.config.tenantID }} + BatchWait {{ .Values.config.batchWait }} + BatchSize {{ .Values.config.batchSize }} Labels {{ .Values.config.labels }} RemoveKeys {{ include "helm-toolkit.utils.joinListWithComma" .Values.config.removeKeys }} + AutoKubernetesLabels {{ .Values.config.autoKubernetesLabels }} LabelMapPath /fluent-bit/etc/labelmap.json LineFormat {{ .Values.config.lineFormat }} LogLevel {{ .Values.config.loglevel }} diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index 6e06046121..673e09eb83 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -8,12 +8,16 @@ loki: # password: pass config: port: 2020 + tenantID: '""' + batchWait: 1 + batchSize: 10240 loglevel: warn lineFormat: json k8sLoggingParser: "Off" removeKeys: - kubernetes - stream + autoKubernetesLabels: false labels: '{job="fluent-bit"}' labelMap: kubernetes: From 1de606d16a9538d427f7eb977268bcf5d5bcc912 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 18 May 2020 16:29:14 +0200 Subject: [PATCH 179/202] Keep scrape config in line with the new Prometheus scrape config This is triggered by https://github.com/grafana/jsonnet-libs/pull/261 . The above PR removes the `instance` label. As it has turned out (see PR linked above), a sane `instance` label in Prometheus has to be unique, and that includes the case where a single container exposes metrics on two different endpoints. However, that scenario would still only result in one log stream for Loki to scrape. Therefore, Loki and Prometheus need to sync via target labels uniquely identifying a container (rather than a metrics endpoint). Those labels are namespace, pod, container, also added here. This commit removes the `container_name` label. It is the same as the `container` label and was already added to Loki previously. However, the `container_name` label is deprecated and has disappeared in K8s 1.16, so that it will soon become useless for direct joining. --- charts/promtail/templates/configmap.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/charts/promtail/templates/configmap.yaml b/charts/promtail/templates/configmap.yaml index 211279538f..58eb9bd496 100644 --- a/charts/promtail/templates/configmap.yaml +++ b/charts/promtail/templates/configmap.yaml @@ -47,11 +47,11 @@ data: - action: replace source_labels: - __meta_kubernetes_pod_name - target_label: instance + target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name - target_label: container_name + target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: @@ -94,11 +94,11 @@ data: - action: replace source_labels: - __meta_kubernetes_pod_name - target_label: instance + target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name - target_label: container_name + target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: @@ -147,11 +147,11 @@ data: - action: replace source_labels: - __meta_kubernetes_pod_name - target_label: instance + target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name - target_label: container_name + target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: @@ -202,11 +202,11 @@ data: - action: replace source_labels: - __meta_kubernetes_pod_name - target_label: instance + target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name - target_label: container_name + target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: @@ -250,11 +250,11 @@ data: - action: replace source_labels: - __meta_kubernetes_pod_name - target_label: instance + target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name - target_label: container_name + target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: From 23102eb73443921d181cb63022f87984114753f4 Mon Sep 17 00:00:00 2001 From: James Hollowell Date: Wed, 27 May 2020 13:33:20 -0400 Subject: [PATCH 180/202] Support Prometheus on a sub-path in Grafana config (#2099) Co-authored-by: Cyril Tovena --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/templates/datasources.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 0c4109214f..1059e27203 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.37.0 +version: 0.37.1 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/templates/datasources.yaml b/charts/loki-stack/templates/datasources.yaml index bb55a5feb3..29e2f4f33a 100644 --- a/charts/loki-stack/templates/datasources.yaml +++ b/charts/loki-stack/templates/datasources.yaml @@ -25,7 +25,7 @@ data: - name: Prometheus type: prometheus access: proxy - url: http://{{ include "prometheus.fullname" .}}:{{ .Values.prometheus.server.service.servicePort }} + url: http://{{ include "prometheus.fullname" .}}:{{ .Values.prometheus.server.service.servicePort }}{{ .Values.prometheus.server.prefixURL }} version: 1 {{- end }} {{- end }} From 0f1c7bb4c9063e74caa75cb44154f16c1181a140 Mon Sep 17 00:00:00 2001 From: Or Sela Date: Fri, 29 May 2020 03:36:46 +0300 Subject: [PATCH 181/202] helm/loki-stack: render loki datasource only if grafana is enabled (#2086) --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/templates/datasources.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 1059e27203..6d809fbf9b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.37.1 +version: 0.37.2 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/templates/datasources.yaml b/charts/loki-stack/templates/datasources.yaml index 29e2f4f33a..9ebce1408e 100644 --- a/charts/loki-stack/templates/datasources.yaml +++ b/charts/loki-stack/templates/datasources.yaml @@ -1,4 +1,4 @@ -{{- if .Values.grafana.sidecar.datasources.enabled }} +{{- if and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled }} apiVersion: v1 kind: ConfigMap metadata: From 2e793d75be21d0dd4192f524e71712602801575f Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 2 Jun 2020 07:53:20 -0400 Subject: [PATCH 182/202] Adds the ability to have a pull secrets for Promtail. (#2141) * Adds the ability to have a pull secrets for Promtail. Fixes #2118 Signed-off-by: Cyril Tovena * bump chart version Signed-off-by: Cyril Tovena --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/daemonset.yaml | 8 +++++++- charts/promtail/values.yaml | 6 ++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 6d809fbf9b..c08e1ad48f 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.37.2 +version: 0.37.3 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index c7a2021dd1..75e9488aae 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.23.0 +version: 0.23.1 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/daemonset.yaml b/charts/promtail/templates/daemonset.yaml index 8a9cee4dc2..2a22e154db 100644 --- a/charts/promtail/templates/daemonset.yaml +++ b/charts/promtail/templates/daemonset.yaml @@ -24,7 +24,7 @@ spec: release: {{ .Release.Name }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} annotations: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} @@ -46,6 +46,12 @@ spec: securityContext: privileged: true {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} containers: - name: promtail image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index eb32111447..324e7d844d 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -19,6 +19,12 @@ image: repository: grafana/promtail tag: 1.5.0 pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName livenessProbe: {} From d837f94dfe282c1fb619054ce07a4d2c916fd84c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20H=C3=BCgli?= Date: Mon, 15 Jun 2020 21:14:25 +0200 Subject: [PATCH 183/202] BUG: add missing namespace in ingress object (#2205) * add missing namespace in ingress object * update chart version * bump loki stack as well --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/ingress.yaml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index c08e1ad48f..ac48adbd1b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.37.3 +version: 0.37.4 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 4e737c8ee5..4f9a1e1111 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.29.0 +version: 0.29.1 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/ingress.yaml b/charts/loki/templates/ingress.yaml index 307ed678e6..f68cda8ea5 100644 --- a/charts/loki/templates/ingress.yaml +++ b/charts/loki/templates/ingress.yaml @@ -9,6 +9,7 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: name: {{ $fullName }} + namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} From 2379cab3968973315d83e69c2f6c84f508a4b041 Mon Sep 17 00:00:00 2001 From: vhrosales <65722070+vhrosales@users.noreply.github.com> Date: Thu, 18 Jun 2020 17:44:38 -0300 Subject: [PATCH 184/202] helm: Add loadBalancerIP option to loki chart (#2238) * Add loadBalancerIP option for LoadBalancer service type * Bump minor version * Bump loki-stack minor version --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/service.yaml | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index ac48adbd1b..b4848bc43a 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.37.4 +version: 0.38.0 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 4f9a1e1111..081f69fe24 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.29.1 +version: 0.30.0 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/service.yaml b/charts/loki/templates/service.yaml index 10d030d4c5..2eb419c02f 100644 --- a/charts/loki/templates/service.yaml +++ b/charts/loki/templates/service.yaml @@ -18,6 +18,9 @@ spec: {{- if (and (eq .Values.service.type "ClusterIP") (not (empty .Values.service.clusterIP))) }} clusterIP: {{ .Values.service.clusterIP }} {{- end }} +{{- if (and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP))) }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} +{{- end }} {{- if .Values.service.loadBalancerSourceRanges }} loadBalancerSourceRanges: {{- range $cidr := .Values.service.loadBalancerSourceRanges }} From 22480f436d458190d9f23ad148dbab16075c5eb8 Mon Sep 17 00:00:00 2001 From: Simone Esposito Date: Mon, 22 Jun 2020 14:47:42 +0200 Subject: [PATCH 185/202] Kubernetes manifests: Remove namespace from cluster-wide resources (#2241) Fixes https://github.com/grafana/loki/issues/2234 --- charts/fluent-bit/templates/clusterrole.yaml | 1 - charts/fluent-bit/templates/podsecuritypolicy.yaml | 1 - charts/loki/templates/podsecuritypolicy.yaml | 1 - charts/promtail/templates/clusterrole.yaml | 1 - charts/promtail/templates/podsecuritypolicy.yaml | 1 - 5 files changed, 5 deletions(-) diff --git a/charts/fluent-bit/templates/clusterrole.yaml b/charts/fluent-bit/templates/clusterrole.yaml index ac9239afa0..6b1a27c2fe 100644 --- a/charts/fluent-bit/templates/clusterrole.yaml +++ b/charts/fluent-bit/templates/clusterrole.yaml @@ -8,7 +8,6 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} name: {{ template "fluent-bit-loki.fullname" . }}-clusterrole - namespace: {{ .Release.Namespace }} rules: - apiGroups: [""] # "" indicates the core API group resources: diff --git a/charts/fluent-bit/templates/podsecuritypolicy.yaml b/charts/fluent-bit/templates/podsecuritypolicy.yaml index c347fdc7c6..59f68dbe72 100644 --- a/charts/fluent-bit/templates/podsecuritypolicy.yaml +++ b/charts/fluent-bit/templates/podsecuritypolicy.yaml @@ -3,7 +3,6 @@ apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: {{ template "fluent-bit-loki.fullname" . }} - namespace: {{ .Release.Namespace }} labels: app: {{ template "fluent-bit-loki.name" . }} chart: {{ template "fluent-bit-loki.chart" . }} diff --git a/charts/loki/templates/podsecuritypolicy.yaml b/charts/loki/templates/podsecuritypolicy.yaml index 6a6444e3fa..9968b4844b 100644 --- a/charts/loki/templates/podsecuritypolicy.yaml +++ b/charts/loki/templates/podsecuritypolicy.yaml @@ -3,7 +3,6 @@ apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: {{ template "loki.fullname" . }} - namespace: {{ .Release.Namespace }} labels: app: {{ template "loki.name" . }} chart: {{ template "loki.chart" . }} diff --git a/charts/promtail/templates/clusterrole.yaml b/charts/promtail/templates/clusterrole.yaml index 7260d2e107..b872041b7a 100644 --- a/charts/promtail/templates/clusterrole.yaml +++ b/charts/promtail/templates/clusterrole.yaml @@ -8,7 +8,6 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} name: {{ template "promtail.fullname" . }}-clusterrole - namespace: {{ .Release.Namespace }} rules: - apiGroups: [""] # "" indicates the core API group resources: diff --git a/charts/promtail/templates/podsecuritypolicy.yaml b/charts/promtail/templates/podsecuritypolicy.yaml index abaca89bf1..a38595eeef 100644 --- a/charts/promtail/templates/podsecuritypolicy.yaml +++ b/charts/promtail/templates/podsecuritypolicy.yaml @@ -3,7 +3,6 @@ apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: {{ template "promtail.fullname" . }} - namespace: {{ .Release.Namespace }} labels: app: {{ template "promtail.name" . }} chart: {{ template "promtail.chart" . }} From d75b5a2d74d8cd3381a23130d88cd1f63ce9fe0b Mon Sep 17 00:00:00 2001 From: vyzigold Date: Mon, 22 Jun 2020 14:48:09 +0200 Subject: [PATCH 186/202] Add readmes to individual helm charts (#2233) --- charts/loki-stack/README.md | 64 +++++++++++++++++++++++++++++ charts/loki/README.md | 62 ++++++++++++++++++++++++++++ charts/promtail/README.md | 81 +++++++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 charts/loki-stack/README.md create mode 100644 charts/loki/README.md create mode 100644 charts/promtail/README.md diff --git a/charts/loki-stack/README.md b/charts/loki-stack/README.md new file mode 100644 index 0000000000..a4548ece1a --- /dev/null +++ b/charts/loki-stack/README.md @@ -0,0 +1,64 @@ +# Loki-Stack Helm Chart + +## Prerequisites + +Make sure you have Helm [installed](https://helm.sh/docs/using_helm/#installing-helm) and +[deployed](https://helm.sh/docs/using_helm/#installing-tiller) to your cluster. Then add +Loki's chart repository to Helm: + +```bash +$ helm repo add loki https://grafana.github.io/loki/charts +``` + +You can update the chart repository by running: + +```bash +$ helm repo update +``` + +## Deploy Loki and Promtail to your cluster + +### Deploy with default config + +```bash +$ helm upgrade --install loki loki/loki-stack +``` + +### Deploy in a custom namespace + +```bash +$ helm upgrade --install loki --namespace=loki-stack loki/loki-stack +``` + +### Deploy with custom config + +```bash +$ helm upgrade --install loki loki/loki-stack --set "key1=val1,key2=val2,..." +``` + +## Deploy Loki and Fluent Bit to your cluster + +```bash +$ helm upgrade --install loki loki/loki-stack \ + --set fluent-bit.enabled=true,promtail.enabled=false +``` + +## Deploy Grafana to your cluster + +The chart loki-stack contains a pre-configured Grafana, simply use `--set grafana.enabled=true` + +To get the admin password for the Grafana pod, run the following command: + +```bash +$ kubectl get secret --namespace loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo +``` + +To access the Grafana UI, run the following command: + +```bash +$ kubectl port-forward --namespace service/loki-grafana 3000:80 +``` + +Navigate to http://localhost:3000 and login with `admin` and the password output above. +Then follow the [instructions for adding the loki datasource](/docs/getting-started/grafana.md), using the URL `http://loki:3100/`. + diff --git a/charts/loki/README.md b/charts/loki/README.md new file mode 100644 index 0000000000..9d0cd692af --- /dev/null +++ b/charts/loki/README.md @@ -0,0 +1,62 @@ +# Loki Helm Chart + +## Prerequisites + +Make sure you have Helm [installed](https://helm.sh/docs/using_helm/#installing-helm) and +[deployed](https://helm.sh/docs/using_helm/#installing-tiller) to your cluster. Then add +Loki's chart repository to Helm: + +```bash +$ helm repo add loki https://grafana.github.io/loki/charts +``` + +You can update the chart repository by running: + +```bash +$ helm repo update +``` + +## Deploy Loki only + +```bash +$ helm upgrade --install loki loki/loki +``` + +## Run Loki behind https ingress + +If Loki and Promtail are deployed on different clusters you can add an Ingress in front of Loki. +By adding a certificate you create an https endpoint. For extra security enable basic authentication on the Ingress. + +In Promtail set the following values to communicate with https and basic auth + +``` +loki: + serviceScheme: https + user: user + password: pass +``` + +Sample helm template for ingress: +``` +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: +annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.class }} + ingress.kubernetes.io/auth-type: "basic" + ingress.kubernetes.io/auth-secret: {{ .Values.ingress.basic.secret }} +name: loki +spec: +rules: +- host: {{ .Values.ingress.host }} + http: + paths: + - backend: + serviceName: loki + servicePort: 3100 +tls: +- secretName: {{ .Values.ingress.cert }} + hosts: + - {{ .Values.ingress.host }} +``` + diff --git a/charts/promtail/README.md b/charts/promtail/README.md new file mode 100644 index 0000000000..9eb09435a2 --- /dev/null +++ b/charts/promtail/README.md @@ -0,0 +1,81 @@ +# Promtail Helm Chart + +## Deploy Promtail only + +```bash +$ helm upgrade --install promtail loki/promtail --set "loki.serviceName=loki" +``` + +## Run Loki behind https ingress + +If Loki and Promtail are deployed on different clusters you can add an Ingress in front of Loki. +By adding a certificate you create an https endpoint. For extra security enable basic authentication on the Ingress. + +In Promtail set the following values to communicate with https and basic auth + +``` +loki: + serviceScheme: https + user: user + password: pass +``` + +## Run promtail with syslog support + +In order to receive and process syslog message into promtail, the following changes will be necessary: + +* Review the [promtail syslog-receiver configuration documentation](/docs/clients/promtail/scraping.md#syslog-receiver) + +* Configure the promtail helm chart with the syslog configuration added to the `extraScrapeConfigs` section and associated service definition to listen for syslog messages. For example: + +```yaml +extraScrapeConfigs: + - job_name: syslog + syslog: + listen_address: 0.0.0.0:1514 + labels: + job: "syslog" + relabel_configs: + - source_labels: ['__syslog_message_hostname'] + target_label: 'host' +syslogService: + enabled: true + type: LoadBalancer + port: 1514 +``` + +## Run promtail with systemd-journal support + +In order to receive and process syslog message into promtail, the following changes will be necessary: + +* Review the [promtail systemd-journal configuration documentation](/docs/clients/promtail/scraping.md#journal-scraping-linux-only) + +* Configure the promtail helm chart with the systemd-journal configuration added to the `extraScrapeConfigs` section and volume mounts for the promtail pods to access the log files. For example: + +```yaml +# Add additional scrape config +extraScrapeConfigs: + - job_name: journal + journal: + path: /var/log/journal + max_age: 12h + labels: + job: systemd-journal + relabel_configs: + - source_labels: ['__journal__systemd_unit'] + target_label: 'unit' + - source_labels: ['__journal__hostname'] + target_label: 'hostname' + +# Mount journal directory into promtail pods +extraVolumes: + - name: journal + hostPath: + path: /var/log/journal + +extraVolumeMounts: + - name: journal + mountPath: /var/log/journal + readOnly: true +``` + From 19125670a9de26276c0d70e53d3c8f99b30ef225 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 22 Jun 2020 08:48:48 -0400 Subject: [PATCH 187/202] Render loki datasources even if Grafana is disabled. (#2197) I didn't realize but this was a perfect ok use case to generate datasource when using a different grafana deployment. Fixes #2186 Fixes #2195 Now regarding https://github.com/grafana/loki/pull/2086 it seems pretty easy to use `.Values.grafana.sidecar.datasources.enabled` if you don't want any datasources. Signed-off-by: Cyril Tovena --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/templates/datasources.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index b4848bc43a..09bbecf23b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.38.0 +version: 0.38.1 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/templates/datasources.yaml b/charts/loki-stack/templates/datasources.yaml index 9ebce1408e..29e2f4f33a 100644 --- a/charts/loki-stack/templates/datasources.yaml +++ b/charts/loki-stack/templates/datasources.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled }} +{{- if .Values.grafana.sidecar.datasources.enabled }} apiVersion: v1 kind: ConfigMap metadata: From ca6f2e0dfcef84ecd712082487ff5a0ee9834a5b Mon Sep 17 00:00:00 2001 From: Mathieu Gaubert <26342638+Synehan@users.noreply.github.com> Date: Mon, 29 Jun 2020 18:38:05 +0200 Subject: [PATCH 188/202] helm: add annotations to service monitor (#2258) --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/templates/servicemonitor.yaml | 4 ++++ charts/fluent-bit/values.yaml | 1 + charts/loki/Chart.yaml | 2 +- charts/loki/templates/servicemonitor.yaml | 4 ++++ charts/loki/values.yaml | 1 + charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/servicemonitor.yaml | 4 ++++ charts/promtail/values.yaml | 1 + 9 files changed, 18 insertions(+), 3 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index d800870361..1a365ab1ee 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.1.3 +version: 0.1.4 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/templates/servicemonitor.yaml b/charts/fluent-bit/templates/servicemonitor.yaml index 1cfb7a6ed9..162f96018f 100644 --- a/charts/fluent-bit/templates/servicemonitor.yaml +++ b/charts/fluent-bit/templates/servicemonitor.yaml @@ -11,6 +11,10 @@ metadata: {{- if .Values.serviceMonitor.additionalLabels }} {{ toYaml .Values.serviceMonitor.additionalLabels | indent 4 }} {{- end }} + {{- if .Values.serviceMonitor.annotations }} + annotations: +{{ toYaml .Values.serviceMonitor.annotations | indent 4 }} + {{- end }} spec: selector: matchLabels: diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index 673e09eb83..2138c888cb 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -109,4 +109,5 @@ serviceMonitor: enabled: false interval: "" additionalLabels: {} + annotations: {} # scrapeTimeout: 10s diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 081f69fe24..7c6c103f0c 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.30.0 +version: 0.30.1 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/servicemonitor.yaml b/charts/loki/templates/servicemonitor.yaml index 1f38cab142..f3b4abe532 100644 --- a/charts/loki/templates/servicemonitor.yaml +++ b/charts/loki/templates/servicemonitor.yaml @@ -11,6 +11,10 @@ metadata: {{- if .Values.serviceMonitor.additionalLabels }} {{ toYaml .Values.serviceMonitor.additionalLabels | indent 4 }} {{- end }} + {{- if .Values.serviceMonitor.annotations }} + annotations: +{{ toYaml .Values.serviceMonitor.annotations | indent 4 }} + {{- end }} spec: selector: matchLabels: diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 4b62a9857e..05b662366e 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -195,6 +195,7 @@ serviceMonitor: enabled: false interval: "" additionalLabels: {} + annotations: {} # scrapeTimeout: 10s initContainers: [] diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 75e9488aae..5f5ffb2308 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.23.1 +version: 0.23.2 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/servicemonitor.yaml b/charts/promtail/templates/servicemonitor.yaml index 43f8aac8c2..7b2d3bef6d 100644 --- a/charts/promtail/templates/servicemonitor.yaml +++ b/charts/promtail/templates/servicemonitor.yaml @@ -11,6 +11,10 @@ metadata: {{- if .Values.serviceMonitor.additionalLabels }} {{ toYaml .Values.serviceMonitor.additionalLabels | indent 4 }} {{- end }} + {{- if .Values.serviceMonitor.annotations }} + annotations: +{{ toYaml .Values.serviceMonitor.annotations | indent 4 }} + {{- end }} spec: selector: matchLabels: diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 324e7d844d..270c2dc5b5 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -163,6 +163,7 @@ serviceMonitor: enabled: false interval: "" additionalLabels: {} + annotations: {} # scrapeTimeout: 10s # Extra env variables to pass to the promtail container From da9a22dc73f747989d1e48eb0333830ee4a8a4a0 Mon Sep 17 00:00:00 2001 From: Steven Reitsma Date: Fri, 17 Jul 2020 15:54:42 +0200 Subject: [PATCH 189/202] Add projected and downwardAPI volume types to PodSecurityPolicy (#2355) (#2366) --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/templates/podsecuritypolicy.yaml | 2 ++ charts/loki/Chart.yaml | 2 +- charts/loki/templates/podsecuritypolicy.yaml | 3 ++- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/podsecuritypolicy.yaml | 2 ++ 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index 1a365ab1ee..fd2afc8763 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.1.4 +version: 0.1.5 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/templates/podsecuritypolicy.yaml b/charts/fluent-bit/templates/podsecuritypolicy.yaml index 59f68dbe72..a38514e679 100644 --- a/charts/fluent-bit/templates/podsecuritypolicy.yaml +++ b/charts/fluent-bit/templates/podsecuritypolicy.yaml @@ -15,6 +15,8 @@ spec: - 'secret' - 'configMap' - 'hostPath' + - 'projected' + - 'downwardAPI' hostNetwork: false hostIPC: false hostPID: false diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 7c6c103f0c..66e27210b3 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.30.1 +version: 0.30.2 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/podsecuritypolicy.yaml b/charts/loki/templates/podsecuritypolicy.yaml index 9968b4844b..ce1c1c109a 100644 --- a/charts/loki/templates/podsecuritypolicy.yaml +++ b/charts/loki/templates/podsecuritypolicy.yaml @@ -16,6 +16,8 @@ spec: - 'emptyDir' - 'persistentVolumeClaim' - 'secret' + - 'projected' + - 'downwardAPI' hostNetwork: false hostIPC: false hostPID: false @@ -37,4 +39,3 @@ spec: requiredDropCapabilities: - ALL {{- end }} - diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 5f5ffb2308..f6580366d5 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.23.2 +version: 0.23.3 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/podsecuritypolicy.yaml b/charts/promtail/templates/podsecuritypolicy.yaml index a38595eeef..440ca7176a 100644 --- a/charts/promtail/templates/podsecuritypolicy.yaml +++ b/charts/promtail/templates/podsecuritypolicy.yaml @@ -15,6 +15,8 @@ spec: - 'secret' - 'configMap' - 'hostPath' + - 'projected' + - 'downwardAPI' hostNetwork: false hostIPC: false hostPID: false From 760bf16c9afe6e668ea43e2f0a2c0737e94e7eaa Mon Sep 17 00:00:00 2001 From: Aditya C S Date: Fri, 17 Jul 2020 19:47:32 +0530 Subject: [PATCH 190/202] Loki Logstash Plugin (#1822) * Logstash plugin * include_labels * include_labels * Removes binary. Signed-off-by: Cyril Tovena * Improve documentation and remove the push path. Signed-off-by: Cyril Tovena * Move to cmd. Signed-off-by: Cyril Tovena * Add more precision for jruby. Signed-off-by: Cyril Tovena * Update docs/clients/logstash/README.md * p Signed-off-by: Cyril Tovena * ignore Signed-off-by: Cyril Tovena * remove ignore file/ Signed-off-by: Cyril Tovena * More precision for installing jruby Signed-off-by: Cyril Tovena * Rename without Grafana Signed-off-by: Cyril Tovena * A lot of refactoring and testing. Signed-off-by: Cyril Tovena * change delay logic * Fully tested version. Now testing/writing docs and charts. Signed-off-by: Cyril Tovena * Forgot to save merge. Signed-off-by: Cyril Tovena * working version. Signed-off-by: Cyril Tovena * Makefile + easier docker build. Signed-off-by: Cyril Tovena * adds ci to build logstash image. Signed-off-by: Cyril Tovena * Fix build for logstash. Signed-off-by: Cyril Tovena * Adds example with helm charts. Signed-off-by: Cyril Tovena * Fix target to send 10 logs with logstash. Signed-off-by: Cyril Tovena * Improved documentation. Signed-off-by: Cyril Tovena * add missing helm add repo for external repo Signed-off-by: Cyril Tovena * Review comment. Signed-off-by: Cyril Tovena * Fixes loki service in Promtail. Signed-off-by: Cyril Tovena * Update loki-stack version Signed-off-by: Cyril Tovena Co-authored-by: Cyril Tovena --- charts/README.md | 16 ++++++++ charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/requirements.yaml | 8 ++++ charts/loki-stack/values.yaml | 55 ++++++++++++++++++++++++++ charts/promtail/templates/_helpers.tpl | 1 - 5 files changed, 80 insertions(+), 2 deletions(-) diff --git a/charts/README.md b/charts/README.md index b35a174b48..85a98628b1 100644 --- a/charts/README.md +++ b/charts/README.md @@ -44,10 +44,19 @@ $ helm upgrade --install loki loki/loki ## Deploy Promtail only +We recommend Promtail to ship your logs to Loki as the configuration is very similar to Prometheus. +This allows you to ensure that labels for metrics and logs are equivalent by re-using the same `scrape_configs` and `relabeling` configuration. +When using Grafana having the same labels will allows you to pivot from Metrics to Logs verify easily by simply switching datasource. + +To only install Promtail use the following command: + ```bash $ helm upgrade --install promtail loki/promtail --set "loki.serviceName=loki" ``` +If you're not familiar with Prometheus and you don't want to migrate your current agent configs from the start, + you can use our output plugins specified below. + ## Deploy Loki and Fluent Bit to your cluster ```bash @@ -62,6 +71,13 @@ $ helm upgrade --install fluent-bit loki/fluent-bit \ --set "loki.serviceName=loki.svc.cluster.local" ``` +## Deploy Loki and Filebeat and logstash to your cluster + +```bash +$ helm upgrade --install loki loki/loki-stack \ + --set filebeat.enabled=true,logstash.enabled=true,promtail.enabled=false +``` + ## Deploy Grafana to your cluster To install Grafana on your cluster with helm, use the following command: diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 09bbecf23b..d12dba1c13 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.38.1 +version: 0.38.2 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/requirements.yaml b/charts/loki-stack/requirements.yaml index 2a10a61adf..c2275d24fd 100644 --- a/charts/loki-stack/requirements.yaml +++ b/charts/loki-stack/requirements.yaml @@ -19,3 +19,11 @@ dependencies: condition: prometheus.enabled version: "~9.3.0" repository: "https://kubernetes-charts.storage.googleapis.com/" +- name: "filebeat" + condition: filebeat.enabled + version: "~7.8.0" + repository: "https://helm.elastic.co" +- name: "logstash" + condition: logstash.enabled + version: "~2.4.0" + repository: "https://kubernetes-charts.storage.googleapis.com/" diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index f7f2cd8f25..78e684a73c 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -1,8 +1,11 @@ loki: + fullnameOverride: loki enabled: true promtail: enabled: true + loki: + serviceName: loki fluent-bit: enabled: false @@ -17,3 +20,55 @@ grafana: prometheus: enabled: false + +filebeat: + enabled: false + filebeatConfig: + filebeat.yml: | + # logging.level: debug + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" + output.logstash: + hosts: ["logstash-loki:5044"] + +logstash: + enabled: false + fullnameOverride: logstash-loki + image: + repository: grafana/logstash-output-loki + tag: 1.0.1 + filters: + main: |- + filter { + if [kubernetes] { + mutate { + add_field => { + "container_name" => "%{[kubernetes][container][name]}" + "namespace" => "%{[kubernetes][namespace]}" + "pod" => "%{[kubernetes][pod][name]}" + } + replace => { "host" => "%{[kubernetes][node][name]}"} + } + } + mutate { + remove_field => ["tags"] + } + } + outputs: + main: |- + output { + loki { + url => "http://loki:3100/loki/api/v1/push" + #username => "test" + #password => "test" + } + # stdout { codec => rubydebug } + } diff --git a/charts/promtail/templates/_helpers.tpl b/charts/promtail/templates/_helpers.tpl index c270b0b281..9298bcbc90 100644 --- a/charts/promtail/templates/_helpers.tpl +++ b/charts/promtail/templates/_helpers.tpl @@ -59,4 +59,3 @@ The service name to connect to Loki. Defaults to the same logic as "loki.fullnam {{- end -}} {{- end -}} {{- end -}} - From b5ba2cafc3e1932f8a3df86f2dcee781d97ab45a Mon Sep 17 00:00:00 2001 From: Steven Reitsma Date: Tue, 21 Jul 2020 15:22:52 +0200 Subject: [PATCH 191/202] Add emptyDir volume type to promtail PSP (#2379) --- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/podsecuritypolicy.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index f6580366d5..8689e9f55d 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.23.3 +version: 0.23.4 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/podsecuritypolicy.yaml b/charts/promtail/templates/podsecuritypolicy.yaml index 440ca7176a..77ab4a493e 100644 --- a/charts/promtail/templates/podsecuritypolicy.yaml +++ b/charts/promtail/templates/podsecuritypolicy.yaml @@ -17,6 +17,7 @@ spec: - 'hostPath' - 'projected' - 'downwardAPI' + - 'emptyDir' hostNetwork: false hostIPC: false hostPID: false From 0c66cfb7ce8997ed913ff37496e914897aeea070 Mon Sep 17 00:00:00 2001 From: Steven Sheehy <17552371+steven-sheehy@users.noreply.github.com> Date: Fri, 24 Jul 2020 10:38:24 -0500 Subject: [PATCH 192/202] Fix regression in chart name (#2406) Signed-off-by: Steven Sheehy --- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/requirements.yaml | 6 +++--- charts/loki-stack/values.yaml | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index d12dba1c13..5ff8159340 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.38.2 +version: 0.38.3 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/requirements.yaml b/charts/loki-stack/requirements.yaml index c2275d24fd..a580a00b93 100644 --- a/charts/loki-stack/requirements.yaml +++ b/charts/loki-stack/requirements.yaml @@ -2,15 +2,15 @@ dependencies: - name: "loki" condition: loki.enabled repository: "file://../loki" - version: "^0.6.0" + version: "^0.30.0" - name: "promtail" condition: promtail.enabled repository: "file://../promtail" - version: "^0.6.0" + version: "^0.23.0" - name: "fluent-bit" condition: fluent-bit.enabled repository: "file://../fluent-bit" - version: "^0.0.1" + version: "^0.1.0" - name: "grafana" condition: grafana.enabled version: "~3.8.15" diff --git a/charts/loki-stack/values.yaml b/charts/loki-stack/values.yaml index 78e684a73c..4fc181b226 100644 --- a/charts/loki-stack/values.yaml +++ b/charts/loki-stack/values.yaml @@ -1,11 +1,8 @@ loki: - fullnameOverride: loki enabled: true promtail: enabled: true - loki: - serviceName: loki fluent-bit: enabled: false @@ -41,7 +38,6 @@ filebeat: logstash: enabled: false - fullnameOverride: logstash-loki image: repository: grafana/logstash-output-loki tag: 1.0.1 From 79ffa13f9c4c909bb84aa4e0631a0b082c9ff8b5 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 29 Jul 2020 09:32:59 -0400 Subject: [PATCH 193/202] Set service values via --set for logstash. (#2415) As per discussed in #2406. Signed-off-by: Cyril Tovena --- charts/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index 85a98628b1..fb333f86f8 100644 --- a/charts/README.md +++ b/charts/README.md @@ -75,7 +75,8 @@ $ helm upgrade --install fluent-bit loki/fluent-bit \ ```bash $ helm upgrade --install loki loki/loki-stack \ - --set filebeat.enabled=true,logstash.enabled=true,promtail.enabled=false + --set filebeat.enabled=true,logstash.enabled=true,promtail.enabled=false \ + --set loki.fullnameOverride=loki,logstash.fullnameOverride=logstash-loki ``` ## Deploy Grafana to your cluster From 8b96ffd4c9ce270d3f38054860029fe2de2ce110 Mon Sep 17 00:00:00 2001 From: Ryan Holt Date: Tue, 11 Aug 2020 20:23:13 -0400 Subject: [PATCH 194/202] promtail: enable externalIPs support for syslog (#2490) * enable externalIPs support Signed-off-by: Ryan Holt * bump loki-stack chart version too Signed-off-by: Ryan Holt --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/templates/service-syslog.yaml | 4 ++++ charts/promtail/values.yaml | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 5ff8159340..01de741595 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.38.3 +version: 0.38.4 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 8689e9f55d..c6ae4223e0 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.23.4 +version: 0.23.5 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/service-syslog.yaml b/charts/promtail/templates/service-syslog.yaml index a701f71197..247ebdfae7 100644 --- a/charts/promtail/templates/service-syslog.yaml +++ b/charts/promtail/templates/service-syslog.yaml @@ -26,6 +26,10 @@ spec: loadBalancerSourceRanges: {{ toYaml .Values.syslogService.loadBalancerSourceRanges | indent 4 }} {{- end -}} + {{- if .Values.syslogService.externalIPs }} + externalIPs: +{{ toYaml .Values.syslogService.externalIPs | indent 4 }} + {{- end }} {{- if .Values.syslogService.externalTrafficPolicy }} externalTrafficPolicy: {{ .Values.syslogService.externalTrafficPolicy }} {{- end }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 270c2dc5b5..88209716d5 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -174,6 +174,7 @@ syslogService: enabled: false type: ClusterIP port: 1514 + # externalIPs: [] ## Specify the nodePort value for the LoadBalancer and NodePort service types. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport ## From 51cb8a908d6cd3691b59a4055810c295509b37c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barri=C3=A8re?= Date: Wed, 12 Aug 2020 14:26:11 +0200 Subject: [PATCH 195/202] Add optional secret to pull Docker image in fluent-bit chart (#2489) --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/templates/daemonset.yaml | 10 ++++++++-- charts/fluent-bit/values.yaml | 5 +++++ charts/loki-stack/Chart.yaml | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index fd2afc8763..53f8a01ec9 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.1.5 +version: 0.2.0 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/templates/daemonset.yaml b/charts/fluent-bit/templates/daemonset.yaml index e57b53232c..6652c29646 100644 --- a/charts/fluent-bit/templates/daemonset.yaml +++ b/charts/fluent-bit/templates/daemonset.yaml @@ -35,9 +35,15 @@ spec: {{- end }} spec: serviceAccountName: {{ template "fluent-bit-loki.serviceAccountName" . }} - {{- if .Values.priorityClassName }} + {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} - {{- end }} + {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} containers: - name: fluent-bit-loki image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index 2138c888cb..eaa94c9ac4 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -50,6 +50,11 @@ image: repository: grafana/fluent-bit-plugin-loki tag: 1.5.0-amd64 pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be present in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + # pullSecrets: + # - myRegistrKeySecretName nameOverride: fluent-bit-loki diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 01de741595..2304d4624a 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.38.4 +version: 0.39.0 appVersion: v1.5.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." From 03b944e6b9db0c5addf789275ecaace1d5d17de8 Mon Sep 17 00:00:00 2001 From: Ed Welch Date: Thu, 13 Aug 2020 14:09:31 -0400 Subject: [PATCH 196/202] updating images to 1.6.0 (#2504) --- charts/fluent-bit/Chart.yaml | 4 ++-- charts/fluent-bit/values.yaml | 2 +- charts/loki-stack/Chart.yaml | 4 ++-- charts/loki/Chart.yaml | 4 ++-- charts/loki/values.yaml | 2 +- charts/promtail/Chart.yaml | 4 ++-- charts/promtail/values.yaml | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index 53f8a01ec9..dd0d67323e 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: fluent-bit -version: 0.2.0 -appVersion: v1.5.0 +version: 0.3.0 +appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index eaa94c9ac4..47db300462 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -48,7 +48,7 @@ deploymentStrategy: RollingUpdate image: repository: grafana/fluent-bit-plugin-loki - tag: 1.5.0-amd64 + tag: 1.6.0-amd64 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be present in the namespace. diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 2304d4624a..0e50c80250 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki-stack -version: 0.39.0 -appVersion: v1.5.0 +version: 0.40.0 +appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 66e27210b3..db7d6e3371 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: loki -version: 0.30.2 -appVersion: v1.5.0 +version: 0.31.0 +appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." home: https://grafana.com/loki diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 05b662366e..7b12fe31f7 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -1,6 +1,6 @@ image: repository: grafana/loki - tag: 1.5.0 + tag: 1.6.0 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index c6ae4223e0..88c3ba48b3 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: "v1" name: promtail -version: 0.23.5 -appVersion: v1.5.0 +version: 0.24.0 +appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" home: https://grafana.com/loki diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 88209716d5..6db64f7789 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -17,7 +17,7 @@ initContainer: image: repository: grafana/promtail - tag: 1.5.0 + tag: 1.6.0 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. From 6c3d8033144e06ce17045db2b2202f4f2ee3a344 Mon Sep 17 00:00:00 2001 From: Alexandre Allard <31347867+alexandre-allard-scality@users.noreply.github.com> Date: Thu, 10 Sep 2020 15:28:14 +0200 Subject: [PATCH 197/202] Add support for PV selector in Loki statefulset (#2554) --- charts/loki-stack/Chart.yaml | 2 +- charts/loki/Chart.yaml | 2 +- charts/loki/templates/statefulset.yaml | 5 ++++- charts/loki/values.yaml | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 0e50c80250..7817560f7b 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.40.0 +version: 0.40.1 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index db7d6e3371..024f4fe375 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.31.0 +version: 0.31.1 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/templates/statefulset.yaml b/charts/loki/templates/statefulset.yaml index 0b22337125..a993cc9349 100644 --- a/charts/loki/templates/statefulset.yaml +++ b/charts/loki/templates/statefulset.yaml @@ -124,5 +124,8 @@ spec: requests: storage: {{ .Values.persistence.size | quote }} storageClassName: {{ .Values.persistence.storageClassName }} + {{- if .Values.persistence.selector }} + selector: + {{- toYaml .Values.persistence.selector | nindent 8 }} + {{- end }} {{- end }} - diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 7b12fe31f7..3b6806eaed 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -122,6 +122,9 @@ persistence: - ReadWriteOnce size: 10Gi annotations: {} + # selector: + # matchLabels: + # app.kubernetes.io/name: loki # subPath: "" # existingClaim: From eafc9b0b3d539bd16f4d83cd4705863d1c90ebe5 Mon Sep 17 00:00:00 2001 From: RS Krishna Date: Tue, 29 Sep 2020 18:30:13 +0000 Subject: [PATCH 198/202] [Promtail] enables configuring psp in helm chart (#2659) * makes pod security policy configurable * bump promtail chart version * bump loki-stack chart version --- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- .../promtail/templates/podsecuritypolicy.yaml | 26 ++----------------- charts/promtail/values.yaml | 25 ++++++++++++++++++ 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index 7817560f7b..c4528943f7 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.40.1 +version: 0.41.0 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index 88c3ba48b3..cc6116dce2 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.24.0 +version: 0.25.0 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/templates/podsecuritypolicy.yaml b/charts/promtail/templates/podsecuritypolicy.yaml index 77ab4a493e..56643cb5b0 100644 --- a/charts/promtail/templates/podsecuritypolicy.yaml +++ b/charts/promtail/templates/podsecuritypolicy.yaml @@ -9,27 +9,5 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} spec: - privileged: false - allowPrivilegeEscalation: false - volumes: - - 'secret' - - 'configMap' - - 'hostPath' - - 'projected' - - 'downwardAPI' - - 'emptyDir' - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' - readOnlyRootFilesystem: true - requiredDropCapabilities: - - ALL - {{- end }} + {{- toYaml .Values.podSecurityPolicy | nindent 2 }} +{{- end }} diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 6db64f7789..4e2c12b19c 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -58,6 +58,31 @@ rbac: create: true pspEnabled: true +podSecurityPolicy: + privileged: false + allowPrivilegeEscalation: false + volumes: + - 'secret' + - 'configMap' + - 'hostPath' + - 'projected' + - 'downwardAPI' + - 'emptyDir' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + readinessProbe: failureThreshold: 5 httpGet: From 51a57d2a94179367072a4199c597076d07ca42cc Mon Sep 17 00:00:00 2001 From: Ed Welch Date: Thu, 1 Oct 2020 14:46:46 -0400 Subject: [PATCH 199/202] Promtail: (and also fluent-bit) change the max batch size to 1MB (#2710) * change the max batch size to 1MB for all the defaults including helm and fluent-bit, attempt to centralize this config a little where possible. * fix test --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/values.yaml | 2 +- charts/loki-stack/Chart.yaml | 2 +- charts/promtail/Chart.yaml | 2 +- charts/promtail/values.yaml | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index dd0d67323e..4c513c51f1 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.3.0 +version: 0.3.1 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index 47db300462..dd099f8d86 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -10,7 +10,7 @@ config: port: 2020 tenantID: '""' batchWait: 1 - batchSize: 10240 + batchSize: 1048576 loglevel: warn lineFormat: json k8sLoggingParser: "Off" diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index c4528943f7..d040c731fb 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.41.0 +version: 0.41.1 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index cc6116dce2..b75a2e5a39 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.25.0 +version: 0.25.1 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/values.yaml b/charts/promtail/values.yaml index 4e2c12b19c..20933b2b5b 100644 --- a/charts/promtail/values.yaml +++ b/charts/promtail/values.yaml @@ -159,18 +159,18 @@ config: # Maximum wait period before sending batch batchwait: 1s # Maximum batch size to accrue before sending, unit is byte - batchsize: 102400 + batchsize: 1048576 # Maximum time to wait for server to respond to a request timeout: 10s backoff_config: # Initial backoff time between retries - min_period: 100ms + min_period: 500ms # Maximum backoff time between retries - max_period: 5s + max_period: 5m # Maximum number of retries when sending batches, 0 means infinite retries - max_retries: 20 + max_retries: 10 # The labels to add to any time series or alerts when communicating with loki external_labels: {} From 1cb467093d8abbc46c0ea614e83a362b4b391e8c Mon Sep 17 00:00:00 2001 From: Torsten Walter Date: Sat, 3 Oct 2020 23:28:53 +0200 Subject: [PATCH 200/202] Migration adjustments - fix markdown linting errors - bump chart versions and align READMEs Added instructions to all READMEs, how to add the grafana repo. Use Grafana repo for install instructions. - Update requirements from loki-stack to this repo (Except Grafana as there is quite a version difference) - Disabling validation of maintainers - Add stable repo as long as it's needed loki-stack depends on it at the moment - Add https://helm.elastic.co/ repo Signed-off-by: Torsten Walter --- charts/fluent-bit/Chart.yaml | 2 +- charts/fluent-bit/README.md | 19 +++++++++++++----- charts/grafana/Chart.yaml | 2 +- charts/grafana/README.md | 7 +++---- charts/loki-stack/Chart.yaml | 2 +- charts/loki-stack/README.md | 31 +++++++++++++---------------- charts/loki-stack/requirements.yaml | 6 +++--- charts/loki/Chart.yaml | 2 +- charts/loki/README.md | 24 ++++++++++------------ charts/promtail/Chart.yaml | 2 +- charts/promtail/README.md | 13 ++++++++++-- ct.yaml | 4 ++++ 12 files changed, 65 insertions(+), 49 deletions(-) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml index 4c513c51f1..be48db1998 100644 --- a/charts/fluent-bit/Chart.yaml +++ b/charts/fluent-bit/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: fluent-bit -version: 0.3.1 +version: 0.4.0 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki" diff --git a/charts/fluent-bit/README.md b/charts/fluent-bit/README.md index 858205d948..35c957df1c 100644 --- a/charts/fluent-bit/README.md +++ b/charts/fluent-bit/README.md @@ -2,14 +2,23 @@ This chart install the Fluent Bit application to ship logs to Loki. It defines daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + ## Installing the Chart > If you don't have `Helm` installed locally, or `Tiller` installed in your Kubernetes cluster, read the [Using Helm](https://docs.helm.sh/using_helm/) documentation to get started. To install the chart with the release name `my-release` using our helm repository: ```bash -helm repo add loki https://grafana.github.io/loki/charts -helm upgrade --install my-release loki/fluent-bit \ +helm repo add grafana https://grafana.github.io/helm-charts +helm upgrade --install my-release grafana/fluent-bit \ --set loki.serviceName=loki.default.svc.cluster.local ``` @@ -20,7 +29,7 @@ The command deploys Fluent Bit on the Kubernetes cluster with the default config To configure the chart to send to [Grafana Cloud](https://grafana.com/products/cloud) use: ```bash -helm upgrade --install my-release loki/fluent-bit \ +helm upgrade --install my-release grafana/fluent-bit \ --set loki.serviceName=logs-us-west1.grafana.net,loki.servicePort=80,loki.serviceScheme=https \ --set loki.user=2830,loki.password=1234 ``` @@ -30,7 +39,7 @@ helm upgrade --install my-release loki/fluent-bit \ To install a custom tag use the following command: ```bash -helm upgrade --install my-release loki/fluent-bit \ +helm upgrade --install my-release grafana/fluent-bit \ --set image.tag= ``` @@ -39,7 +48,7 @@ The full list of available tags on [docker hub](https://cloud.docker.com/u/grafa Alternatively you can install the full [Loki stack](../loki-stack) (Loki + Fluent Bit) using: ```bash -helm upgrade --install my-release loki/loki-stack \ +helm upgrade --install my-release grafana/loki-stack \ --set fluent-bit.enabled=true,promtail.enabled=false ``` diff --git a/charts/grafana/Chart.yaml b/charts/grafana/Chart.yaml index 94b9f71091..5b85a3810a 100644 --- a/charts/grafana/Chart.yaml +++ b/charts/grafana/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: grafana -version: 5.7.0 +version: 5.8.0 appVersion: 7.2.0 kubeVersion: "^1.8.0-0" description: The leading tool for querying and visualizing time series and metrics. diff --git a/charts/grafana/README.md b/charts/grafana/README.md index ce6eb90561..ff59b13fa5 100644 --- a/charts/grafana/README.md +++ b/charts/grafana/README.md @@ -2,15 +2,14 @@ * Installs the web dashboarding system [Grafana](http://grafana.org/) -## TL;DR +## Get Repo Info ```console helm repo add grafana https://grafana.github.io/helm-charts +helm repo update ``` -```console -helm install grafana/grafana -``` +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ ## Installing the Chart diff --git a/charts/loki-stack/Chart.yaml b/charts/loki-stack/Chart.yaml index d040c731fb..00e770b6d1 100644 --- a/charts/loki-stack/Chart.yaml +++ b/charts/loki-stack/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki-stack -version: 0.41.1 +version: 0.42.0 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki-stack/README.md b/charts/loki-stack/README.md index a4548ece1a..41bd9147cf 100644 --- a/charts/loki-stack/README.md +++ b/charts/loki-stack/README.md @@ -2,44 +2,41 @@ ## Prerequisites -Make sure you have Helm [installed](https://helm.sh/docs/using_helm/#installing-helm) and -[deployed](https://helm.sh/docs/using_helm/#installing-tiller) to your cluster. Then add -Loki's chart repository to Helm: +Make sure you have Helm [installed](https://helm.sh/docs/using_helm/#installing-helm) installed. -```bash -$ helm repo add loki https://grafana.github.io/loki/charts -``` - -You can update the chart repository by running: +## Get Repo Info -```bash -$ helm repo update +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update ``` +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + ## Deploy Loki and Promtail to your cluster ### Deploy with default config ```bash -$ helm upgrade --install loki loki/loki-stack +helm upgrade --install loki grafana/loki-stack ``` ### Deploy in a custom namespace ```bash -$ helm upgrade --install loki --namespace=loki-stack loki/loki-stack +helm upgrade --install loki --namespace=loki-stack grafana/loki-stack ``` ### Deploy with custom config ```bash -$ helm upgrade --install loki loki/loki-stack --set "key1=val1,key2=val2,..." +helm upgrade --install loki grafana/loki-stack --set "key1=val1,key2=val2,..." ``` ## Deploy Loki and Fluent Bit to your cluster ```bash -$ helm upgrade --install loki loki/loki-stack \ +helm upgrade --install loki loki/loki-stack \ --set fluent-bit.enabled=true,promtail.enabled=false ``` @@ -50,15 +47,15 @@ The chart loki-stack contains a pre-configured Grafana, simply use `--set grafan To get the admin password for the Grafana pod, run the following command: ```bash -$ kubectl get secret --namespace loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo +kubectl get secret --namespace loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo ``` To access the Grafana UI, run the following command: ```bash -$ kubectl port-forward --namespace service/loki-grafana 3000:80 +kubectl port-forward --namespace service/loki-grafana 3000:80 ``` -Navigate to http://localhost:3000 and login with `admin` and the password output above. +Navigate to and login with `admin` and the password output above. Then follow the [instructions for adding the loki datasource](/docs/getting-started/grafana.md), using the URL `http://loki:3100/`. diff --git a/charts/loki-stack/requirements.yaml b/charts/loki-stack/requirements.yaml index a580a00b93..aa0d0d6ebc 100644 --- a/charts/loki-stack/requirements.yaml +++ b/charts/loki-stack/requirements.yaml @@ -2,15 +2,15 @@ dependencies: - name: "loki" condition: loki.enabled repository: "file://../loki" - version: "^0.30.0" + version: "^0.32.0" - name: "promtail" condition: promtail.enabled repository: "file://../promtail" - version: "^0.23.0" + version: "^0.26.0" - name: "fluent-bit" condition: fluent-bit.enabled repository: "file://../fluent-bit" - version: "^0.1.0" + version: "^0.4.0" - name: "grafana" condition: grafana.enabled version: "~3.8.15" diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 024f4fe375..e663acd7f0 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: loki -version: 0.31.1 +version: 0.32.0 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Loki: like Prometheus, but for logs." diff --git a/charts/loki/README.md b/charts/loki/README.md index 9d0cd692af..4eeda3c689 100644 --- a/charts/loki/README.md +++ b/charts/loki/README.md @@ -2,24 +2,22 @@ ## Prerequisites -Make sure you have Helm [installed](https://helm.sh/docs/using_helm/#installing-helm) and -[deployed](https://helm.sh/docs/using_helm/#installing-tiller) to your cluster. Then add -Loki's chart repository to Helm: +Make sure you have Helm [installed](https://helm.sh/docs/using_helm/#installing-helm). -```bash -$ helm repo add loki https://grafana.github.io/loki/charts +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update ``` -You can update the chart repository by running: +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ -```bash -$ helm repo update -``` ## Deploy Loki only ```bash -$ helm upgrade --install loki loki/loki +helm upgrade --install loki grafana/loki ``` ## Run Loki behind https ingress @@ -29,7 +27,7 @@ By adding a certificate you create an https endpoint. For extra security enable In Promtail set the following values to communicate with https and basic auth -``` +```yaml loki: serviceScheme: https user: user @@ -37,7 +35,8 @@ loki: ``` Sample helm template for ingress: -``` + +```yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: @@ -59,4 +58,3 @@ tls: hosts: - {{ .Values.ingress.host }} ``` - diff --git a/charts/promtail/Chart.yaml b/charts/promtail/Chart.yaml index b75a2e5a39..b4a50591d8 100644 --- a/charts/promtail/Chart.yaml +++ b/charts/promtail/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v1" name: promtail -version: 0.25.1 +version: 0.26.0 appVersion: v1.6.0 kubeVersion: "^1.10.0-0" description: "Responsible for gathering logs and sending them to Loki" diff --git a/charts/promtail/README.md b/charts/promtail/README.md index 9eb09435a2..ed977aceba 100644 --- a/charts/promtail/README.md +++ b/charts/promtail/README.md @@ -1,9 +1,18 @@ # Promtail Helm Chart +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + ## Deploy Promtail only ```bash -$ helm upgrade --install promtail loki/promtail --set "loki.serviceName=loki" +helm upgrade --install promtail grafana/promtail --set "loki.serviceName=loki" ``` ## Run Loki behind https ingress @@ -13,7 +22,7 @@ By adding a certificate you create an https endpoint. For extra security enable In Promtail set the following values to communicate with https and basic auth -``` +```yaml loki: serviceScheme: https user: user diff --git a/ct.yaml b/ct.yaml index c0770cd561..caec8658fe 100644 --- a/ct.yaml +++ b/ct.yaml @@ -3,4 +3,8 @@ remote: origin target-branch: main chart-dirs: - charts +chart-repos: + - stable=https://kubernetes-charts.storage.googleapis.com/ + - elastic=https://helm.elastic.co/ helm-extra-args: --timeout 600s +validate-maintainers: false From 2ba63dea9e3b59d80e3c2c38f394339658edc2be Mon Sep 17 00:00:00 2001 From: Torsten Walter Date: Tue, 6 Oct 2020 16:09:17 +0200 Subject: [PATCH 201/202] Move alignment of grafana chart README to another PR Signed-off-by: Torsten Walter --- charts/grafana/Chart.yaml | 2 +- charts/grafana/README.md | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/grafana/Chart.yaml b/charts/grafana/Chart.yaml index 5b85a3810a..8110fb682f 100644 --- a/charts/grafana/Chart.yaml +++ b/charts/grafana/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: grafana -version: 5.8.0 +version: 5.7.1 appVersion: 7.2.0 kubeVersion: "^1.8.0-0" description: The leading tool for querying and visualizing time series and metrics. diff --git a/charts/grafana/README.md b/charts/grafana/README.md index ff59b13fa5..ce6eb90561 100644 --- a/charts/grafana/README.md +++ b/charts/grafana/README.md @@ -2,14 +2,15 @@ * Installs the web dashboarding system [Grafana](http://grafana.org/) -## Get Repo Info +## TL;DR ```console helm repo add grafana https://grafana.github.io/helm-charts -helm repo update ``` -_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ +```console +helm install grafana/grafana +``` ## Installing the Chart From 3a97c0e083c42e75e5fcaf79094b01403ee2fbe6 Mon Sep 17 00:00:00 2001 From: Torsten Walter Date: Wed, 9 Dec 2020 08:02:04 +0100 Subject: [PATCH 202/202] remove stable repo from ct.yaml --- ct.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ct.yaml b/ct.yaml index caec8658fe..0000a1ab44 100644 --- a/ct.yaml +++ b/ct.yaml @@ -4,7 +4,6 @@ target-branch: main chart-dirs: - charts chart-repos: - - stable=https://kubernetes-charts.storage.googleapis.com/ - elastic=https://helm.elastic.co/ helm-extra-args: --timeout 600s validate-maintainers: false