Skip to content

Commit afbe3e8

Browse files
Joey EspinosaGitHub Enterprise
Joey Espinosa
authored and
GitHub Enterprise
committed
Merge pull request fluent#3 from pyrex-containers/prometheus
Adding prometheus plugin to kafka images
2 parents b956280 + 184e546 commit afbe3e8

File tree

147 files changed

+3650
-63
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

147 files changed

+3650
-63
lines changed

Makefile

+10
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,13 @@ ALL_IMAGES := \
1616
v0.12/alpine-loggly:v0.12.33-loggly,v0.12-loggly,stable-loggly,loggly \
1717
v0.12/alpine-logentries:v0.12.33-logentries,v0.12-logentries,stable-logentries,logentries \
1818
v0.12/alpine-cloudwatch:v0.12.33-cloudwatch,v0.12-cloudwatch,stable-cloudwatch,cloudwatch \
19+
v0.12/alpine-stackdriver:v0.12.33-stackdriver,v0.12-alpine-stackdriver,stable-stackdriver,stackdriver \
1920
v0.12/alpine-s3:v0.12.33-s3,v0.12-s3,stable-s3,s3 \
21+
v0.12/alpine-gcs:v0.12.33-gcs,v0.12-gcs,stable-gcs,gcs \
2022
v0.12/alpine-papertrail:v0.12.33-papertrail,v0.12-papertrail,stable-papertrail,papertrail \
23+
v0.12/alpine-syslog:v0.12.33-syslog,v0.12-syslog,stable-syslog,syslog \
24+
v0.12/alpine-graylog:v0.12.33-graylog,v0.12-graylog,stable-graylog,graylog \
25+
v0.12/alpine-logzio:v0.12.33-logzio,v0.12-logzio,stable-logzio,logzio \
2126
v0.12/debian-elasticsearch:v0.12.33-debian-elasticsearch,v0.12-debian-elasticsearch,debian-elasticsearch \
2227
v0.12/alpine-kafka:v0.12.33-kafka,v0.12-kafka,stable-kafka,kafka \
2328
v0.12/debian-loggly:v0.12.33-debian-loggly,v0.12-debian-loggly,debian-loggly \
@@ -26,6 +31,11 @@ ALL_IMAGES := \
2631
v0.12/debian-stackdriver:v0.12.33-debian-stackdriver,v0.12-debian-stackdriver,debian-stackdriver \
2732
v0.12/debian-s3:v0.12.33-debian-s3,v0.12-debian-s3,debian-s3 \
2833
v0.12/debian-papertrail:v0.12.33-debian-papertrail,v0.12-debian-papertrail,debian-papertrail \
34+
v0.12/debian-syslog:v0.12.33-debian-syslog,v0.12-debian-syslog,debian-syslog \
35+
v0.12/debian-gcs:v0.12.33-debian-gcs,v0.12-debian-gcs,debian-gcs \
36+
v0.12/debian-papertrail:v0.12.33-debian-papertrail,v0.12-debian-papertrail,debian-papertrail \
37+
v0.12/debian-graylog:v0.12.33-debian-graylog,v0.12-debian-graylog,debian-stable-graylog,debian-graylog \
38+
v0.12/debian-papertrail:v0.12.33-debian-papertrail,v0.12-debian-papertrail,debian-papertrail \
2939
v0.12/debian-kafka:v0.12.33-debian-kafka,v0.12-debian-kafka,debian-kafka
3040

3141
# <Dockerfile>:<version>,<tag1>,<tag2>,...

README.md

+8
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
- `v0.12-debian-logentries` [(v0.12/debian-logentries/Dockerfile)][debian-logentries-dockerfile]
1616
- `v0.12-debian-loggly` [(v0.12/debian-loggly/Dockerfile)][debian-loggly-dockerfile]
1717
- `v0.12-debian-s3` [(v0.12/debian-s3/Dockerfile)][debian-s3-dockerfile]
18+
- `v0.12-debian-gcs` [(v0.12/debian-gcs/Dockerfile)][debian-gcs-dockerfile]
1819
- `v0.12-debian-stackdriver` [(v0.12/debian-stackdriver/Dockerfile)][debian-stackdriver-dockerfile]
1920
- `v0.12-debian-papertrail` [(v0.12/debian-papertrail/Dockerfile)][debian-papertrail-dockerfile]
2021

@@ -25,6 +26,9 @@
2526
- `v0.12-alpine-kafka` [(v0.12/alpine-kafka/Dockerfile)][alpine-kafka-dockerfile]
2627
- `v0.12-alpine-logentries` [(v0.12/alpine-logentries/Dockerfile)][alpine-logentries-dockerfile]
2728
- `v0.12-alpine-loggly` [(v0.12/alpine-loggly/Dockerfile)][alpine-loggly-dockerfile]
29+
- `v0.12-alpine-s3` [(v0.12/alpine-s3/Dockerfile)][alpine-s3-dockerfile]
30+
- `v0.12-alpine-gcs` [(v0.12/alpine-gcs/Dockerfile)][alpine-gcs-dockerfile]
31+
- `v0.12-alpine-stackdriver` [(v0.12/alpine-stackdriver/Dockerfile)][alpine-stackdriver-dockerfile]
2832
- `v0.12-alpine-papertrail` [(v0.12/alpine-papertrail/Dockerfile)][alpine-papertrail-dockerfile]
2933

3034
## What is Fluentd?
@@ -70,6 +74,8 @@ through a [GitHub issue](https://github.com/fluent/fluentd-kubernetes-daemonset/
7074
[alpine-logentries-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/alpine-logentries/Dockerfile
7175
[alpine-cloudwatch-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/alpine-cloudwatch/Dockerfile
7276
[alpine-s3-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/alpine-s3/Dockerfile
77+
[alpine-gcs-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/alpine-gcs/Dockerfile
78+
[alpine-stackdriver-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/alpine-stackdriver/Dockerfile
7379
[alpine-papertrail-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/alpine-papertrail/Dockerfile
7480
[alpine-kafka-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/alpine-kafka/Dockerfile
7581

@@ -78,6 +84,8 @@ through a [GitHub issue](https://github.com/fluent/fluentd-kubernetes-daemonset/
7884
[debian-logentries-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/debian-logentries/Dockerfile
7985
[debian-cloudwatch-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/debian-cloudwatch/Dockerfile
8086
[debian-s3-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/debian-s3/Dockerfile
87+
[debian-gcs-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/debian-gcs/Dockerfile
88+
[debian-stackdriver-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/debian-stackdriver/Dockerfile
8189
[debian-papertrail-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/debian-papertrail/Dockerfile
8290
[debian-kafka-dockerfile]: https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/debian-kafka/Dockerfile
8391

docker-image/v0.12/alpine-cloudwatch/Dockerfile

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ RUN set -ex \
1616
&& echo 'gem: --no-document' >> /etc/gemrc \
1717
&& gem install fluent-plugin-secure-forward \
1818
&& gem install fluent-plugin-record-reformer \
19+
&& gem install fluent-plugin-detect-exceptions -v 0.0.9 \
1920
&& gem install aws-sdk-core -v 2.10.50 \
2021
&& gem install fluent-plugin-cloudwatch-logs -v 0.4.0 \
2122
&& gem install fluent-plugin-kubernetes_metadata_filter \

docker-image/v0.12/alpine-cloudwatch/conf/fluent.conf

+1
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@
1010
auto_create_stream true
1111
use_tag_as_stream true
1212
</match>
13+

docker-image/v0.12/alpine-cloudwatch/conf/systemd.conf

+9-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@
1111
tag kubelet
1212
</source>
1313

14+
# Logs from docker-systemd
15+
<source>
16+
@type systemd
17+
filters [{ "_SYSTEMD_UNIT": "docker.service" }]
18+
pos_file /var/log/fluentd-journald-docker.pos
19+
read_from_head true
20+
tag docker.systemd
21+
</source>
22+
1423
# Logs from systemd-journal for interesting services.
1524
<source>
1625
@type systemd
@@ -20,4 +29,3 @@
2029
tag bootkube
2130
</source>
2231

23-

docker-image/v0.12/alpine-cloudwatch/entrypoint.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22

33
set -e
44

5-
if [[ -z ${FLUENT_ELASTICSEARCH_USER} ]] ; then
5+
if [ -z ${FLUENT_ELASTICSEARCH_USER} ] ; then
66
sed -i '/FLUENT_ELASTICSEARCH_USER/d' /fluentd/etc/${FLUENTD_CONF}
77
fi
88

9-
if [[ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ]] ; then
9+
if [ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ] ; then
1010
sed -i '/FLUENT_ELASTICSEARCH_PASSWORD/d' /fluentd/etc/${FLUENTD_CONF}
1111
fi
1212

13-
exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}
13+
exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}

docker-image/v0.12/alpine-elasticsearch/Dockerfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ RUN set -ex \
1616
&& echo 'gem: --no-document' >> /etc/gemrc \
1717
&& gem install fluent-plugin-secure-forward \
1818
&& gem install fluent-plugin-record-reformer \
19-
&& gem install fluent-plugin-elasticsearch \
19+
&& gem install fluent-plugin-detect-exceptions -v 0.0.9 \
20+
&& gem install fluent-plugin-elasticsearch -v "<2" \
2021
&& gem install fluent-plugin-kubernetes_metadata_filter \
2122
&& apk del .build-deps \
2223
&& gem sources --clear-all \

docker-image/v0.12/alpine-elasticsearch/conf/fluent.conf

+2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
1212
port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
1313
scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
14+
ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
1415
user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
1516
password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}"
1617
reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'true'}"
@@ -23,3 +24,4 @@
2324
disable_retry_limit
2425
num_threads 8
2526
</match>
27+

docker-image/v0.12/alpine-elasticsearch/conf/systemd.conf

+9-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@
1111
tag kubelet
1212
</source>
1313

14+
# Logs from docker-systemd
15+
<source>
16+
@type systemd
17+
filters [{ "_SYSTEMD_UNIT": "docker.service" }]
18+
pos_file /var/log/fluentd-journald-docker.pos
19+
read_from_head true
20+
tag docker.systemd
21+
</source>
22+
1423
# Logs from systemd-journal for interesting services.
1524
<source>
1625
@type systemd
@@ -20,4 +29,3 @@
2029
tag bootkube
2130
</source>
2231

23-

docker-image/v0.12/alpine-elasticsearch/entrypoint.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22

33
set -e
44

5-
if [[ -z ${FLUENT_ELASTICSEARCH_USER} ]] ; then
5+
if [ -z ${FLUENT_ELASTICSEARCH_USER} ] ; then
66
sed -i '/FLUENT_ELASTICSEARCH_USER/d' /fluentd/etc/${FLUENTD_CONF}
77
fi
88

9-
if [[ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ]] ; then
9+
if [ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ] ; then
1010
sed -i '/FLUENT_ELASTICSEARCH_PASSWORD/d' /fluentd/etc/${FLUENTD_CONF}
1111
fi
1212

13-
exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}
13+
exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
**/*.gitkeep
+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# AUTOMATICALLY GENERATED
2+
# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb
3+
4+
FROM fluent/fluentd:v0.12.33
5+
MAINTAINER Eduardo Silva <eduardo@treasure-data.com>
6+
USER root
7+
WORKDIR /home/fluent
8+
ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH
9+
10+
RUN set -ex \
11+
&& apk update && apk upgrade \
12+
&& apk add --no-cache --virtual .build-deps \
13+
build-base \
14+
ruby-dev \
15+
libffi-dev \
16+
&& echo 'gem: --no-document' >> /etc/gemrc \
17+
&& gem install fluent-plugin-secure-forward \
18+
&& gem install fluent-plugin-record-reformer \
19+
&& gem install fluent-plugin-detect-exceptions -v 0.0.9 \
20+
&& gem install fluent-plugin-gcs -v "~> 0.3" --no-document \
21+
&& gem install fluent-plugin-kubernetes_metadata_filter \
22+
&& apk del .build-deps \
23+
&& gem sources --clear-all \
24+
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
25+
26+
# Copy configuration files
27+
COPY ./conf/fluent.conf /fluentd/etc/
28+
COPY ./conf/kubernetes.conf /fluentd/etc/
29+
30+
# Copy plugins
31+
COPY plugins /fluentd/plugins/
32+
COPY entrypoint.sh /fluentd/entrypoint.sh
33+
34+
# Environment variables
35+
ENV FLUENTD_OPT=""
36+
ENV FLUENTD_CONF="fluent.conf"
37+
38+
# jemalloc is memory optimization only available for td-agent
39+
# td-agent is provided and QA'ed by treasuredata as rpm/deb/.. package
40+
# -> td-agent (stable) vs fluentd (edge)
41+
#ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
42+
43+
# Run Fluentd
44+
CMD ["/fluentd/entrypoint.sh"]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
2+
# AUTOMATICALLY GENERATED
3+
# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb
4+
5+
@include kubernetes.conf
6+
7+
<match **>
8+
# docs: https://github.com/daichirata/fluent-plugin-gcs
9+
# this configuration relies on the nodes having permission to write on your gs bucket
10+
type gcs
11+
project "#{ENV['GCS_BUCKET_PROJECT']}"
12+
bucket "#{ENV['GCS_BUCKET_NAME']}"
13+
object_key_format %{path}%{time_slice}/%{hostname}/%{index}.%{file_extension}
14+
path logs/
15+
buffer_path /var/log/fluentd-buffers/gcs.buffer
16+
time_slice_format %Y/%m/%d
17+
time_slice_wait 10m
18+
utc
19+
</match>
20+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# AUTOMATICALLY GENERATED
2+
# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb
3+
4+
<match fluent.**>
5+
type null
6+
</match>
7+
8+
<source>
9+
type tail
10+
path /var/log/containers/*.log
11+
pos_file /var/log/fluentd-containers.log.pos
12+
time_format %Y-%m-%dT%H:%M:%S.%NZ
13+
tag kubernetes.*
14+
format json
15+
read_from_head true
16+
</source>
17+
18+
<source>
19+
type tail
20+
format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
21+
time_format %Y-%m-%d %H:%M:%S
22+
path /var/log/salt/minion
23+
pos_file /var/log/fluentd-salt.pos
24+
tag salt
25+
</source>
26+
27+
<source>
28+
type tail
29+
format syslog
30+
path /var/log/startupscript.log
31+
pos_file /var/log/fluentd-startupscript.log.pos
32+
tag startupscript
33+
</source>
34+
35+
<source>
36+
type tail
37+
format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
38+
path /var/log/docker.log
39+
pos_file /var/log/fluentd-docker.log.pos
40+
tag docker
41+
</source>
42+
43+
<source>
44+
type tail
45+
format none
46+
path /var/log/etcd.log
47+
pos_file /var/log/fluentd-etcd.log.pos
48+
tag etcd
49+
</source>
50+
51+
<source>
52+
type tail
53+
format kubernetes
54+
multiline_flush_interval 5s
55+
path /var/log/kubelet.log
56+
pos_file /var/log/fluentd-kubelet.log.pos
57+
tag kubelet
58+
</source>
59+
60+
<source>
61+
type tail
62+
format kubernetes
63+
multiline_flush_interval 5s
64+
path /var/log/kube-proxy.log
65+
pos_file /var/log/fluentd-kube-proxy.log.pos
66+
tag kube-proxy
67+
</source>
68+
69+
<source>
70+
type tail
71+
format kubernetes
72+
multiline_flush_interval 5s
73+
path /var/log/kube-apiserver.log
74+
pos_file /var/log/fluentd-kube-apiserver.log.pos
75+
tag kube-apiserver
76+
</source>
77+
78+
<source>
79+
type tail
80+
format kubernetes
81+
multiline_flush_interval 5s
82+
path /var/log/kube-controller-manager.log
83+
pos_file /var/log/fluentd-kube-controller-manager.log.pos
84+
tag kube-controller-manager
85+
</source>
86+
87+
<source>
88+
type tail
89+
format kubernetes
90+
multiline_flush_interval 5s
91+
path /var/log/kube-scheduler.log
92+
pos_file /var/log/fluentd-kube-scheduler.log.pos
93+
tag kube-scheduler
94+
</source>
95+
96+
<source>
97+
type tail
98+
format kubernetes
99+
multiline_flush_interval 5s
100+
path /var/log/rescheduler.log
101+
pos_file /var/log/fluentd-rescheduler.log.pos
102+
tag rescheduler
103+
</source>
104+
105+
<source>
106+
type tail
107+
format kubernetes
108+
multiline_flush_interval 5s
109+
path /var/log/glbc.log
110+
pos_file /var/log/fluentd-glbc.log.pos
111+
tag glbc
112+
</source>
113+
114+
<source>
115+
type tail
116+
format kubernetes
117+
multiline_flush_interval 5s
118+
path /var/log/cluster-autoscaler.log
119+
pos_file /var/log/fluentd-cluster-autoscaler.log.pos
120+
tag cluster-autoscaler
121+
</source>
122+
123+
# Example:
124+
# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
125+
# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
126+
<source>
127+
type tail
128+
format multiline
129+
multiline_flush_interval 5s
130+
format_firstline /^\S+\s+AUDIT:/
131+
# Fields must be explicitly captured by name to be parsed into the record.
132+
# Fields may not always be present, and order may change, so this just looks
133+
# for a list of key="\"quoted\" value" pairs separated by spaces.
134+
# Unknown fields are ignored.
135+
# Note: We can't separate query/response lines as format1/format2 because
136+
# they don't always come one after the other for a given query.
137+
format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
138+
time_format %FT%T.%L%Z
139+
path /var/log/kubernetes/kube-apiserver-audit.log
140+
pos_file /var/log/kube-apiserver-audit.log.pos
141+
tag kube-apiserver-audit
142+
</source>
143+
144+
<filter kubernetes.**>
145+
type kubernetes_metadata
146+
</filter>
147+

0 commit comments

Comments
 (0)