Merge pull request #521 from fluent/support-cri-parser

Support CRI parser
fluent · Mar 5, 2021 · 39da6a4 · 39da6a4
2 parents 45f077d + 9178731
commit 39da6a4
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 4 deletions.
diff --git a/Makefile b/Makefile
@@ -224,6 +224,7 @@ kubernetes.conf:
 			dockerfile='$(DOCKERFILE)' \
 			version='$(VERSION)' \
 		/kubernetes.conf.erb > docker-image/$(DOCKERFILE)/conf/kubernetes.conf
+	cp $(PWD)/templates/conf/tail_container_parse.conf docker-image/$(DOCKERFILE)/conf
 
 systemd.conf:
 	mkdir -p docker-image/$(DOCKERFILE)/conf

diff --git a/templates/Gemfile.erb b/templates/Gemfile.erb
@@ -16,6 +16,7 @@ gem 'fluent-plugin-json-in-json-2', ">= 1.0.2"
 gem "fluent-plugin-record-modifier", "~> 2.1.0"
 gem "fluent-plugin-detect-exceptions", "~> 0.0.13"
 gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0"
+gem "fluent-plugin-parser-cri", "~> 0.1.0"
 <% case target when "elasticsearch6" %>
 gem "elasticsearch", "~> 6.0"
 gem "fluent-plugin-elasticsearch", "~> 4.0.9"

diff --git a/templates/README.md.erb b/templates/README.md.erb
@@ -117,6 +117,7 @@ Each image has following configurations:
 
 - fluent.conf: Destination setting, Elaticsearch, kafka and etc.
 - kubernetes.conf: k8s specific setting. `tail` input for log files and `kubernetes_metadata` filter
+- tail_container_parse.conf: parser setting for `/var/log/containers/*.log`. See also "Use CRI parser for containerd/cri-o" logs section
 - prometheus.conf: prometheus plugin for fluentd monitoring
 - systemd.conf: systemd plugin for collecting systemd-journal log. See also "Disable systemd input" section.
 
@@ -125,6 +126,24 @@ Overwrite conf file via ConfigMap. See also several examples:
 - [Cluster-level Logging in Kubernetes with Fluentd](https://medium.com/kubernetes-tutorials/cluster-level-logging-in-kubernetes-with-fluentd-e59aa2b6093a)
 - https://github.com/fluent/fluentd-kubernetes-daemonset/pull/349#issuecomment-579097659
 
+### Use CRI parser for containerd/cri-o logs
+
+This feature is available since v1.12.0-xxx-1.1.
+
+By default, these images use `json` parser for `/var/log/containers/` files because docker generates json formatted logs.
+On the other hand, containerd/cri-o use different log format. To parse such logs, you need to use [`cri` parser](https://github.com/fluent/fluent-plugin-parser-cri) instead.
+
+You can use `cri` parser by overwriting `tail_container_parse.conf` via ConfigMap.
+
+```
+# configuration example
+<parse>
+  @type cri
+</parse>
+```
+
+See also [CRI parser README](https://github.com/fluent/fluent-plugin-parser-cri#log-and-configuration-example)
+
 ### Use FLUENT_CONTAINER_TAIL_EXCLUDE_PATH to exclude specific container logs
 
 Since v1.9.3 or later images.

diff --git a/templates/conf/kubernetes.conf.erb b/templates/conf/kubernetes.conf.erb
@@ -17,10 +17,7 @@
   tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}"
   exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}"
   read_from_head true
-  <parse>
-    @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
-    time_format %Y-%m-%dT%H:%M:%S.%NZ
-  </parse>
+  @include tail_container_parse.conf
 </source>
 
 <source>

diff --git a/templates/conf/tail_container_parse.conf b/templates/conf/tail_container_parse.conf
@@ -0,0 +1,4 @@
+<parse>
+  @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
+  time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}"
+</parse>