fix(fluent bit): update filters to work for various k8s distros (#2170)

* Change filter to work for multiple k8s distros Signed-off-by: Sanya Kochhar <kochhars@microsoft.com>
openservicemesh · Jan 11, 2021 · df88f4a · df88f4a
1 parent d8b189c
commit df88f4a
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 11 deletions.
diff --git a/charts/osm/README.md b/charts/osm/README.md
@@ -29,6 +29,7 @@ A Helm chart to install the OSM control plane on Kubernetes
 | OpenServiceMesh.fluentBit.enableProxySupport | bool | `false` |  |
 | OpenServiceMesh.fluentBit.httpProxy | string | `""` |  |
 | OpenServiceMesh.fluentBit.httpsProxy | string | `""` |  |
+| OpenServiceMesh.fluentBit.logLevel | string | `"error"` |  |
 | OpenServiceMesh.fluentBit.name | string | `"fluentbit-logger"` |  |
 | OpenServiceMesh.fluentBit.outputPlugin | string | `"stdout"` |  |
 | OpenServiceMesh.fluentBit.primaryKey | string | `""` |  |

diff --git a/charts/osm/templates/fluentbit-configmap.yaml b/charts/osm/templates/fluentbit-configmap.yaml
@@ -17,10 +17,28 @@ data:
       Path    /var/log/containers/osm-controller-*_{{ .Release.Namespace }}_osm-controller-*.log
       Parser  cri
       Read_from_Head  on
+    # Helps grep filter identify logs of specified level generated by clusters running on cri-o
     [FILTER]
-      name       grep
-      match      *
-      regex       message /"level":"error"/
+      Name           modify
+      Match          kube.*
+      Condition      Key_value_matches message /"level":"{{ .Values.OpenServiceMesh.fluentBit.logLevel }}"/
+      Set            keep true
+    # Helps grep filter identify logs of specified level generated by clusters running on moby, containerd
+    [FILTER]
+      Name           modify
+      Match          kube.*
+      Condition      Key_value_matches log \\"level\\":\\"{{ .Values.OpenServiceMesh.fluentBit.logLevel }}\\"
+      Set            keep true
+    # Matches logs that have met conditions in any of the above filters
+    [FILTER]
+      Name           grep
+      Match          kube.*
+      Regex          keep true
+    # Removes extra "keep: true" key/value pair once matching is complete 
+    [FILTER]
+      Name           modify
+      Match          kube.*
+      Remove         keep
     [OUTPUT]
       Name    {{ .Values.OpenServiceMesh.fluentBit.outputPlugin }}
       Match   *

diff --git a/charts/osm/values.schema.json b/charts/osm/values.schema.json
@@ -215,6 +215,7 @@
                             "registry": "fluent",
                             "tag": "1.6.4",
                             "pullPolicy": "IfNotPresent",
+                            "logLevel": "error",
                             "outputPlugin": "stdout",
                             "enableProxySupport": "false",
                             "httpProxy": "",
@@ -226,6 +227,7 @@
                         "registry",
                         "tag",
                         "pullPolicy",
+                        "logLevel",
                         "outputPlugin",
                         "enableProxySupport",
                         "httpProxy",
@@ -269,6 +271,16 @@
                                 "IfNotPresent"
                             ]
                         },
+                        "logLevel": {
+                            "$id": "#/properties/OpenServiceMesh/properties/fluentBit/properties/logLevel",
+                            "type": "string",
+                            "title": "The logLevel schema",
+                            "description": "The Fluent Bit log level.",
+                            "pattern": "^(debug|info|warn|error|fatal|panic|disabled|trace)$",
+                            "examples": [
+                                "error"
+                            ]
+                        },
                         "outputPlugin": {
                             "$id": "#/properties/OpenServiceMesh/properties/fluentBit/properties/outputPlugin",
                             "type": "string",

diff --git a/charts/osm/values.yaml b/charts/osm/values.yaml
@@ -43,6 +43,7 @@ OpenServiceMesh:
     registry: fluent
     tag: 1.6.4
     pullPolicy: IfNotPresent
+    logLevel: error
     outputPlugin: stdout
     workspaceId: ""
     primaryKey: ""

diff --git a/docs/patterns/observability/logs.md b/docs/patterns/observability/logs.md
@@ -16,14 +16,10 @@ To customize log forwarding to your output, follow these steps and then reinstal
 
 2. The default configuration uses CRI log format parsing. If you are using a kubernetes distribution that causes your logs to be formatted differently, you may need to add a new parser to the `[PARSER]` section and change the `parser` name in the `[INPUT]` section to one of the parsers defined [here](https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf).
 
-3. To view all logs irrespective of log level, you may remove the `[FILTER]` section. To change the log level being filtered on, you can update the "error" value below to "debug", "info", "warn", "fatal", "panic" or "trace":
-   ```    
-   [FILTER]
-         name       grep
-         match      *
-         regex      message /"level":"error"/
-   ```
-    This regular expression may have to be customized if your logs are formatted or nested differently. If you wish to apply further filtering, explore [Fluent Bit filters](https://docs.fluentbit.io/manual/pipeline/filters).
+3. The logs are currently filtered to match "error" level logs and multiple filters have been used to cover differences in log formatting on various Kubernetes distros. 
+    * To change the log level being filtered on, you can update the `logLevel` value in `values.yaml` to "debug", "info", "warn", "fatal", "panic", "disabled" or "trace". 
+    * To view all logs irrespective of log level, you may remove the `[FILTER]` sections. 
+    * If you wish to apply further filtering, explore [Fluent Bit filters](https://docs.fluentbit.io/manual/pipeline/filters).
 
 4. Once you have updated the Fluent Bit configmap, you can deploy the sidecar during OSM installation using the `--enable-fluentbit` flag. You should now be able to interact with error logs in the output of your choice as they get generated.