diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 971fa079ff0a..4b9f73f4ef76 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -209,6 +209,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add Logstash module support for main log and the slow log, support the plain text or structured JSON format {pull}5481[5481] - Add stream filtering when using `docker` prospector. {pull}6057[6057] - Add support for CRI logs format. {issue}5630[5630] +- Add json.ignore_decoding_error config to not log json decoding erors. {issue}6547[6547] *Heartbeat* diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index 3f247c32d236..8b4357ecf172 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -422,6 +422,9 @@ key must be at the top level in the JSON object and the value associated with the key must be a string, otherwise no filtering or multiline aggregation will occur. +*`ignore_decoding_error`*:: An optional configuration setting that specifies if JSON decoding errors should be +logged or not. If set to true, errors will not be logged. The default is false. + [float] ==== `multiline` diff --git a/filebeat/harvester/reader/json.go b/filebeat/harvester/reader/json.go index e84a90c809ed..0a7d2dbedb83 100644 --- a/filebeat/harvester/reader/json.go +++ b/filebeat/harvester/reader/json.go @@ -29,7 +29,9 @@ func (r *JSON) decodeJSON(text []byte) ([]byte, common.MapStr) { err := unmarshal(text, &jsonFields) if err != nil || jsonFields == nil { - logp.Err("Error decoding JSON: %v", err) + if !r.cfg.IgnoreDecodingError { + logp.Err("Error decoding JSON: %v", err) + } if r.cfg.AddErrorKey { jsonFields = common.MapStr{"error": createJSONError(fmt.Sprintf("Error decoding JSON: %v", err))} } diff --git a/filebeat/harvester/reader/json_config.go b/filebeat/harvester/reader/json_config.go index ed993fa1c5f8..41e0911d7114 100644 --- a/filebeat/harvester/reader/json_config.go +++ b/filebeat/harvester/reader/json_config.go @@ -1,10 +1,11 @@ package reader type JSONConfig struct { - MessageKey string `config:"message_key"` - KeysUnderRoot bool `config:"keys_under_root"` - OverwriteKeys bool `config:"overwrite_keys"` - AddErrorKey bool `config:"add_error_key"` + MessageKey string `config:"message_key"` + KeysUnderRoot bool `config:"keys_under_root"` + OverwriteKeys bool `config:"overwrite_keys"` + AddErrorKey bool `config:"add_error_key"` + IgnoreDecodingError bool `config:"ignore_decoding_error"` } func (c *JSONConfig) Validate() error { diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index 56ddae6d81fe..d86abbede60d 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -74,6 +74,7 @@ filebeat.{{input_config | default("inputs")}}: {% if json.keys_under_root %}keys_under_root: true{% endif %} {% if json.overwrite_keys %}overwrite_keys: true{% endif %} {% if json.add_error_key %}add_error_key: true{% endif %} + {% if json.ignore_decoding_error %}ignore_decoding_error: true{% endif %} {% endif %} {% if multiline %} diff --git a/filebeat/tests/system/test_json.py b/filebeat/tests/system/test_json.py index 0e2f58b3a05b..74996c788fee 100644 --- a/filebeat/tests/system/test_json.py +++ b/filebeat/tests/system/test_json.py @@ -316,6 +316,72 @@ def test_with_generic_filtering(self): # We drop null values during the generic event conversion. assert "res" not in o + def test_json_decoding_error_true(self): + """ + Test if json_decoding_error is set to true, that no errors are logged. + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/*", + json=dict( + message_key="message", + ignore_decoding_error=True + ), + ) + + os.mkdir(self.working_dir + "/log/") + + testfile1 = self.working_dir + "/log/test.log" + + message = "invalidjson" + with open(testfile1, 'a') as f: + f.write(message + "\n") + + proc = self.start_beat() + self.wait_until( + lambda: self.output_has(lines=1), + max_timeout=10) + proc.check_kill_and_wait() + + output = self.read_output( + required_fields=["@timestamp"], + ) + assert len(output) == 1 + assert output[0]["message"] == message + assert False == self.log_contains_count("Error decoding JSON") + + def test_json_decoding_error_false(self): + """ + Test if json_decoding_error is set to false, that an errors is logged. + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/*", + json=dict( + message_key="message", + ignore_decoding_error=False + ), + ) + + os.mkdir(self.working_dir + "/log/") + + testfile1 = self.working_dir + "/log/test.log" + + message = "invalidjson" + with open(testfile1, 'a') as f: + f.write(message + "\n") + + proc = self.start_beat() + self.wait_until( + lambda: self.output_has(lines=1), + max_timeout=10) + proc.check_kill_and_wait() + + output = self.read_output( + required_fields=["@timestamp"], + ) + assert len(output) == 1 + assert output[0]["message"] == message + assert True == self.log_contains_count("Error decoding JSON") + def test_with_generic_filtering_remove_headers(self): """ It should work fine to combine JSON decoding with diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 067989f80e31..9ac14e597c9c 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -210,7 +210,7 @@ def render_config_template(self, template_name=None, output_path = os.path.join(self.working_dir, output) with open(output_path, "wb") as f: - os.chmod(output_path, 0600) + os.chmod(output_path, 0o600) f.write(output_str.encode('utf8')) # Returns output as JSON object with flattened fields (. notation)