From 361eb92a7128376f7540f78ac3ccffaf15381b27 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 16 Feb 2021 12:54:40 -0500 Subject: [PATCH] [Filebeat] Convert logstash logEvent.action objects to strings (#23944) (#24063) To prevent index failures resulting from indexing a mix of strings and nested objects as logstash.log.log_event.action, this converts any objects to a stringified version of the object using painless toString(). Fixes #20709 (cherry picked from commit 0d4109da65b2db5301055de9fca3e7cc4a4e78f8) --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 7 +++++++ filebeat/module/logstash/fields.go | 2 +- filebeat/module/logstash/log/_meta/fields.yml | 2 ++ .../logstash/log/ingest/pipeline-json.yml | 10 +++++++++ .../logstash/log/test/logstash-json.log | 1 + .../log/test/logstash-json.log-expected.json | 21 +++++++++++++++++++ 7 files changed, 43 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2935cb15a65..fd5a2525be4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -231,6 +231,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Use rfc6587 framing for fortinet firewall and clientendpoint filesets when transferring over tcp. {pull}23837[23837] - Fix httpjson input logging so it doesn't conflict with ECS. {pull}23972[23972] - Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025] +- Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 972e7c4bf55..8acfa934e19 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -86603,6 +86603,13 @@ type: object -- +*`logstash.log.log_event.action`*:: ++ +-- +type: keyword + +-- + *`logstash.log.pipeline_id`*:: + -- diff --git a/filebeat/module/logstash/fields.go b/filebeat/module/logstash/fields.go index 2097117ebf7..c65c5f1955d 100644 --- a/filebeat/module/logstash/fields.go +++ b/filebeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJzsVU1v2zAMvedXED23+QE+9LKtQIB9ANvuhmLTMhdJFPSRxP9+sB0ntiN3SNsVGDDdLFp875FP1APssMlAsfRB+HoFECgozOBu2LpbAZToC0c2EJsMHlcAcD4BX7iMClcAFaEqfdZFH8AIjZO87QqNxQyk42hPO4nM00yzbOe9C9HPI6Ln2BXQIli/njpIqBxrCDXCkLRTsB79Ouc25qeHUoxXT2WHzYFdOYs9Q6hdP2s85QR2UCjhPRxqdNhRxD2aAOxIkhEB10lKoXYo5rCvoLQxFTst2jCILcfQUXHRGDLyhDbiqFguMQTQUQXKUwWdKMBjuAoOGq6CI6vkXYGS0nn7C4t56A/Kd9iAMCXshYoIJW6jlK1mulQk3QJLFhUZzOmWPuBRaNv6Wwsyt9tm8xG46lowwF/Ipb2L3guZNq9QJPwsYkWol05pkk707IKLmG4Q7lHdiKZYrlPnlvAGLK/4MJsdt86H6xT/B8E/MQiWh8CLZH8XByijtsPtOmlSSZy/qcuqKMnk7cfbqfsqNJ7nRgfwHHYL9IYeb+wMO4ONsTH4e3giFdD5e/gWQ7vT3oIPXGLhF8zOvMvJ5JqUovks6TkqNvI2gp+OWMTO74E0QsVuxBXIQI+GBZtygdepcFY4odO0XlS6H8G1l65/mSYlhIJNRTL24/H97dkrzZPv7eue4ofHqd6JUIgeS9g2o0qkG/IerxAkjGmE4bQBlnG7CbMuF3o5A/8dAAD//7rhE3k=" + return "eJzsVk1v2zAMvedXED23+QE+9LKtQIF9ANvuhmLTMhdJFPSRxP9+sB0ntiN3TdsVGDDdIlp871FPZO5gi00GiqUPwtcrgEBBYQY3w9bNCqBEXziygdhkcL8CgNMJ+MJlVLgCqAhV6bMuegdGaJzkbVdoLGYgHUd73ElknmaaZTvtnYl+HhE9xS6AFsH69dBBQuVYQ6gRhqSdgvXo0zm3MT89lGK8eipbbPbsylnsCULt+lnjMSewg0IJ72Ffo8OOIu7QBGBHkowIuE5SCrVDMYd9BaVHU7HTog2D2HAMHRUXjSEjj2gjjorlEkMAHVWgPFXQiQI8hIvgoOEiOLJK3hUoKZ03v7CYh/6gfIsNCFPCTqiIUOImStlqpnNF0ldworIWRfvVsy9jSGDJoiKDOV1zkXgQ2rYPRAuaYz7Dd48fgavuDgf4s7q0+dF7IdPuF4qEn0WsCPXSKU3SiZ5dcBHTZcUdqivRFMt16twS3oDlFe9nzefaBnOZ4n8n+Sc6yXIXeZHs72IPZdR2eF1HTSqJ8zd1WRUlmbz98XbqvgqNp77RATyF3QK9occbO8PO4NHYGPwtPJAK6PwtfIuh3WlfwQcusfALZmfe5mRyTUrRvJf0HBUbeR3BTwcsYuf3QBqhYjfiCmSgR8OCTbnA61g4K5zQaVovKt2P4NpH14+2SQmhYFORjH17fH979krz5MB+3Sy/u5/qnQiF6LGETTOqxMJwf4cpBAljGmE4bYBl3P5PSLlwlzPw3wEAAP//W7kkpQ==" } diff --git a/filebeat/module/logstash/log/_meta/fields.yml b/filebeat/module/logstash/log/_meta/fields.yml index 6ca12ca11bf..36fa8bfb0cb 100644 --- a/filebeat/module/logstash/log/_meta/fields.yml +++ b/filebeat/module/logstash/log/_meta/fields.yml @@ -19,6 +19,8 @@ type: object description: > key and value debugging information. + - name: log_event.action + type: keyword - name: pipeline_id type: keyword example: main diff --git a/filebeat/module/logstash/log/ingest/pipeline-json.yml b/filebeat/module/logstash/log/ingest/pipeline-json.yml index f14a3be2855..807079ed84e 100644 --- a/filebeat/module/logstash/log/ingest/pipeline-json.yml +++ b/filebeat/module/logstash/log/ingest/pipeline-json.yml @@ -31,6 +31,16 @@ processors: - rename: field: logstash.log.level target_field: log.level +- script: + description: Convert logstash.log.log_event.action elements to string. + if: ctx?.logstash?.log?.log_event?.action instanceof List + lang: painless + source: | + def items = []; + ctx.logstash.log.log_event.action.forEach(v -> { + items.add(v.toString()); + }); + ctx.logstash.log.log_event.action = items; - set: field: event.kind value: event diff --git a/filebeat/module/logstash/log/test/logstash-json.log b/filebeat/module/logstash/log/test/logstash-json.log index bfd931653ab..503d6ce2449 100644 --- a/filebeat/module/logstash/log/test/logstash-json.log +++ b/filebeat/module/logstash/log/test/logstash-json.log @@ -1,3 +1,4 @@ {"level":"INFO","loggerName":"logstash.agent","timeMillis":1546896321871,"thread":"Ruby-0-Thread-1: /Users/mat/work/elastic/releases/6.5.1/logstash/lib/bootstrap/environment.rb:6","logEvent":{"message":"Pipelines running","count":1,"running_pipelines":[{"metaClass":{"metaClass":{"metaClass":{"running_pipelines":"[:main]","non_running_pipelines":[]}}}}]}} {"level":"INFO","loggerName":"logstash.pipeline","timeMillis":1546896322538,"thread":"[main]>worker7","logEvent":{"message":"Pipeline has terminated","pipeline_id":"main","thread":"#"}} {"level":"INFO","loggerName":"logstash.agent","timeMillis":1546896322594,"thread":"Api Webserver","logEvent":{"message":"Successfully started Logstash API endpoint","port":9600}} +{"level":"WARN","loggerName":"logstash.outputs.elasticsearch","timeMillis":1612827484046,"thread":"[foo]>worker1","logEvent":{"message":"Could not index event to Elasticsearch.","status":400,"action":["update",{"_id":"foo-1234abcd-96c6-4828-bcd4-51d33a156431","_index":"filebeat-foo-2021.02","_type":"_doc","retry_on_conflict":1},{"metaClass":{"metaClass":{"metaClass":{"action":"[\"update\", {:_id=>\"foo-1234abcd-96c6-4828-bcd4-51d33a156431\", :_index=>\"filebeat-foo-2021.02\", :routing=>nil, :_type=>\"_doc\", :retry_on_conflict=>1}, #]","response":{"update":{"_index":"filebeat-foo-2021.02","_type":"_doc","_id":"foo-1234abcd-96c6-4828-bcd4-51d33a156431","status":400,"error":{"type":"mapper_parsing_exception","reason":"failed to parse field [bar] of type [long] in document with id 'foo-1234abcd-96c6-4828-bcd4-51d33a156431'. Preview of field's value: 'ABCDEFGHIJ'","caused_by":{"type":"illegal_argument_exception","reason":"For input string: \"ABCDEFGHIJ\""}}}}}}}}]}} diff --git a/filebeat/module/logstash/log/test/logstash-json.log-expected.json b/filebeat/module/logstash/log/test/logstash-json.log-expected.json index 4bbf77ad25f..9cf6c292e39 100644 --- a/filebeat/module/logstash/log/test/logstash-json.log-expected.json +++ b/filebeat/module/logstash/log/test/logstash-json.log-expected.json @@ -59,5 +59,26 @@ "logstash.log.thread": "Api Webserver", "message": "Successfully started Logstash API endpoint", "service.type": "logstash" + }, + { + "@timestamp": "2021-02-08T23:38:04.046Z", + "event.dataset": "logstash.log", + "event.kind": "event", + "event.module": "logstash", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "WARN", + "log.offset": 745, + "logstash.log.log_event.action": [ + "update", + "{_index=filebeat-foo-2021.02, _type=_doc, _id=foo-1234abcd-96c6-4828-bcd4-51d33a156431, retry_on_conflict=1}", + "{metaClass={metaClass={metaClass={response={update={_index=filebeat-foo-2021.02, _type=_doc, _id=foo-1234abcd-96c6-4828-bcd4-51d33a156431, error={reason=failed to parse field [bar] of type [long] in document with id 'foo-1234abcd-96c6-4828-bcd4-51d33a156431'. Preview of field's value: 'ABCDEFGHIJ', caused_by={reason=For input string: \"ABCDEFGHIJ\", type=illegal_argument_exception}, type=mapper_parsing_exception}, status=400}}, action=[\"update\", {:_id=>\"foo-1234abcd-96c6-4828-bcd4-51d33a156431\", :_index=>\"filebeat-foo-2021.02\", :routing=>nil, :_type=>\"_doc\", :retry_on_conflict=>1}, #]}}}}" + ], + "logstash.log.log_event.status": 400, + "logstash.log.module": "logstash.outputs.elasticsearch", + "logstash.log.thread": "[foo]>worker1", + "message": "Could not index event to Elasticsearch.", + "service.type": "logstash" } ] \ No newline at end of file