Don't use multiline options in Logstash module with json format (elas…

…tic#11346)
ycombinator · Mar 21, 2019 · f76f293 · f76f293
1 parent 4588257
commit f76f293
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 3 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -179,6 +179,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change URLPATH grok pattern to support brackets. {issue}11135[11135] {pull}11252[11252]
 - Add support for iis log with different address format. {issue}11255[11255] {pull}11256[11256]
 - Add fix to parse syslog message with priority value 0. {issue}11010[11010]
+- Don't apply multiline rules in Logstash json logs. {pull}11346[11346]
 
 *Heartbeat*
 

diff --git a/filebeat/module/logstash/log/config/log.yml b/filebeat/module/logstash/log/config/log.yml
@@ -4,10 +4,13 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+
+{{ if eq .format "plain" }}
 multiline:
  pattern: ^\[[0-9]{4}-[0-9]{2}-[0-9]{2}
  negate: true
  match: after
+{{ end }}
 
 {{ if .convert_timezone }}
 processors:

diff --git a/filebeat/module/logstash/log/test/logstash-json.log-expected.json b/filebeat/module/logstash/log/test/logstash-json.log-expected.json
@@ -6,9 +6,6 @@
  "event.module": "logstash",
  "fileset.name": "log",
  "input.type": "log",
- "log.flags": [
- "multiline"
- ],
  "log.level": "INFO",
  "log.offset": 0,
  "logstash.log.log_event.count": 1,
@@ -28,5 +25,36 @@
  "logstash.log.thread": "Ruby-0-Thread-1: /Users/mat/work/elastic/releases/6.5.1/logstash/lib/bootstrap/environment.rb:6",
  "message": "Pipelines running",
  "service.type": "logstash"
+ },
+ {
+ "@timestamp": "2019-01-07T21:25:22.538Z",
+ "ecs.version": "1.0.0",
+ "event.dataset": "logstash.log",
+ "event.module": "logstash",
+ "fileset.name": "log",
+ "input.type": "log",
+ "log.level": "INFO",
+ "log.offset": 357,
+ "logstash.log.log_event.pipeline_id": "main",
+ "logstash.log.log_event.thread": "#<Thread:0x7d16ffef run>",
+ "logstash.log.module": "logstash.pipeline",
+ "logstash.log.thread": "[main]>worker7",
+ "message": "Pipeline has terminated",
+ "service.type": "logstash"
+ },
+ {
+ "@timestamp": "2019-01-07T21:25:22.594Z",
+ "ecs.version": "1.0.0",
+ "event.dataset": "logstash.log",
+ "event.module": "logstash",
+ "fileset.name": "log",
+ "input.type": "log",
+ "log.level": "INFO",
+ "log.offset": 566,
+ "logstash.log.log_event.port": 9600,
+ "logstash.log.module": "logstash.agent",
+ "logstash.log.thread": "Api Webserver",
+ "message": "Successfully started Logstash API endpoint",
+ "service.type": "logstash"
  }
 ]