Fix MongoDB Filebeat module (#7568)

* add missing log lines to mongodb/log module

* fix mongodb Grok errors and follow up in expected.json

* add changelog entry

* fix expected output

CHANGELOG.asciidoc

@@ -102,6 +102,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Optimize PostgreSQL ingest pipeline to use anchored regexp and merge multiple regexp into a single expression. {pull}7269[7269]
 - Fix offset field pointing at end of a line. {issue}6514[6514]
 - Fix an issue when parsing ISO8601 dates with timezone definition {issue}7367[7367]
+- Fix MongoDB Filebeat module. {pull}7568[7568]
 
 *Heartbeat*
 - Fix race due to updates of shared a map, that was not supposed to be shared between multiple go-routines. {issue}6616[6616]

filebeat/module/mongodb/log/ingest/pipeline.json

@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "%{TIMESTAMP_ISO8601:mongodb.log.timestamp} %{WORD:mongodb.log.severity} %{WORD:mongodb.log.component}  *\\[%{WORD:mongodb.log.context}\\] %{GREEDYDATA:mongodb.log.message}"
+        "%{TIMESTAMP_ISO8601:mongodb.log.timestamp} %{WORD:mongodb.log.severity} %{WORD:mongodb.log.component} \\s*\\[%{WORD:mongodb.log.context}\\] %{GREEDYDATA:mongodb.log.message}"
       ],
       "ignore_missing": true
     }

filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log

@@ -0,0 +1,34 @@
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten] git version: 009580ad490190ba33d1c6253ebd8d91808923e4
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten] modules: none
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten] OpenSSL version: OpenSSL 1.0.2l  25 May 2017
+2018-02-05T13:44:56.677+0100 I STORAGE  [initandlisten] wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
+2018-02-05T13:44:56.724+0100 I FTDC     [initandlisten] Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'
+2018-02-05T13:44:56.724+0100 I NETWORK  [HostnameCanonicalizationWorker] Starting hostname canonicalization worker
+2018-02-05T13:44:56.744+0100 I NETWORK  [initandlisten] waiting for connections on port 27017
+2018-02-05T13:50:55.170+0100 I NETWORK  [conn1] end connection 127.0.0.1:55404 (0 connections now open)
+2018-02-05T13:50:55.487+0100 I NETWORK  [initandlisten] connection accepted from 127.0.0.1:55406 #2 (1 connection now open)
+2018-02-05T14:49:45.606+0100 I CONTROL  [signalProcessingThread] now exiting
+2018-02-05T14:49:45.606+0100 I NETWORK  [signalProcessingThread] closing listening socket: 7
+2018-02-05T14:49:45.606+0100 I NETWORK  [signalProcessingThread] removing socket file: /run/mongodb/mongodb-27017.sock
+2018-02-05T14:49:45.606+0100 I NETWORK  [signalProcessingThread] shutdown: going to flush diaglog...
+2018-02-05T14:49:45.606+0100 I NETWORK  [signalProcessingThread] shutdown: going to close sockets...
+2018-02-05T14:49:45.688+0100 I STORAGE  [signalProcessingThread] shutdown: removing fs lock...
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten] db version v3.2.11
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten] build environment:
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten]     distarch: x86_64
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten] options: { config: "/etc/mongodb.conf", net: { bindIp: "127.0.0.1", unixDomainSocket: { pathPrefix: "/run/mongodb" } }, storage: { dbPath: "/var/lib/mongodb", journal: { enabled: true } }, systemLog: { destination: "file", logAppend: true, path: "/var/log/mongodb/mongodb.log" } }
+2018-02-05T13:50:55.170+0100 I NETWORK  [initandlisten] connection accepted from 127.0.0.1:55404 #1 (1 connection now open)
+2018-02-05T13:50:56.180+0100 I NETWORK  [conn3] end connection 127.0.0.1:55414 (0 connections now open)
+2018-02-05T14:15:42.095+0100 I NETWORK  [conn4] end connection 127.0.0.1:58336 (0 connections now open)
+2018-02-05T14:49:45.606+0100 I NETWORK  [signalProcessingThread] shutdown: going to close listening sockets...
+2018-02-05T14:49:45.606+0100 I STORAGE  [signalProcessingThread] WiredTigerKVEngine shutting down
+2018-02-05T14:49:45.688+0100 I CONTROL  [signalProcessingThread] dbexit:  rc: 0
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten] MongoDB starting : pid=29803 port=27017 dbpath=/var/lib/mongodb 64-bit host=sleipnir
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten] allocator: tcmalloc
+2018-02-05T13:44:56.657+0100 I CONTROL  [initandlisten]     target_arch: x86_64
+2018-02-05T13:50:55.487+0100 I NETWORK  [conn2] end connection 127.0.0.1:55406 (0 connections now open)
+2018-02-05T13:50:56.180+0100 I NETWORK  [initandlisten] connection accepted from 127.0.0.1:55414 #3 (1 connection now open)
+2018-02-05T14:11:41.401+0100 I NETWORK  [initandlisten] connection accepted from 127.0.0.1:58336 #4 (1 connection now open)
+2018-02-05T14:49:45.605+0100 I CONTROL  [signalProcessingThread] got signal 15 (Terminated), will terminate after current cmd ends
+2018-02-05T14:49:45.605+0100 I FTDC     [signalProcessingThread] Shutting down full-time diagnostic data capture
+2018-02-05T14:49:45.606+0100 I NETWORK  [signalProcessingThread] closing listening socket: 6