ES slowlog module improvements

CHANGELOG.next.asciidoc

@@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Support IPv6 addresses with zone id in IIS ingest pipeline.
   {issue}9836[9836] error log: {pull}9869[9869], access log: {pull}9955[9955].
 - Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958]
+- Minor improvements in slowlog Elasticsearch module, including field type change {pull}8416[8416]
 - Make elasticsearch/audit fileset be more lenient in parsing node name. {issue}10035[10035] {pull}10135[10135]
 
 *Heartbeat*

filebeat/docs/fields.asciidoc

@@ -4471,7 +4471,7 @@ Extra source information
 *`elasticsearch.slowlog.took_millis`*::
 +
 --
-type: keyword
+type: integer
 
 example: 42
 
@@ -4482,7 +4482,7 @@ Time took in milliseconds
 *`elasticsearch.slowlog.total_hits`*::
 +
 --
-type: keyword
+type: integer
 
 example: 42
 
@@ -4493,7 +4493,7 @@ Total hits
 *`elasticsearch.slowlog.total_shards`*::
 +
 --
-type: keyword
+type: integer
 
 example: 22
 

filebeat/module/elasticsearch/slowlog/_meta/fields.yml

@@ -34,15 +34,15 @@
   - name: took_millis
     description: "Time took in milliseconds"
     example: 42
-    type: keyword
+    type: integer
   - name: total_hits
     description: "Total hits"
     example: 42
-    type: keyword
+    type: integer
   - name: total_shards
     description: "Total queried shards"
     example: 22
-    type: keyword
+    type: integer
   - name: routing
     description: "Routing"
     example: "s01HZ2QBk9jw4gtgaFtn"

filebeat/module/elasticsearch/slowlog/ingest/pipeline.json

@@ -15,7 +15,7 @@
                   "INDEXNAME": "[a-zA-Z0-9_.-]*"
               },
               "patterns": [
-                  "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{WORD:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.took_millis:int}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?"
+                  "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}%{SPACE}*\\]%{SPACE}\\[%{WORD:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.took_millis:int}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?"
               ]
           }
       },

filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog_2.log

@@ -0,0 +1,10 @@
+[2018-09-24T15:30:37,246][WARN ][i.s.s.query              ] [elasticsearch] [a][0] took[105.5micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{}], 
+[2018-09-24T15:30:37,246][WARN ][i.s.s.query              ] [elasticsearch] [a][2] took[102.7micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{}], 
+[2018-09-24T15:30:37,246][WARN ][i.s.s.query              ] [elasticsearch] [a][1] took[102.5micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{}], 
+[2018-09-24T15:30:37,246][WARN ][i.s.s.query              ] [elasticsearch] [a][3] took[102.5micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{}], 
+[2018-09-24T15:30:37,246][WARN ][i.s.s.query              ] [elasticsearch] [a][4] took[145.6micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{}], 
+[2018-09-24T15:34:48,188][WARN ][i.s.s.query              ] [elasticsearch] [a][1] took[244.5micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{"query":{"match":{"a":{"query":"xyz","operator":"OR","prefix_length":0,"max_expansions":50,"fuzzy_transpositions":true,"lenient":false,"zero_terms_query":"NONE","boost":1.0}}}}], 
+[2018-09-24T15:34:48,188][WARN ][i.s.s.query              ] [elasticsearch] [a][3] took[223.2micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{"query":{"match":{"a":{"query":"xyz","operator":"OR","prefix_length":0,"max_expansions":50,"fuzzy_transpositions":true,"lenient":false,"zero_terms_query":"NONE","boost":1.0}}}}], 
+[2018-09-24T15:34:48,188][WARN ][i.s.s.query              ] [elasticsearch] [a][2] took[223.9micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{"query":{"match":{"a":{"query":"xyz","operator":"OR","prefix_length":0,"max_expansions":50,"fuzzy_transpositions":true,"lenient":false,"zero_terms_query":"NONE","boost":1.0}}}}], 
+[2018-09-24T15:34:48,188][WARN ][i.s.s.query              ] [elasticsearch] [a][4] took[263.5micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{"query":{"match":{"a":{"query":"xyz","operator":"OR","prefix_length":0,"max_expansions":50,"fuzzy_transpositions":true,"lenient":false,"zero_terms_query":"NONE","boost":1.0}}}}], 
+[2018-09-24T15:34:48,188][WARN ][i.s.s.query              ] [elasticsearch] [a][0] took[116.1micros], took_millis[0], types[doc,doc2], stats[], search_type[QUERY_THEN_FETCH], total_shards[5], source[{"query":{"match":{"a":{"query":"xyz","operator":"OR","prefix_length":0,"max_expansions":50,"fuzzy_transpositions":true,"lenient":false,"zero_terms_query":"NONE","boost":1.0}}}}],