[Filebeat] Improve ECS categorization mappings in envoyproxy module. (#…

…18395) (#18577)

* Improve ECS categorization mappings in envoyproxy module.

- convert pipeline to yaml
- explicitly set ECS version
- event.category
- event.kind
- event.outcome
- event.type
- network.protocol
- network.transport
- related.ip

Closes #16161

(cherry picked from commit 7c3e38d)

CHANGELOG.next.asciidoc

@@ -411,6 +411,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379]
 - The `logstash` module can now automatically detect the log file format (JSON or plaintext) and process it accordingly. {issue}9964[9964] {pull}18095[18095]
 - Improve ECS categorization field mappings in coredns module. {issue}16159[16159] {pull}18424[18424]
+- Improve ECS categorization field mappings in envoyproxy module. {issue}16161[16161] {pull}18395[18395]
 
 *Heartbeat*
 

x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml

@@ -5,3 +5,7 @@ paths:
 {{ end }}
 tags: {{.tags}}
 processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml

@@ -0,0 +1,47 @@
+description: Pipeline for normalizing envoyproxy logs
+processors:
+- pipeline:
+    if: ctx.message.charAt(0) != (char)("{")
+    name: '{< IngestPipeline "pipeline-plaintext" >}'
+- pipeline:
+    if: ctx.message.charAt(0) == (char)("{")
+    name: '{< IngestPipeline "pipeline-json" >}'
+- pipeline:
+    name: '{< IngestPipeline "pipeline-geo-as" >}'
+- set:
+    field: event.created
+    value: '{{@timestamp}}'
+- append:
+    field: related.ip
+    value: "{{source.ip}}"
+    if: "ctx?.source?.ip != null"
+- append:
+    field: related.ip
+    value: "{{destination.ip}}"
+    if: "ctx?.destination?.ip != null"
+- set:
+    field: event.kind
+    value: event
+- append:
+    field: event.category
+    value: network
+- append:
+    field: event.outcome
+    value: failure
+    if: "ctx?.envoyproxy?.response_flags != null"
+- append:
+    field: event.outcome
+    value: success
+    if: "ctx?.envoyproxy?.response_flags == null"
+- set:
+    field: '@timestamp'
+    value: '{{timestamp}}'
+    if: ctx.timestamp != null
+- remove:
+    field:
+    - timestamp
+    ignore_failure: true
+on_failure:
+- set:
+    field: error.message
+    value: 'pipeline-entry: {{ _ingest.on_failure_message }}'

x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-http.yml

@@ -0,0 +1,61 @@
+description: Pipeline for normalizing envoy HTTP ACCESS logs
+processors:
+- script:
+    lang: painless
+    source: >-
+      ctx['http'] = new HashMap();
+      def p = ctx.proto.indexOf ('/');
+      def l = ctx.proto.length();
+      ctx.http.version = ctx.proto.substring(p+1, l);
+    ignore_failure: true
+- rename:
+    field: method
+    target_field: http.request.method
+- rename:
+    field: path
+    target_field: url.path
+- convert:
+    field: response_code
+    type: long
+- rename:
+    field: response_code
+    target_field: http.response.status_code
+- rename:
+    field: bytes_received
+    target_field: http.response.body.bytes
+- convert:
+    field: http.response.body.bytes
+    type: long
+- rename:
+    field: bytes_sent
+    target_field: http.request.body.bytes
+- convert:
+    field: http.request.body.bytes
+    type: long
+- script:
+    lang: painless
+    source: ctx.envoyproxy.upstream_service_time = Math.round(Double.parseDouble(ctx.upstream_service_time) * params.scale)
+    params:
+      scale: 1000000
+    if: ctx.upstream_service_time != null && ctx.upstream_service_time != '-'
+- set:
+    field: envoyproxy.proxy_type
+    value: http
+- set:
+    field: url.domain
+    value: '{{envoyproxy.authority}}'
+- user_agent:
+    field: user_agent.original
+    ignore_missing: true
+- append:
+    field: event.type
+    value:
+      - connection
+      - protocol
+- set:
+    field: network.protocol
+    value: http
+on_failure:
+- set:
+    field: error.message
+    value: 'pipeline-http: {{ _ingest.on_failure_message }}'

x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-json.yml

@@ -0,0 +1,26 @@
+description: Pipeline for normalizing envoyproxy access logs
+processors:
+- json:
+    field: message
+    target_field: json
+- remove:
+    field:
+    - message
+    ignore_failure: true
+- rename:
+    field: json.message
+    target_field: message
+    ignore_failure: true
+- rename:
+    field: json.kubernetes
+    target_field: kubernetes
+    ignore_failure: true
+- remove:
+    field:
+    - json
+- pipeline:
+    name: '{< IngestPipeline "pipeline-plaintext" >}'
+on_failure:
+- set:
+    field: error.message
+    value: 'pipeline-json: {{ _ingest.on_failure_message }}'