diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1ed3adc4a83..41709892e44 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -505,6 +505,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add event.ingested for CrowdStrike module {pull}20138[20138] - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138] - Add event.ingested for Suricata module {pull}20220[20220] +- Add event.ingested to all Filebeat modules. {pull}20386[20386] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index 6311bfef12b..a9f23eb4a10 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index 967f7a34b69..aad4c3f4a5f 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing apache error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/auditd/log/ingest/pipeline.yml b/filebeat/module/auditd/log/ingest/pipeline.yml index 2b7c114f10a..26a8bf2ab91 100644 --- a/filebeat/module/auditd/log/ingest/pipeline.yml +++ b/filebeat/module/auditd/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing Linux auditd logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml index 8f1093f5eea..ec3873d2b9f 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch audit logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml index 59b8cf882f9..e1f4838df9b 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch deprecation logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml index fc8ec5c73e3..d0980763ecc 100644 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Elasticsearch JVM garbage collection logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 6e09a9dbde8..4d4e634cc4b 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch server logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml index 360e86d9d77..ea501d9b3e0 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch slow logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml index fdcfc828701..d9315df0f02 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.yml +++ b/filebeat/module/haproxy/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing HAProxy http, tcp and default logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/debug/ingest/pipeline.yml b/filebeat/module/icinga/debug/ingest/pipeline.yml index ee25b38e90e..dbe9f1ee39d 100644 --- a/filebeat/module/icinga/debug/ingest/pipeline.yml +++ b/filebeat/module/icinga/debug/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga debug logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/main/ingest/pipeline.yml b/filebeat/module/icinga/main/ingest/pipeline.yml index 5db480e07ab..654e8c3c4e7 100644 --- a/filebeat/module/icinga/main/ingest/pipeline.yml +++ b/filebeat/module/icinga/main/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga main logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/startup/ingest/pipeline.yml b/filebeat/module/icinga/startup/ingest/pipeline.yml index 61e0e6fef27..aee7377b140 100644 --- a/filebeat/module/icinga/startup/ingest/pipeline.yml +++ b/filebeat/module/icinga/startup/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga startup logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 8344cccac1b..84fabdc59b8 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing IIS access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index 4611744d3c9..a16fde841da 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing IIS error logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/kafka/log/ingest/pipeline.yml b/filebeat/module/kafka/log/ingest/pipeline.yml index a1072489122..aa72addb642 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.yml +++ b/filebeat/module/kafka/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Kafka log messages processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message trace_match: true diff --git a/filebeat/module/kibana/log/ingest/pipeline.yml b/filebeat/module/kibana/log/ingest/pipeline.yml index 0112e09fcfc..ced76d42c23 100644 --- a/filebeat/module/kibana/log/ingest/pipeline.yml +++ b/filebeat/module/kibana/log/ingest/pipeline.yml @@ -4,6 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/log/ingest/pipeline.yml b/filebeat/module/logstash/log/ingest/pipeline.yml index 0a416e5758e..e7dc228a76d 100644 --- a/filebeat/module/logstash/log/ingest/pipeline.yml +++ b/filebeat/module/logstash/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash node logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline.yml b/filebeat/module/logstash/slowlog/ingest/pipeline.yml index 061a4f8c636..949ffdcb91e 100644 --- a/filebeat/module/logstash/slowlog/ingest/pipeline.yml +++ b/filebeat/module/logstash/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash slow logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/mongodb/log/ingest/pipeline.yml b/filebeat/module/mongodb/log/ingest/pipeline.yml index 6460a2b02c6..9355e031802 100644 --- a/filebeat/module/mongodb/log/ingest/pipeline.yml +++ b/filebeat/module/mongodb/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MongoDB logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index b11f280d1ea..baf4c11aa40 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MySQL error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index 93ce577a330..d3fbe49707c 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -1,6 +1,11 @@ { "description": "Pipeline for parsing MySQL slow logs.", "processors": [{ + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "grok": { "field": "message", "patterns":[ diff --git a/filebeat/module/nats/log/ingest/pipeline.yml b/filebeat/module/nats/log/ingest/pipeline.yml index 53c4f774b5e..bece77c1b8e 100644 --- a/filebeat/module/nats/log/ingest/pipeline.yml +++ b/filebeat/module/nats/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing nats log logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index f07e82f2b60..57fe9031b55 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -145,7 +148,7 @@ processors: - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - append: field: related.ip value: "{{source.ip}}" diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml index 5a33c34710c..05691eeb737 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.yml +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing the Nginx error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 74118b7405e..c9f4a5860c7 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx ingress controller access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/osquery/result/ingest/pipeline.json b/filebeat/module/osquery/result/ingest/pipeline.json index cbc45c202f9..c14b9664d1e 100644 --- a/filebeat/module/osquery/result/ingest/pipeline.json +++ b/filebeat/module/osquery/result/ingest/pipeline.json @@ -2,6 +2,11 @@ "description": "Pipeline for parsing osquery result logs", "processors": [ { + "set":{ + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "rename": { "field": "@timestamp", "target_field": "event.created" diff --git a/filebeat/module/postgresql/log/ingest/pipeline.yml b/filebeat/module/postgresql/log/ingest/pipeline.yml index bd208d1eb72..9233ed95c5f 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.yml +++ b/filebeat/module/postgresql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing PostgreSQL logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/redis/log/ingest/pipeline.yml b/filebeat/module/redis/log/ingest/pipeline.yml index d1c08cab378..472c3398e36 100644 --- a/filebeat/module/redis/log/ingest/pipeline.yml +++ b/filebeat/module/redis/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing redis logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/santa/log/ingest/pipeline.yml b/filebeat/module/santa/log/ingest/pipeline.yml index 9b68cce3644..e914253f8ee 100644 --- a/filebeat/module/santa/log/ingest/pipeline.yml +++ b/filebeat/module/santa/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Google Santa logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/system/auth/ingest/pipeline.yml b/filebeat/module/system/auth/ingest/pipeline.yml index 3f45705416a..a958855936a 100644 --- a/filebeat/module/system/auth/ingest/pipeline.yml +++ b/filebeat/module/system/auth/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing system authorisation/secure logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/system/syslog/ingest/pipeline.yml b/filebeat/module/system/syslog/ingest/pipeline.yml index e0c80b9aad6..2963ba410b0 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.yml +++ b/filebeat/module/system/syslog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Syslog messages. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index ce489a4a92c..dd5de1b0b0b 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - dissect: field: message pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 9360abf51af..b02a98b2f51 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -161,6 +161,10 @@ def run_on_file(self, module, fileset, test_file, cfgfile): assert obj["event"]["module"] == module, "expected event.module={} but got {}".format( module, obj["event"]["module"]) + # All modules must include a set processor that adds the time that + # the event was ingested to Elasticsearch + assert "ingested" in obj["event"], "missing event.ingested timestamp" + assert "error" not in obj, "not error expected but got: {}".format( obj) diff --git a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml index afc4c50f3dc..c5cb5ee8ed1 100644 --- a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ audit logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml index c33d77295e5..b84807be893 100644 --- a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index 36773124439..42395228853 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for AWS CloudTrail Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: "message" target_field: "event.original" @@ -614,7 +617,7 @@ processors: if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) { ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin); } - + def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); diff --git a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml index ff7e20d1c3d..05f25463414 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for CloudWatch logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml index 0ada24c6f77..878aa14aef5 100644 --- a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for EC2 logs in CloudWatch" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index fbd1195dcae..de772ccdf01 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for ELB logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message # Classic ELB patterns documented in https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index efd1a9d358a..dd8613a904a 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for s3 server access logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index 0ad04419cbd..bd9b1d32769 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for AWS VPC Flow Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # Convert Unix epoch to timestamp - date: field: "aws.vpcflow.end" diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml index dac11495608..f8f10132a0d 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml index 2bf26322faf..e6a29f6cc13 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml index 9d5351bf36a..77ccfa32dec 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure signin logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml b/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml index dffea972086..1616836706d 100644 --- a/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml +++ b/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Barracuda Web Application Firewall processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml b/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml index e26891a1ad0..8f8064017d4 100644 --- a/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml +++ b/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Blue Coat Director processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml index 75a86ea2758..7dab1ca3382 100644 --- a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Filebeat CEF processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml index 9a74b0b7c72..d21d421ce0f 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing checkpoint firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -157,7 +160,7 @@ processors: target_field: source.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true if: "ctx.checkpoint?.xlatesport != '0'" - rename: field: checkpoint.mac_source_address @@ -691,7 +694,7 @@ processors: field: client.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: client.bytes type: long @@ -711,7 +714,7 @@ processors: field: server.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: server.bytes type: long @@ -721,7 +724,7 @@ processors: field: server.packets type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - script: lang: painless source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" @@ -797,4 +800,4 @@ processors: on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml index 6ffe20df8f5..a09d2b31c5e 100644 --- a/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Cisco IOS logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index d34f3562e68..28a2750b6d4 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1,6 +1,9 @@ --- description: "Pipeline for Cisco {< .internal_PREFIX >} logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # # Parse the syslog header # diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml index f25d3417836..0a14b12f4c1 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -1,6 +1,9 @@ --- description: Pipeline for normalizing Kubernetes CoreDNS logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - pipeline: if: ctx.message.charAt(0) == (char)("{") name: '{< IngestPipeline "pipeline-json" >}' diff --git a/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml b/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml index d6bca1e8c47..286058aea62 100644 --- a/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for CylanceProtect processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml index 2bc7e14fb4f..296d932f2ce 100644 --- a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml @@ -1,5 +1,8 @@ description: Pipeline for normalizing envoyproxy logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - pipeline: if: ctx.message.charAt(0) != (char)("{") name: '{< IngestPipeline "pipeline-plaintext" >}' diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index 0ea72c6ba4d..2de20fc1a50 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Big-IP Access Policy Manager processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml index 1897a785e50..1fd14e58bd6 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Fortinet FortiClient Endpoint Security processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 2aaf7065ec1..eeb5368db55 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing fortinet firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml index 6c148a0c07c..8e0d3ac6fdb 100644 --- a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Google Cloud audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml index 8d68de684a6..b01435b7b62 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for Google Cloud Firewall Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml index 161de8ea031..a8af06f2f4b 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for Google Cloud VPC Flow Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml index 80db3a86a86..87c3deacb97 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MQ error logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - gsub: field: message pattern: ^[\-]{5}[a-z0-9\. :]*[\-]{5,} diff --git a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml index 4a84f2a8bc8..63671e09e97 100644 --- a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml +++ b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Imperva SecureSphere processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml b/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml index 5693b4aea49..cc784492797 100644 --- a/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml +++ b/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Infoblox NIOS processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml index 4eb24ff7d03..ecaa40ce67c 100644 --- a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for IPTables processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index 6dedd5e8a1f..392f3a441a7 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing microsoft atp logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - remove: field: - message @@ -76,9 +79,6 @@ processors: - set: field: event.provider value: defender_atp -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' - set: field: event.created value: '{{json.alertCreationTime}}' @@ -284,7 +284,7 @@ processors: ## Cleanup ## ############# - remove: - field: + field: - json.alertCreationTime - json.severity - json.relatedUser diff --git a/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml index 184e6c3e4a9..fd43032ff6e 100644 --- a/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Microsoft DHCP processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/misp/threat/ingest/pipeline.json b/x-pack/filebeat/module/misp/threat/ingest/pipeline.json index 0d710feeb24..59abc2fc21e 100644 --- a/x-pack/filebeat/module/misp/threat/ingest/pipeline.json +++ b/x-pack/filebeat/module/misp/threat/ingest/pipeline.json @@ -1,6 +1,12 @@ { "description": "Pipeline for normalizing MISP threat", "processors": [ + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "geoip": { "field": "destination.ip", diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml index 39a10a9ff99..cae8f53ab34 100644 --- a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline to parse MSSQL logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml index 66f9ab7bcc1..5525c2ba70f 100644 --- a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml +++ b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Arbor Peakflow SP processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 98fd4f0ff58..49371346925 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Office 365 Audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml index 78f6fa37047..e3d92540d5f 100644 --- a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml +++ b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Okta system logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true @@ -44,7 +47,7 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - + on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml index 7cc44f287b6..25d1ba1681b 100644 --- a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: "Pipeline for Palo Alto Networks PAN-OS Logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # keep message as log.original. - rename: diff --git a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml index 58097c578d8..e69402c6a95 100644 --- a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing RabbitMQ logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml b/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml index d558e7071ea..816d612b6a7 100644 --- a/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Rapid7 NeXpose processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml index 75670b6f441..fdfb0f7f9a0 100644 --- a/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Sonicwall-FW processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml index f408b6f01cd..8102bb92514 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing sophosxg firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml index caeba41fcbc..574cfafde0a 100644 --- a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Squid processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml b/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml index e5cd87682ea..16a25fde6f2 100644 --- a/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Apache Tomcat processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml index 3c6171bc045..76e5178572e 100644 --- a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek capture_loss.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.capture_loss.ts formats: diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml index b660079324a..a243e057420 100644 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek conn.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.connection.ts formats: diff --git a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml index 1ecda252cc8..e77cd5646f0 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dce_rpc.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dce_rpc.ts formats: diff --git a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml index 49df687ecc3..49216c077c2 100644 --- a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dhcp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dhcp.ts formats: diff --git a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml index ad4670dc350..f1a1e527cfc 100644 --- a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dnp3.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dnp3.ts formats: diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml index db603d93dbb..77ea898c66b 100644 --- a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Filebeat Zeek dns.log processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml index f30ff172fa8..32d1852c3e2 100644 --- a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dpd.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dpd.ts formats: diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml index 0d5abf9bdda..754720e9209 100644 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek files.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.files.ts formats: @@ -47,7 +50,7 @@ processors: - set: field: client.ip value: "{{zeek.files.rx_host}}" - if: "ctx?.zeek?.files?.rx_host != null" + if: "ctx?.zeek?.files?.rx_host != null" - append: field: related.hash value: "{{file.hash.md5}}" diff --git a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml index 7c15dce3ac5..f1f7d0b4f52 100644 --- a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ftp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ftp.ts formats: diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml index a382c25a74d..a2c4a85b994 100644 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek http.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.http.ts formats: diff --git a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml index 6a2bd6382ad..c6e64e016b8 100644 --- a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml @@ -1,9 +1,12 @@ --- description: Pipeline for normalizing Zeek intel.log. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - set: field: event.created - value: "{{_ingest.timestamp}}" + value: '{{@timestamp}}' # IP Geolocation Lookup - geoip: diff --git a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml index ec04f4e7c93..dd1e37a7035 100644 --- a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek irc.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.irc.ts formats: diff --git a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml index 05005491115..3604287cb5e 100644 --- a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek kerberos.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.kerberos.ts formats: diff --git a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml index d053a541ef5..d918b2de09a 100644 --- a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek modbus.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.modbus.ts formats: diff --git a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml index ca2c6c57172..ce2de353549 100644 --- a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek mysql.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.mysql.ts formats: diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml index c4dee6b78f2..c741d355361 100644 --- a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek notice.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.notice.ts formats: diff --git a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml index 9f76d461392..690fd54a54b 100644 --- a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ntlm.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ntlm.ts formats: diff --git a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml index 63a878825d7..462c1f36612 100644 --- a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ocsp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ocsp.ts formats: diff --git a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml index 6a7fa7dca87..6e1272a8ab2 100644 --- a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek pe.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.pe.ts formats: diff --git a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml index c69dfaefbb4..acc7fad2f03 100644 --- a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek radius.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.radius.ts formats: diff --git a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml index d6b70dd92e6..bbe4abcee9f 100644 --- a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek rdp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.rdp.ts formats: diff --git a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml index 8cf2cebdf4d..2ce5fda4e16 100644 --- a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek rfb.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.rfb.ts formats: diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml index 9982cb82d87..d8408c51133 100644 --- a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek sip.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.sip.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml index 838e9f2e8bc..0a853104351 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_cmd.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_cmd.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml index b2c7f52a29b..b1c0d3a6992 100644 --- a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_files.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_files.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml index b5752120267..e116e1bfb60 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_mapping.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_mapping.ts formats: diff --git a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml index 4424d3674ff..03e2ffb6a25 100644 --- a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smtp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smtp.ts formats: diff --git a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml index f0070ef790d..1aefc539733 100644 --- a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek snmp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.snmp.ts formats: diff --git a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml index 04a84b13177..210d9771023 100644 --- a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek socks.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.socks.ts formats: diff --git a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml index 019a44b89e0..26980d26f3d 100644 --- a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ssh.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ssh.ts formats: diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml index bbeaa24d1bd..c40801f4243 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml @@ -2,8 +2,11 @@ description: Pipeline for normalizing Zeek ssl.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ssl.ts formats: diff --git a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml index c0347161190..04e851e14a9 100644 --- a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek stats.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.stats.ts formats: diff --git a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml index 7fd848682b1..5f3432ec488 100644 --- a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek syslog.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.syslog.ts formats: diff --git a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml index 6fa5a0bc993..f4744c540d7 100644 --- a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek traceroute.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.traceroute.ts formats: diff --git a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml index 402bce5fa5d..9ca83da3305 100644 --- a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek tunnel.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.tunnel.ts formats: diff --git a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml index e0325d9a1c5..d791eb77a09 100644 --- a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek weird.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.weird.ts formats: diff --git a/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json b/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json index e35b8bbbafc..aeb1dbf36fb 100644 --- a/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json @@ -3,10 +3,16 @@ "processors": [ { "set": { - "field": "event.created", + "field": "event.ingested", "value": "{{_ingest.timestamp}}" } }, + { + "set": { + "field": "event.created", + "value": "{{@timestamp}}" + } + }, { "date": { "field": "zeek.x509.ts", diff --git a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml index 3354fb0674a..884dd6392a5 100644 --- a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Zscaler NSS processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original