diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8c636b95014..e63eabc3f1d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -271,6 +271,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Improve logging of request and response with request trace logging in error conditions. {pull}39455[39455] - Add HTTP metrics to CEL input. {issue}39501[39501] {pull}39503[39503] - Add default user-agent to CEL HTTP requests. {issue}39502[39502] {pull}39587[39587] +- Improve reindexing support in security module pipelines. {issue}38224[38224] {pull}[] +- Improve reindexing support in security module pipelines. {issue}38224[38224] {pull}39588[39588] *Auditbeat* diff --git a/filebeat/module/santa/log/ingest/pipeline.yml b/filebeat/module/santa/log/ingest/pipeline.yml index 47f0dce8fc8..e5d8711a1eb 100644 --- a/filebeat/module/santa/log/ingest/pipeline.yml +++ b/filebeat/module/santa/log/ingest/pipeline.yml @@ -13,6 +13,8 @@ processors: - rename: field: message target_field: event.original + ignore_missing: true + if: ctx.event?.original == null - date: field: process.start target_field: process.start diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml index 3fd9f440c33..4aed2d6c0e2 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml @@ -11,6 +11,7 @@ processors: field: message target_field: event.original ignore_missing: true + if: ctx.event?.original == null - grok: field: event.original patterns: diff --git a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml index 914e0b8093d..b54ed783cfe 100644 --- a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml @@ -12,6 +12,7 @@ processors: - set: field: event.original value: "{{message}}" + if: ctx.event?.original == null ############ # DNS Logs # ############ diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml index 0b89ab35252..54ebb542b65 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml @@ -5,6 +5,7 @@ processors: field: message target_field: event.original ignore_failure: true + if: ctx.event?.original == null - json: field: event.original target_field: json diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml index 43b4b8e9e5a..075dd7920a7 100644 --- a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml @@ -54,6 +54,7 @@ processors: - rename: field: message target_field: event.original + if: ctx.event?.original == null - grok: field: iptables.ubiquiti.rule_set ignore_missing: true diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml index 7e34e24fdc2..ca93fc9693a 100644 --- a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml @@ -14,6 +14,7 @@ processors: field: message target_field: event.original ignore_failure: true + if: ctx.event?.original == null # Get the timezone from the IETF header if present. Otherwise the timezone # value added by the add_locale processor will be used. diff --git a/x-pack/filebeat/module/threatintel/abusemalware/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/abusemalware/ingest/pipeline.yml index 61528d6d625..47e6a8d8710 100644 --- a/x-pack/filebeat/module/threatintel/abusemalware/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/abusemalware/ingest/pipeline.yml @@ -24,6 +24,7 @@ processors: field: message target_field: event.original ignore_missing: true + if: ctx.event?.original == null - json: field: event.original target_field: abusech.malware diff --git a/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml index b8d3168354f..1533ef4c50c 100644 --- a/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml @@ -24,6 +24,7 @@ processors: field: message target_field: event.original ignore_missing: true + if: ctx.event?.original == null - json: field: event.original target_field: abusech.url diff --git a/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml index 3ac6f78ecee..b7eb0f8287a 100644 --- a/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml @@ -24,6 +24,7 @@ processors: field: message target_field: event.original ignore_missing: true + if: ctx.event?.original == null - json: field: event.original target_field: anomali.limo diff --git a/x-pack/filebeat/module/threatintel/malwarebazaar/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/malwarebazaar/ingest/pipeline.yml index f2b81b4e25f..3865aaf0d22 100644 --- a/x-pack/filebeat/module/threatintel/malwarebazaar/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/malwarebazaar/ingest/pipeline.yml @@ -24,6 +24,7 @@ processors: field: message target_field: event.original ignore_missing: true + if: ctx.event?.original == null - json: field: event.original target_field: abusech.malwarebazaar diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml index 8caba8e738c..b071e4e0767 100644 --- a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml @@ -24,6 +24,7 @@ processors: field: message target_field: event.original ignore_missing: true + if: ctx.event?.original == null - json: field: event.original target_field: json diff --git a/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml index 1cb82b7ecad..7dea4e815bb 100644 --- a/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml @@ -24,6 +24,7 @@ processors: field: message target_field: event.original ignore_missing: true + if: ctx.event?.original == null - json: field: event.original target_field: otx diff --git a/x-pack/filebeat/module/threatintel/threatq/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/threatq/ingest/pipeline.yml index 9fc8229f2ea..9d75fb9ed13 100644 --- a/x-pack/filebeat/module/threatintel/threatq/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/threatq/ingest/pipeline.yml @@ -24,6 +24,7 @@ processors: field: message target_field: event.original ignore_missing: true + if: ctx.event?.original == null - json: field: event.original target_field: json