Remove event.original removal processors (integrations B* to C*) (ela…

…stic#10897) Delete the remove event.original processors from the pipelines because as of 8.11 the Fleet final pipeline now does this automatically when the `preserve_original_event` tag is not present in the event. This bumps the minimum version to 8.11.0 because it depends on the Fleet final pipeline changes. Relates elastic#10072
harnish-elastic · Sep 3, 2024 · 8c719de · 8c719de
1 parent 272d8b9
commit 8c719de
Show file tree

Hide file tree

Showing 136 changed files with 203 additions and 415 deletions.
diff --git a/packages/barracuda/changelog.yml b/packages/barracuda/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.15.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "1.14.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/barracuda/data_stream/waf/elasticsearch/ingest_pipeline/default.yml b/packages/barracuda/data_stream/waf/elasticsearch/ingest_pipeline/default.yml
@@ -80,11 +80,6 @@ processors:
   - remove:
       field:
         - _temp
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 ################################################################
 # Handle failures
 ################################################################

diff --git a/packages/barracuda/manifest.yml b/packages/barracuda/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: barracuda
 title: "Barracuda Web Application Firewall"
-version: "1.14.0"
+version: "1.15.0"
 description: "Collect logs from Barracuda Web Application Firewall with Elastic Agent."
 type: integration
 source:

diff --git a/packages/barracuda_cloudgen_firewall/changelog.yml b/packages/barracuda_cloudgen_firewall/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.13.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "1.12.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/...ges/barracuda_cloudgen_firewall/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/...ges/barracuda_cloudgen_firewall/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -169,11 +169,6 @@ processors:
             list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
         }
         handleMap(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/barracuda_cloudgen_firewall/manifest.yml b/packages/barracuda_cloudgen_firewall/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: barracuda_cloudgen_firewall
 title: Barracuda CloudGen Firewall Logs
-version: "1.12.0"
+version: "1.13.0"
 description: Collect logs from Barracuda CloudGen Firewall devices with Elastic Agent.
 categories: ["network", "security", "firewall_security"]
 type: integration

diff --git a/packages/bitdefender/changelog.yml b/packages/bitdefender/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.1.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "2.0.0"
   changes:
     - description: Add jsonRPC format as recommended default.

diff --git a/...ages/bitdefender/data_stream/push_configuration/elasticsearch/ingest_pipeline/default.yml b/...ages/bitdefender/data_stream/push_configuration/elasticsearch/ingest_pipeline/default.yml
@@ -29,11 +29,6 @@ processors:
         - json
         - bitdefender.push.configuration.serviceSettings.authorization
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/bitdefender/data_stream/push_statistics/elasticsearch/ingest_pipeline/default.yml b/packages/bitdefender/data_stream/push_statistics/elasticsearch/ingest_pipeline/default.yml
@@ -29,11 +29,6 @@ processors:
         - json
         - bitdefender.push.stats.lastUpdateTime
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/bitdefender/manifest.yml b/packages/bitdefender/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: bitdefender
 title: "BitDefender"
-version: "2.0.0"
+version: "2.1.0"
 source:
   license: "Elastic-2.0"
 description: "Ingest BitDefender GravityZone logs and data"

diff --git a/packages/box_events/changelog.yml b/packages/box_events/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.10.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "2.9.1"
   changes:
     - description: Fix handling of empty API responses.

diff --git a/packages/box_events/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/box_events/data_stream/events/elasticsearch/ingest_pipeline/default.yml
@@ -1223,12 +1223,6 @@ processors:
       field: threat.indicator.as.organization_name
       target_field: threat.indicator.as.organization.name
       ignore_missing: true
-# Conditionally, remove event.original
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 # Drop empty/null fields
   - script:
       description: Drops null/empty values recursively

diff --git a/packages/box_events/manifest.yml b/packages/box_events/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: box_events
 title: Box Events
-version: "2.9.1"
+version: "2.10.0"
 description: "Collect logs from Box with Elastic Agent"
 type: integration
 categories:

diff --git a/packages/carbon_black_cloud/changelog.yml b/packages/carbon_black_cloud/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.5.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "2.4.0"
   changes:
     - description: Deprecate global SQS Queue URL to avoid data loss.

diff --git a/packages/carbon_black_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/carbon_black_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
@@ -327,11 +327,6 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - lowercase:
       field: json.category
       ignore_missing: true

diff --git a/packages/carbon_black_cloud/data_stream/alert_v7/elasticsearch/ingest_pipeline/default.yml b/packages/carbon_black_cloud/data_stream/alert_v7/elasticsearch/ingest_pipeline/default.yml
@@ -838,11 +838,6 @@ processors:
       field: carbon_black_cloud.alert.category
       value: 'THREAT'     
   # cleanup & final additions
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - remove:
       field:
         - json.severity

diff --git a/...k_cloud/data_stream/asset_vulnerability_summary/elasticsearch/ingest_pipeline/default.yml b/...k_cloud/data_stream/asset_vulnerability_summary/elasticsearch/ingest_pipeline/default.yml
@@ -149,10 +149,6 @@ processors:
           return false;
         }
         dropEmptyFields(ctx);
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/carbon_black_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/carbon_black_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -113,11 +113,6 @@ processors:
           return false;
         }
         dropEmptyFields(ctx);
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/...s/carbon_black_cloud/data_stream/endpoint_event/elasticsearch/ingest_pipeline/default.yml b/...s/carbon_black_cloud/data_stream/endpoint_event/elasticsearch/ingest_pipeline/default.yml
@@ -792,10 +792,6 @@ processors:
       if: ctx.carbon_black_cloud?.endpoint_event?.device?.external_ip != null
       allow_duplicates: false
       ignore_failure: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
   - remove:
       field:
         - json.remote_port

diff --git a/...es/carbon_black_cloud/data_stream/watchlist_hit/elasticsearch/ingest_pipeline/default.yml b/...es/carbon_black_cloud/data_stream/watchlist_hit/elasticsearch/ingest_pipeline/default.yml
@@ -350,10 +350,6 @@ processors:
           return false;
         }
         dropEmptyFields(ctx);
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
   - remove:
       field:
         - json.process_pid

diff --git a/packages/carbon_black_cloud/manifest.yml b/packages/carbon_black_cloud/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: carbon_black_cloud
 title: VMware Carbon Black Cloud
-version: "2.4.0"
+version: "2.5.0"
 description: Collect logs from VMWare Carbon Black Cloud with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/cassandra/changelog.yml b/packages/cassandra/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.16.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "1.15.0"
   changes:
     - description: Add processor support for metrics data stream.

diff --git a/packages/cassandra/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cassandra/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -72,11 +72,6 @@ processors:
               return false;
           }
           drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
-      ignore_failure: true
 on_failure:
 - set:
     field: error.message

diff --git a/packages/cassandra/manifest.yml b/packages/cassandra/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: cassandra
 title: Cassandra
-version: "1.15.0"
+version: "1.16.0"
 description: This Elastic integration collects logs and metrics from cassandra.
 type: integration
 categories:

diff --git a/packages/ceph/changelog.yml b/packages/ceph/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.7.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "1.6.0"
   changes:
     - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/ceph/data_stream/cluster_disk/elasticsearch/ingest_pipeline/default.yml b/packages/ceph/data_stream/cluster_disk/elasticsearch/ingest_pipeline/default.yml
@@ -58,10 +58,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - remove:
       field:
         - json

diff --git a/packages/ceph/data_stream/cluster_health/elasticsearch/ingest_pipeline/default.yml b/packages/ceph/data_stream/cluster_health/elasticsearch/ingest_pipeline/default.yml
@@ -54,10 +54,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - remove:
       field:
         - json

diff --git a/packages/ceph/data_stream/cluster_status/elasticsearch/ingest_pipeline/default.yml b/packages/ceph/data_stream/cluster_status/elasticsearch/ingest_pipeline/default.yml
@@ -142,10 +142,6 @@ processors:
       field:
         - json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

diff --git a/packages/ceph/data_stream/osd_performance/elasticsearch/ingest_pipeline/default.yml b/packages/ceph/data_stream/osd_performance/elasticsearch/ingest_pipeline/default.yml
@@ -54,10 +54,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - remove:
       field:
         - json

diff --git a/packages/ceph/data_stream/osd_pool_stats/elasticsearch/ingest_pipeline/default.yml b/packages/ceph/data_stream/osd_pool_stats/elasticsearch/ingest_pipeline/default.yml
@@ -78,10 +78,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - remove:
       field:
         - json

diff --git a/packages/ceph/data_stream/osd_tree/elasticsearch/ingest_pipeline/default.yml b/packages/ceph/data_stream/osd_tree/elasticsearch/ingest_pipeline/default.yml
@@ -106,10 +106,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - remove:
       field:
         - json

diff --git a/packages/ceph/data_stream/pool_disk/elasticsearch/ingest_pipeline/default.yml b/packages/ceph/data_stream/pool_disk/elasticsearch/ingest_pipeline/default.yml
@@ -70,10 +70,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - remove:
       field:
         - json

diff --git a/packages/ceph/manifest.yml b/packages/ceph/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: ceph
 title: Ceph
-version: "1.6.0"
+version: "1.7.0"
 description: This Elastic integration collects metrics from Ceph instance.
 type: integration
 categories:

diff --git a/packages/checkpoint/changelog.yml b/packages/checkpoint/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.33.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "1.32.0"
   changes:
     - description: Migrate log stream visualization to saved search.

diff --git a/packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml b/packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml
@@ -1282,11 +1282,6 @@ processors:
           return false;
         }
         dropEmptyFields(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/checkpoint/manifest.yml b/packages/checkpoint/manifest.yml
@@ -1,13 +1,13 @@
 name: checkpoint
 title: Check Point
-version: "1.32.0"
+version: "1.33.0"
 description: Collect logs from Check Point with Elastic Agent.
 type: integration
 format_version: "3.0.3"
 categories: [security, network, firewall_security]
 conditions:
   kibana:
-    version: "^8.10.1"
+    version: "^8.11.0"
 icons:
   - src: /img/checkpoint-logo.svg
     title: Check Point

diff --git a/packages/cisa_kevs/changelog.yml b/packages/cisa_kevs/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10897
 - version: "1.2.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.