sync with recent changes in zeek package (elastic#21155) (elastic#21181)

- always attempt community_id processor
- dnp3 reorder pipeline to be same as package

(cherry picked from commit 1932f9f)

CHANGELOG.next.asciidoc

@@ -679,6 +679,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Improve Suricata Eve module with `x509` ECS mappings {pull}20973[20973]
 - Added new module for Zoom webhooks {pull}20414[20414]
 - Add type and sub_type to panw panos fileset {pull}20912[20912]
+- Always attempt community_id processor on zeek module {pull}21155[21155]
 
 *Heartbeat*
 

x-pack/filebeat/module/zeek/connection/config/connection.yml

@@ -90,7 +90,6 @@ processors:
  kind: event
  category:
  - network
-{{ if .community_id }}
  - if:
  equals.network.transport: icmp
  then:
@@ -100,7 +99,6 @@ processors:
  icmp_code: zeek.connection.icmp.code
  else:
  community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/connection/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/conn.log
  - name: tags
  default: [zeek.connection]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/connection.yml

x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml

@@ -54,9 +54,7 @@ processors:
  - connection
  - protocol
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/dce_rpc/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/dce_rpc.log
  - name: tags
  default: [zeek.dce_rpc]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dce_rpc.yml

x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml

@@ -116,9 +116,7 @@ processors:
  - connection
  - protocol
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/dhcp/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/dhcp.log
  - name: tags
  default: [zeek.dhcp]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dhcp.yml

x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml

@@ -64,9 +64,7 @@ processors:
  - connection
  - protocol
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml

@@ -23,14 +23,6 @@ processors:
 - lowercase:
  field: event.action
  ignore_missing: true
-- append:
- field: related.ip
- value: '{{source.ip}}'
- if: ctx?.source?.ip != null
-- append:
- field: related.ip
- value: '{{destination.ip}}'
- if: ctx?.destination?.ip != null
 - geoip:
  field: destination.ip
  target_field: destination.geo
@@ -69,6 +61,14 @@ processors:
  field: destination.as.organization_name
  target_field: destination.as.organization.name
  ignore_missing: true
+- append:
+ field: related.ip
+ value: '{{source.ip}}'
+ if: ctx?.source?.ip != null
+- append:
+ field: related.ip
+ value: '{{destination.ip}}'
+ if: ctx?.destination?.ip != null
 on_failure:
 - set:
  field: error.message

x-pack/filebeat/module/zeek/dnp3/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/dnp3.log
  - name: tags
  default: [zeek.dnp3]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dnp3.yml

x-pack/filebeat/module/zeek/dns/config/dns.yml

@@ -193,9 +193,7 @@ processors:
  - connection
  - info
  - protocol
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - drop_fields:
  ignore_missing: true
  fields:

x-pack/filebeat/module/zeek/dns/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/dns.log
  - name: tags
  default: [zeek.dns]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dns.yml

x-pack/filebeat/module/zeek/dpd/config/dpd.yml

@@ -53,9 +53,7 @@ processors:
  type:
  - connection
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/dpd/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/dpd.log
  - name: tags
  default: [zeek.dpd]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dpd.yml

x-pack/filebeat/module/zeek/ftp/config/ftp.yml

@@ -82,9 +82,7 @@ processors:
  - connection
  - info
  - protocol
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/ftp/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/ftp.log
  - name: tags
  default: [zeek.ftp]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/ftp.yml

x-pack/filebeat/module/zeek/http/config/http.yml

@@ -89,9 +89,7 @@ processors:
  - connection
  - info
  - protocol
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/http/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/http.log
  - name: tags
  default: [zeek.http]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/http.yml

x-pack/filebeat/module/zeek/intel/config/intel.yml

@@ -63,9 +63,7 @@ processors:
  kind: alert
  type:
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/intel/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/intel.log
  - name: tags
  default: [zeek.intel]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/intel.yml

x-pack/filebeat/module/zeek/irc/config/irc.yml

@@ -68,9 +68,7 @@ processors:
  - connection
  - protocol
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/irc/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/irc.log
  - name: tags
  default: [zeek.irc]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/irc.yml

x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml

@@ -100,9 +100,7 @@ processors:
  tokenizer: "%{user.name}/%{user.domain}"
  field: zeek.kerberos.client
  target_prefix: ""
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/kerberos/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/kerberos.log
  - name: tags
  default: [zeek.kerberos]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/kerberos.yml

x-pack/filebeat/module/zeek/modbus/config/modbus.yml

@@ -69,9 +69,7 @@ processors:
  target: event
  fields:
  outcome: success
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/modbus/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/modbus.log
  - name: tags
  default: [zeek.modbus]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/modbus.yml

x-pack/filebeat/module/zeek/mysql/config/mysql.yml

@@ -68,9 +68,7 @@ processors:
  target: event
  fields:
  outcome: failure
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/mysql/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/mysql.log
  - name: tags
  default: [zeek.mysql]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/mysql.yml

x-pack/filebeat/module/zeek/notice/config/notice.yml

@@ -100,9 +100,7 @@ processors:
  - intrusion_detection
  type:
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/notice/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/notice.log
  - name: tags
  default: [zeek.notice]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/notice.yml

x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml

@@ -82,9 +82,7 @@ processors:
  target: event
  fields:
  outcome: failure
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/ntlm/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/ntlm.log
  - name: tags
  default: [zeek.ntlm]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/ntlm.yml

x-pack/filebeat/module/zeek/radius/config/radius.yml

@@ -54,9 +54,7 @@ processors:
  type:
  - info
  - connection
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/radius/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/radius.log
  - name: tags
  default: [zeek.radius]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/radius.yml

x-pack/filebeat/module/zeek/rdp/config/rdp.yml

@@ -84,9 +84,7 @@ processors:
  type:
  - protocol
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/rdp/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/rdp.log
  - name: tags
  default: [zeek.rdp]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/rdp.yml

x-pack/filebeat/module/zeek/rfb/config/rfb.yml

@@ -69,9 +69,7 @@ processors:
  type:
  - connection
  - info
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/rfb/manifest.yml

@@ -10,8 +10,6 @@ var:
  - /usr/local/var/logs/current/rfb.log
  - name: tags
  default: [zeek.rfb]
- - name: community_id
- default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/rfb.yml

x-pack/filebeat/module/zeek/sip/config/sip.yml

@@ -91,9 +91,7 @@ processors:
  type:
  - connection
  - protocol
-{{ if .community_id }}
  - community_id:
-{{ end }}
  - add_fields:
  target: ''
  fields: