Skip to content

Commit

Permalink
[Filebeat][zeek] Add new x509 fields to zeek (#20867) (#20896)
Browse files Browse the repository at this point in the history 
* Add new x509 fields to zeek

* Add changelog entry

* Make requested changes

(cherry picked from commit 0712468)
  • Loading branch information
@marc-gr
marc-gr authored Sep 2, 2020
1 parent 3c312f7 commit 40e60aa
Show file tree
Hide file tree
Showing 3 changed files with 230 additions and 2 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ field. You can revert this change by configuring tags for the module and omittin
- Add event.ingested to all Filebeat modules. {pull}20386[20386]
- Return error when log harvester tries to open a named pipe. {issue}18682[18682] {pull}20450[20450]
- Avoid goroutine leaks in Filebeat readers. {issue}19193[19193] {pull}20455[20455]

- Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867]

*Heartbeat*

Expand Down
141 changes: 140 additions & 1 deletion x-pack/filebeat/module/zeek/x509/ingest/pipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,42 +17,133 @@ processors:
field: event.id
value: '{{zeek.session_id}}'
if: ctx.zeek.session_id != null
- set:
field: file.x509.signature_algorithm
value: '{{zeek.x509.certificate.signature_algorithm}}'
ignore_empty_value: true
- script:
lang: painless
params:
"md2WithRSAEncryption": MD2-RSA
"md5WithRSAEncryption": MD5-RSA
"sha-1WithRSAEncryption": SHA1-RSA
"sha256WithRSAEncryption": SHA256-RSA
"sha384WithRSAEncryption": SHA384-RSA
"sha512WithRSAEncryption": SHA512-RSA
"dsaWithSha1": DSA-SHA1
"dsaWithSha256": DSA-SHA256
"ecdsa-with-SHA1": ECDSA-SHA1
"ecdsa-with-SHA256": ECDSA-SHA256
"ecdsa-with-SHA384": ECDSA-SHA384
"ecdsa-with-SHA512": ECDSA-SHA512
"id-Ed25519": Ed25519
source: |
String algo = params.get(ctx.file.x509.signature_algorithm);
if (algo != null) {
ctx.file.x509.signature_algorithm = algo;
}
if: ctx?.file?.x509?.signature_algorithm != null
- set:
field: file.x509.public_key_algorithm
value: '{{zeek.x509.certificate.key.algorithm}}'
ignore_empty_value: true
- convert:
field: zeek.x509.certificate.key.length
target_field: file.x509.public_key_size
type: long
ignore_missing: true
- dot_expander:
field: certificate.exponent
path: zeek.x509
- convert:
field: zeek.x509.certificate.exponent
target_field: file.x509.public_key_exponent
type: long
ignore_missing: true
- dot_expander:
field: certificate.serial
path: zeek.x509
- set:
field: file.x509.serial_number
value: '{{zeek.x509.certificate.serial}}'
ignore_empty_value: true
- dot_expander:
field: certificate.version
path: zeek.x509
- set:
field: file.x509.version_number
value: '{{zeek.x509.certificate.version}}'
ignore_empty_value: true
- dot_expander:
field: san.dns
path: zeek.x509
- foreach:
field: zeek.x509.san.dns
ignore_missing: true
processor:
append:
field: file.x509.alternative_names
value: '{{_ingest._value}}'
- dot_expander:
field: san.uri
path: zeek.x509
- foreach:
field: zeek.x509.san.uri
ignore_missing: true
processor:
append:
field: file.x509.alternative_names
value: '{{_ingest._value}}'
- dot_expander:
field: san.email
path: zeek.x509
- foreach:
field: zeek.x509.san.email
ignore_missing: true
processor:
append:
field: file.x509.alternative_names
value: '{{_ingest._value}}'
- dot_expander:
field: san.ip
path: zeek.x509
- foreach:
field: zeek.x509.san.ip
ignore_missing: true
processor:
append:
field: file.x509.alternative_names
value: '{{_ingest._value}}'
- dot_expander:
field: san.other_fields
path: zeek.x509
- foreach:
field: zeek.x509.san.other_fields
ignore_missing: true
processor:
append:
field: file.x509.alternative_names
value: '{{_ingest._value}}'
- date:
field: zeek.x509.certificate.valid.from
target_field: zeek.x509.certificate.valid.from
formats:
- UNIX
if: ctx.zeek.x509.certificate?.valid?.from != null
- set:
field: file.x509.not_before
value: '{{zeek.x509.certificate.valid.from}}'
ignore_empty_value: true
- date:
field: zeek.x509.certificate.valid.until
target_field: zeek.x509.certificate.valid.until
formats:
- UNIX
if: ctx.zeek.x509.certificate?.valid?.until != null
- set:
field: file.x509.not_after
value: '{{zeek.x509.certificate.valid.until}}'
ignore_empty_value: true
- gsub:
field: zeek.x509.certificate.iss
pattern: \\,
Expand All @@ -71,26 +162,50 @@ processors:
field: zeek.x509.certificate.issuer.C
target_field: zeek.x509.certificate.issuer.country
ignore_missing: true
- set:
field: file.x509.issuer.country
value: '{{zeek.x509.certificate.issuer.country}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.issuer.CN
target_field: zeek.x509.certificate.issuer.common_name
ignore_missing: true
- set:
field: file.x509.issuer.common_name
value: '{{zeek.x509.certificate.issuer.common_name}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.issuer.L
target_field: zeek.x509.certificate.issuer.locality
ignore_missing: true
- set:
field: file.x509.issuer.locality
value: '{{zeek.x509.certificate.issuer.locality}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.issuer.O
target_field: zeek.x509.certificate.issuer.organization
ignore_missing: true
- set:
field: file.x509.issuer.organization
value: '{{zeek.x509.certificate.issuer.organization}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.issuer.OU
target_field: zeek.x509.certificate.issuer.organizational_unit
ignore_missing: true
- set:
field: file.x509.issuer.organizational_unit
value: '{{zeek.x509.certificate.issuer.organizational_unit}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.issuer.ST
target_field: zeek.x509.certificate.issuer.state
ignore_missing: true
- set:
field: file.x509.issuer.state_or_province
value: '{{zeek.x509.certificate.issuer.state}}'
ignore_empty_value: true
- gsub:
field: zeek.x509.certificate.sub
pattern: \\,
Expand All @@ -109,27 +224,51 @@ processors:
field: zeek.x509.certificate.subject.C
target_field: zeek.x509.certificate.subject.country
ignore_missing: true
- set:
field: file.x509.subject.country
value: '{{zeek.x509.certificate.subject.country}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.subject.CN
target_field: zeek.x509.certificate.subject.common_name
ignore_missing: true
- set:
field: file.x509.subject.common_name
value: '{{zeek.x509.certificate.subject.common_name}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.subject.L
target_field: zeek.x509.certificate.subject.locality
ignore_missing: true
- set:
field: file.x509.subject.locality
value: '{{zeek.x509.certificate.subject.locality}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.subject.O
target_field: zeek.x509.certificate.subject.organization
ignore_missing: true
- set:
field: file.x509.subject.organization
value: '{{zeek.x509.certificate.subject.organization}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.subject.OU
target_field: zeek.x509.certificate.subject.organizational_unit
ignore_missing: true
- set:
field: file.x509.subject.organizational_unit
value: '{{zeek.x509.certificate.subject.organizational_unit}}'
ignore_empty_value: true
- rename:
field: zeek.x509.certificate.subject.ST
target_field: zeek.x509.certificate.subject.state
ignore_missing: true
- set:
field: file.x509.subject.state_or_province
value: '{{zeek.x509.certificate.subject.state}}'
ignore_empty_value: true
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
value: '{{_ingest.on_failure_message}}'
89 changes: 89 additions & 0 deletions x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,95 @@
"event.type": [
"info"
],
"file.x509.alternative_names": [
"www.bing.com",
"dict.bing.com.cn",
"*.platform.bing.com",
"*.bing.com",
"bing.com",
"ieonline.microsoft.com",
"*.windowssearch.com",
"cn.ieonline.microsoft.com",
"*.origin.bing.com",
"*.mm.bing.net",
"*.api.bing.com",
"ecn.dev.virtualearth.net",
"*.cn.bing.net",
"*.cn.bing.com",
"ssl-api.bing.com",
"ssl-api.bing.net",
"*.api.bing.net",
"*.bingapis.com",
"bingsandbox.com",
"feedback.microsoft.com",
"insertmedia.bing.office.net",
"r.bat.bing.com",
"*.r.bat.bing.com",
"*.dict.bing.com.cn",
"*.dict.bing.com",
"*.ssl.bing.com",
"*.appex.bing.com",
"*.platform.cn.bing.com",
"wp.m.bing.com",
"*.m.bing.com",
"global.bing.com",
"windowssearch.com",
"search.msn.com",
"*.bingsandbox.com",
"*.api.tiles.ditu.live.com",
"*.ditu.live.com",
"*.t0.tiles.ditu.live.com",
"*.t1.tiles.ditu.live.com",
"*.t2.tiles.ditu.live.com",
"*.t3.tiles.ditu.live.com",
"*.tiles.ditu.live.com",
"3d.live.com",
"api.search.live.com",
"beta.search.live.com",
"cnweb.search.live.com",
"dev.live.com",
"ditu.live.com",
"farecast.live.com",
"image.live.com",
"images.live.com",
"local.live.com.au",
"localsearch.live.com",
"ls4d.search.live.com",
"mail.live.com",
"mapindia.live.com",
"local.live.com",
"maps.live.com",
"maps.live.com.au",
"mindia.live.com",
"news.live.com",
"origin.cnweb.search.live.com",
"preview.local.live.com",
"search.live.com",
"test.maps.live.com",
"video.live.com",
"videos.live.com",
"virtualearth.live.com",
"wap.live.com",
"webmaster.live.com",
"webmasters.live.com",
"www.local.live.com.au",
"www.maps.live.com.au"
],
"file.x509.issuer.common_name": "Microsoft IT TLS CA 5",
"file.x509.issuer.country": "US",
"file.x509.issuer.locality": "Redmond",
"file.x509.issuer.organization": "Microsoft Corporation",
"file.x509.issuer.organizational_unit": "Microsoft IT",
"file.x509.issuer.state_or_province": "Washington",
"file.x509.not_after": "2019-07-10T17:47:08.000Z",
"file.x509.not_before": "2017-07-20T17:47:08.000Z",
"file.x509.public_key_algorithm": "rsaEncryption",
"file.x509.public_key_exponent": 65537,
"file.x509.public_key_size": 2048,
"file.x509.serial_number": "2D00003299D7071DB7D1708A42000000003299",
"file.x509.signature_algorithm": "SHA256-RSA",
"file.x509.subject.common_name": "www.bing.com",
"file.x509.version_number": "3",
"fileset.name": "x509",
"input.type": "log",
"log.offset": 0,
Expand Down

0 comments on commit 40e60aa

Please sign in to comment.