diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6eb475d8ab7..4d24881b951 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -626,7 +626,7 @@ field. You can revert this change by configuring tags for the module and omittin - Add event.ingested to all Filebeat modules. {pull}20386[20386] - Return error when log harvester tries to open a named pipe. {issue}18682[18682] {pull}20450[20450] - Avoid goroutine leaks in Filebeat readers. {issue}19193[19193] {pull}20455[20455] - +- Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867] *Heartbeat* diff --git a/x-pack/filebeat/module/zeek/x509/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/x509/ingest/pipeline.yml index 060db4179c1..db9317cca6e 100644 --- a/x-pack/filebeat/module/zeek/x509/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/x509/ingest/pipeline.yml @@ -17,42 +17,133 @@ processors: field: event.id value: '{{zeek.session_id}}' if: ctx.zeek.session_id != null + - set: + field: file.x509.signature_algorithm + value: '{{zeek.x509.certificate.signature_algorithm}}' + ignore_empty_value: true + - script: + lang: painless + params: + "md2WithRSAEncryption": MD2-RSA + "md5WithRSAEncryption": MD5-RSA + "sha-1WithRSAEncryption": SHA1-RSA + "sha256WithRSAEncryption": SHA256-RSA + "sha384WithRSAEncryption": SHA384-RSA + "sha512WithRSAEncryption": SHA512-RSA + "dsaWithSha1": DSA-SHA1 + "dsaWithSha256": DSA-SHA256 + "ecdsa-with-SHA1": ECDSA-SHA1 + "ecdsa-with-SHA256": ECDSA-SHA256 + "ecdsa-with-SHA384": ECDSA-SHA384 + "ecdsa-with-SHA512": ECDSA-SHA512 + "id-Ed25519": Ed25519 + source: | + String algo = params.get(ctx.file.x509.signature_algorithm); + if (algo != null) { + ctx.file.x509.signature_algorithm = algo; + } + if: ctx?.file?.x509?.signature_algorithm != null + - set: + field: file.x509.public_key_algorithm + value: '{{zeek.x509.certificate.key.algorithm}}' + ignore_empty_value: true + - convert: + field: zeek.x509.certificate.key.length + target_field: file.x509.public_key_size + type: long + ignore_missing: true - dot_expander: field: certificate.exponent path: zeek.x509 + - convert: + field: zeek.x509.certificate.exponent + target_field: file.x509.public_key_exponent + type: long + ignore_missing: true - dot_expander: field: certificate.serial path: zeek.x509 + - set: + field: file.x509.serial_number + value: '{{zeek.x509.certificate.serial}}' + ignore_empty_value: true - dot_expander: field: certificate.version path: zeek.x509 + - set: + field: file.x509.version_number + value: '{{zeek.x509.certificate.version}}' + ignore_empty_value: true - dot_expander: field: san.dns path: zeek.x509 + - foreach: + field: zeek.x509.san.dns + ignore_missing: true + processor: + append: + field: file.x509.alternative_names + value: '{{_ingest._value}}' - dot_expander: field: san.uri path: zeek.x509 + - foreach: + field: zeek.x509.san.uri + ignore_missing: true + processor: + append: + field: file.x509.alternative_names + value: '{{_ingest._value}}' - dot_expander: field: san.email path: zeek.x509 + - foreach: + field: zeek.x509.san.email + ignore_missing: true + processor: + append: + field: file.x509.alternative_names + value: '{{_ingest._value}}' - dot_expander: field: san.ip path: zeek.x509 + - foreach: + field: zeek.x509.san.ip + ignore_missing: true + processor: + append: + field: file.x509.alternative_names + value: '{{_ingest._value}}' - dot_expander: field: san.other_fields path: zeek.x509 + - foreach: + field: zeek.x509.san.other_fields + ignore_missing: true + processor: + append: + field: file.x509.alternative_names + value: '{{_ingest._value}}' - date: field: zeek.x509.certificate.valid.from target_field: zeek.x509.certificate.valid.from formats: - UNIX if: ctx.zeek.x509.certificate?.valid?.from != null + - set: + field: file.x509.not_before + value: '{{zeek.x509.certificate.valid.from}}' + ignore_empty_value: true - date: field: zeek.x509.certificate.valid.until target_field: zeek.x509.certificate.valid.until formats: - UNIX if: ctx.zeek.x509.certificate?.valid?.until != null + - set: + field: file.x509.not_after + value: '{{zeek.x509.certificate.valid.until}}' + ignore_empty_value: true - gsub: field: zeek.x509.certificate.iss pattern: \\, @@ -71,26 +162,50 @@ processors: field: zeek.x509.certificate.issuer.C target_field: zeek.x509.certificate.issuer.country ignore_missing: true + - set: + field: file.x509.issuer.country + value: '{{zeek.x509.certificate.issuer.country}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.issuer.CN target_field: zeek.x509.certificate.issuer.common_name ignore_missing: true + - set: + field: file.x509.issuer.common_name + value: '{{zeek.x509.certificate.issuer.common_name}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.issuer.L target_field: zeek.x509.certificate.issuer.locality ignore_missing: true + - set: + field: file.x509.issuer.locality + value: '{{zeek.x509.certificate.issuer.locality}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.issuer.O target_field: zeek.x509.certificate.issuer.organization ignore_missing: true + - set: + field: file.x509.issuer.organization + value: '{{zeek.x509.certificate.issuer.organization}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.issuer.OU target_field: zeek.x509.certificate.issuer.organizational_unit ignore_missing: true + - set: + field: file.x509.issuer.organizational_unit + value: '{{zeek.x509.certificate.issuer.organizational_unit}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.issuer.ST target_field: zeek.x509.certificate.issuer.state ignore_missing: true + - set: + field: file.x509.issuer.state_or_province + value: '{{zeek.x509.certificate.issuer.state}}' + ignore_empty_value: true - gsub: field: zeek.x509.certificate.sub pattern: \\, @@ -109,27 +224,51 @@ processors: field: zeek.x509.certificate.subject.C target_field: zeek.x509.certificate.subject.country ignore_missing: true + - set: + field: file.x509.subject.country + value: '{{zeek.x509.certificate.subject.country}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.subject.CN target_field: zeek.x509.certificate.subject.common_name ignore_missing: true + - set: + field: file.x509.subject.common_name + value: '{{zeek.x509.certificate.subject.common_name}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.subject.L target_field: zeek.x509.certificate.subject.locality ignore_missing: true + - set: + field: file.x509.subject.locality + value: '{{zeek.x509.certificate.subject.locality}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.subject.O target_field: zeek.x509.certificate.subject.organization ignore_missing: true + - set: + field: file.x509.subject.organization + value: '{{zeek.x509.certificate.subject.organization}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.subject.OU target_field: zeek.x509.certificate.subject.organizational_unit ignore_missing: true + - set: + field: file.x509.subject.organizational_unit + value: '{{zeek.x509.certificate.subject.organizational_unit}}' + ignore_empty_value: true - rename: field: zeek.x509.certificate.subject.ST target_field: zeek.x509.certificate.subject.state ignore_missing: true + - set: + field: file.x509.subject.state_or_province + value: '{{zeek.x509.certificate.subject.state}}' + ignore_empty_value: true on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' + value: '{{_ingest.on_failure_message}}' diff --git a/x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json b/x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json index fff83c5969e..603a125ee62 100644 --- a/x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json @@ -8,6 +8,95 @@ "event.type": [ "info" ], + "file.x509.alternative_names": [ + "www.bing.com", + "dict.bing.com.cn", + "*.platform.bing.com", + "*.bing.com", + "bing.com", + "ieonline.microsoft.com", + "*.windowssearch.com", + "cn.ieonline.microsoft.com", + "*.origin.bing.com", + "*.mm.bing.net", + "*.api.bing.com", + "ecn.dev.virtualearth.net", + "*.cn.bing.net", + "*.cn.bing.com", + "ssl-api.bing.com", + "ssl-api.bing.net", + "*.api.bing.net", + "*.bingapis.com", + "bingsandbox.com", + "feedback.microsoft.com", + "insertmedia.bing.office.net", + "r.bat.bing.com", + "*.r.bat.bing.com", + "*.dict.bing.com.cn", + "*.dict.bing.com", + "*.ssl.bing.com", + "*.appex.bing.com", + "*.platform.cn.bing.com", + "wp.m.bing.com", + "*.m.bing.com", + "global.bing.com", + "windowssearch.com", + "search.msn.com", + "*.bingsandbox.com", + "*.api.tiles.ditu.live.com", + "*.ditu.live.com", + "*.t0.tiles.ditu.live.com", + "*.t1.tiles.ditu.live.com", + "*.t2.tiles.ditu.live.com", + "*.t3.tiles.ditu.live.com", + "*.tiles.ditu.live.com", + "3d.live.com", + "api.search.live.com", + "beta.search.live.com", + "cnweb.search.live.com", + "dev.live.com", + "ditu.live.com", + "farecast.live.com", + "image.live.com", + "images.live.com", + "local.live.com.au", + "localsearch.live.com", + "ls4d.search.live.com", + "mail.live.com", + "mapindia.live.com", + "local.live.com", + "maps.live.com", + "maps.live.com.au", + "mindia.live.com", + "news.live.com", + "origin.cnweb.search.live.com", + "preview.local.live.com", + "search.live.com", + "test.maps.live.com", + "video.live.com", + "videos.live.com", + "virtualearth.live.com", + "wap.live.com", + "webmaster.live.com", + "webmasters.live.com", + "www.local.live.com.au", + "www.maps.live.com.au" + ], + "file.x509.issuer.common_name": "Microsoft IT TLS CA 5", + "file.x509.issuer.country": "US", + "file.x509.issuer.locality": "Redmond", + "file.x509.issuer.organization": "Microsoft Corporation", + "file.x509.issuer.organizational_unit": "Microsoft IT", + "file.x509.issuer.state_or_province": "Washington", + "file.x509.not_after": "2019-07-10T17:47:08.000Z", + "file.x509.not_before": "2017-07-20T17:47:08.000Z", + "file.x509.public_key_algorithm": "rsaEncryption", + "file.x509.public_key_exponent": 65537, + "file.x509.public_key_size": 2048, + "file.x509.serial_number": "2D00003299D7071DB7D1708A42000000003299", + "file.x509.signature_algorithm": "SHA256-RSA", + "file.x509.subject.common_name": "www.bing.com", + "file.x509.version_number": "3", "fileset.name": "x509", "input.type": "log", "log.offset": 0,