diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b79ac8d9702c..16345c981c0d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -853,6 +853,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Mark `filestream` input beta. {pull}25560[25560] - Update PanOS module to parse Global Protect & User ID logs. {issue}24722[24722] {issue}24724[24724] {pull}24927[24927] - Add HMAC signature validation support for http_endpoint input. {pull}24918[24918] +- Add new grok pattern for iptables module for Ubiquiti UDM {issue}25615[25615] {pull}25616[25616] *Heartbeat* diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml index 3afa3d818863..e5af7c8d258d 100644 --- a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml @@ -8,10 +8,11 @@ processors: patterns: - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{SPACE}%{IPTABLES_HOSTNAME}%{GREEDYDATA}\[%{UBIQUITI_LABEL}\]%{IPTABLES}%{SPACE}' - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{SPACE}%{IPTABLES_ACTION}%{GREEDYDATA}%{IPTABLES}%{SPACE}' + - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{SPACE}%{IPTABLES_HOSTNAME}%{SPACE}%{UDM_LOGS}%{IPTABLES_IP_PAYLOAD}' - '%{GREEDYDATA}\[%{UBIQUITI_LABEL}\]%{IPTABLES}%{SPACE}' - '%{GREEDYDATA}%{IPTABLES}%{SPACE}' pattern_definitions: - IPTABLES_HOSTNAME: '%{HOSTNAME:observer.name}%{SPACE}kernel:' + IPTABLES_HOSTNAME: '%{HOSTNAME:observer.name}%{SPACE}(%{NOTSPACE}%{SPACE})?kernel:' IPTABLES_ACTION: '(:?%{WORD:event.action}:|%{IPTABLES_HOSTNAME}%{SPACE}iptables%{SPACE}%{WORD:event.action}|%{IPTABLES_HOSTNAME})' UNSIGNED_INT: '[0-9]+' ETHTYPE: (?:[A-Fa-f0-9]{2}):(?:[A-Fa-f0-9]{2}) @@ -49,6 +50,7 @@ processors: UBIQUITI_FIELD: '[^-\]]*' UBIQUITI_RULESET_NAME: '[^\]]*' UBIQUITI_LABEL: '%{UBIQUITI_RULESET_NAME:iptables.ubiquiti.rule_set}-%{UBIQUITI_FIELD:iptables.ubiquiti.rule_number}-%{UBIQUITI_FIELD:event.action}' + UDM_LOGS: '(%{UNSIGNED_INT}%{SPACE})?(TTL|TL|L)=(%{UNSIGNED_INT:iptables.ttl:int})%{SPACE}(ID=(%{UNSIGNED_INT:iptables.id:int})%{SPACE})?(DF%{SPACE})?' - rename: field: message target_field: log.original diff --git a/x-pack/filebeat/module/iptables/log/test/ubiquiti.log b/x-pack/filebeat/module/iptables/log/test/ubiquiti.log index c795c77a5168..b15c3a33a65a 100644 --- a/x-pack/filebeat/module/iptables/log/test/ubiquiti.log +++ b/x-pack/filebeat/module/iptables/log/test/ubiquiti.log @@ -1,5 +1,11 @@ -Jan 5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520 -Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0 -Jan 5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0 -Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0 -Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0 +Jan 5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520 +Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0 +Jan 5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0 +Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0 +Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0 +May 5 20:46:45 My-Office-Gateway user.info kernel: TTL=126 ID=15317 DF PROTO=TCP SPT=59344 DPT=443 WINDOW=8212 RES=0x00 ACK PSH URGP=0 +May 5 20:46:46 My-Office-Gateway user.info kernel: TTL=126 ID=51392 DF PROTO=TCP SPT=51653 DPT=7914 WINDOW=1024 RES=0x00 ACK PSH URGP=0 +May 5 20:46:46 My-Office-Gateway user.info kernel: L=126 ID=8698 DF PROTO=TCP SPT=88 DPT=51179 WINDOW=2053 RES=0x00 ACK URGP=0 +May 5 20:47:09 My-Office-Gateway user.info kernel: 0 TTL=126 ID=15461 DF PROTO=TCP SPT=59289 DPT=443 WINDOW=8208 RES=0x00 ACK PSH URGP=0 +May 5 20:46:56 My-Office-Gateway user.info kernel: L=126 ID=8702 DF PROTO=TCP SPT=88 DPT=51182 WINDOW=2053 RES=0x00 ACK URGP=0 +May 5 20:45:44 My-Office-Gateway user.info kernel: TL=126 ID=4622 DF PROTO=TCP SPT=389 DPT=49209 WINDOW=8192 RES=0x00 ECE ACK SYN URGP=0 diff --git a/x-pack/filebeat/module/iptables/log/test/ubiquiti.log-expected.json b/x-pack/filebeat/module/iptables/log/test/ubiquiti.log-expected.json index 14fac58cb166..a3295d0c5f4b 100644 --- a/x-pack/filebeat/module/iptables/log/test/ubiquiti.log-expected.json +++ b/x-pack/filebeat/module/iptables/log/test/ubiquiti.log-expected.json @@ -29,7 +29,7 @@ "iptables.ubiquiti.rule_set": "LAN_LOCAL", "iptables.udp.length": 520, "log.offset": 0, - "log.original": "Jan 5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520 ", + "log.original": "Jan 5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520", "network.community_id": "1:3qoibVBmc9hsnHpP4Ms5HO6ls7Q=", "network.transport": "udp", "network.type": "ipv4", @@ -86,8 +86,8 @@ "iptables.ttl": 63, "iptables.ubiquiti.rule_number": "2000", "iptables.ubiquiti.rule_set": "WAN_OUT", - "log.offset": 252, - "log.original": "Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0 ", + "log.offset": 251, + "log.original": "Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0", "network.community_id": "1:7bPQdYPL4yePwQJZt0I1dvVXLHc=", "network.transport": "tcp", "network.type": "ipv4", @@ -145,8 +145,8 @@ "iptables.ubiquiti.output_zone": "dest", "iptables.ubiquiti.rule_number": "default", "iptables.ubiquiti.rule_set": "source-dest", - "log.offset": 513, - "log.original": "Jan 5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0 ", + "log.offset": 511, + "log.original": "Jan 5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0", "network.community_id": "1:6BwNFzns3BNljtYZJCwhPO5Qoq0=", "network.transport": "tcp", "network.type": "ipv4", @@ -204,8 +204,8 @@ "iptables.ttl": 63, "iptables.ubiquiti.rule_number": "2000", "iptables.ubiquiti.rule_set": "WAN_OUT", - "log.offset": 774, - "log.original": "Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0 ", + "log.offset": 771, + "log.original": "Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0", "network.community_id": "1:6BwNFzns3BNljtYZJCwhPO5Qoq0=", "network.transport": "tcp", "network.type": "ipv4", @@ -261,8 +261,8 @@ "iptables.ttl": 63, "iptables.ubiquiti.rule_number": "2000", "iptables.ubiquiti.rule_set": "WAN_OUT", - "log.offset": 1028, - "log.original": "Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0 ", + "log.offset": 1024, + "log.original": "Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0", "network.community_id": "1:6BwNFzns3BNljtYZJCwhPO5Qoq0=", "network.transport": "tcp", "network.type": "ipv4", @@ -281,5 +281,184 @@ "iptables", "forwarded" ] + }, + { + "destination.port": 443, + "event.category": [ + "network" + ], + "event.dataset": "iptables.log", + "event.kind": "event", + "event.module": "iptables", + "event.timezone": "-02:00", + "fileset.name": "log", + "input.type": "log", + "iptables.id": 15317, + "iptables.tcp.flags": [ + "ACK", + "PSH" + ], + "iptables.tcp.reserved_bits": 0, + "iptables.tcp.window": 8212, + "iptables.ttl": 126, + "log.offset": 1277, + "log.original": "May 5 20:46:45 My-Office-Gateway user.info kernel: TTL=126 ID=15317 DF PROTO=TCP SPT=59344 DPT=443 WINDOW=8212 RES=0x00 ACK PSH URGP=0", + "network.transport": "tcp", + "observer.name": "My-Office-Gateway", + "service.type": "iptables", + "source.port": 59344, + "tags": [ + "iptables", + "forwarded" + ] + }, + { + "destination.port": 7914, + "event.category": [ + "network" + ], + "event.dataset": "iptables.log", + "event.kind": "event", + "event.module": "iptables", + "event.timezone": "-02:00", + "fileset.name": "log", + "input.type": "log", + "iptables.id": 51392, + "iptables.tcp.flags": [ + "ACK", + "PSH" + ], + "iptables.tcp.reserved_bits": 0, + "iptables.tcp.window": 1024, + "iptables.ttl": 126, + "log.offset": 1413, + "log.original": "May 5 20:46:46 My-Office-Gateway user.info kernel: TTL=126 ID=51392 DF PROTO=TCP SPT=51653 DPT=7914 WINDOW=1024 RES=0x00 ACK PSH URGP=0", + "network.transport": "tcp", + "observer.name": "My-Office-Gateway", + "service.type": "iptables", + "source.port": 51653, + "tags": [ + "iptables", + "forwarded" + ] + }, + { + "destination.port": 51179, + "event.category": [ + "network" + ], + "event.dataset": "iptables.log", + "event.kind": "event", + "event.module": "iptables", + "event.timezone": "-02:00", + "fileset.name": "log", + "input.type": "log", + "iptables.id": 8698, + "iptables.tcp.flags": [ + "ACK" + ], + "iptables.tcp.reserved_bits": 0, + "iptables.tcp.window": 2053, + "iptables.ttl": 126, + "log.offset": 1551, + "log.original": "May 5 20:46:46 My-Office-Gateway user.info kernel: L=126 ID=8698 DF PROTO=TCP SPT=88 DPT=51179 WINDOW=2053 RES=0x00 ACK URGP=0", + "network.transport": "tcp", + "observer.name": "My-Office-Gateway", + "service.type": "iptables", + "source.port": 88, + "tags": [ + "iptables", + "forwarded" + ] + }, + { + "destination.port": 443, + "event.category": [ + "network" + ], + "event.dataset": "iptables.log", + "event.kind": "event", + "event.module": "iptables", + "event.timezone": "-02:00", + "fileset.name": "log", + "input.type": "log", + "iptables.id": 15461, + "iptables.tcp.flags": [ + "ACK", + "PSH" + ], + "iptables.tcp.reserved_bits": 0, + "iptables.tcp.window": 8208, + "iptables.ttl": 126, + "log.offset": 1679, + "log.original": "May 5 20:47:09 My-Office-Gateway user.info kernel: 0 TTL=126 ID=15461 DF PROTO=TCP SPT=59289 DPT=443 WINDOW=8208 RES=0x00 ACK PSH URGP=0", + "network.transport": "tcp", + "observer.name": "My-Office-Gateway", + "service.type": "iptables", + "source.port": 59289, + "tags": [ + "iptables", + "forwarded" + ] + }, + { + "destination.port": 51182, + "event.category": [ + "network" + ], + "event.dataset": "iptables.log", + "event.kind": "event", + "event.module": "iptables", + "event.timezone": "-02:00", + "fileset.name": "log", + "input.type": "log", + "iptables.id": 8702, + "iptables.tcp.flags": [ + "ACK" + ], + "iptables.tcp.reserved_bits": 0, + "iptables.tcp.window": 2053, + "iptables.ttl": 126, + "log.offset": 1817, + "log.original": "May 5 20:46:56 My-Office-Gateway user.info kernel: L=126 ID=8702 DF PROTO=TCP SPT=88 DPT=51182 WINDOW=2053 RES=0x00 ACK URGP=0", + "network.transport": "tcp", + "observer.name": "My-Office-Gateway", + "service.type": "iptables", + "source.port": 88, + "tags": [ + "iptables", + "forwarded" + ] + }, + { + "destination.port": 49209, + "event.category": [ + "network" + ], + "event.dataset": "iptables.log", + "event.kind": "event", + "event.module": "iptables", + "event.timezone": "-02:00", + "fileset.name": "log", + "input.type": "log", + "iptables.id": 4622, + "iptables.tcp.flags": [ + "ECE", + "ACK", + "SYN" + ], + "iptables.tcp.reserved_bits": 0, + "iptables.tcp.window": 8192, + "iptables.ttl": 126, + "log.offset": 1945, + "log.original": "May 5 20:45:44 My-Office-Gateway user.info kernel: TL=126 ID=4622 DF PROTO=TCP SPT=389 DPT=49209 WINDOW=8192 RES=0x00 ECE ACK SYN URGP=0", + "network.transport": "tcp", + "observer.name": "My-Office-Gateway", + "service.type": "iptables", + "source.port": 389, + "tags": [ + "iptables", + "forwarded" + ] } ] \ No newline at end of file