[Filebeat] Update Grok pattern for UDM logs (#25616)

* Update Grok pattern for UDM logs * update changelog (cherry picked from commit fcf19c3)
elastic · May 14, 2021 · fad54dd · fad54dd
1 parent 0423324
commit fad54dd
Show file tree

Hide file tree

Showing 4 changed files with 205 additions and 15 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -579,6 +579,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - New module `zookeeper` for Zookeeper service and audit logs {issue}25061[25061] {pull}25128[25128]
 - Add parsing for `haproxy.http.request.raw_request_line` field  {issue}25480[25480] {pull}25482[25482]
 - Mark `filestream` input beta. {pull}25560[25560]
+- Update PanOS module to parse Global Protect & User ID logs. {issue}24722[24722] {issue}24724[24724] {pull}24927[24927]
+- Add HMAC signature validation support for http_endpoint input. {pull}24918[24918]
+- Add new grok pattern for iptables module for Ubiquiti UDM {issue}25615[25615] {pull}25616[25616]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml
@@ -8,10 +8,11 @@ processors:
     patterns:
     - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{SPACE}%{IPTABLES_HOSTNAME}%{GREEDYDATA}\[%{UBIQUITI_LABEL}\]%{IPTABLES}%{SPACE}'
     - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{SPACE}%{IPTABLES_ACTION}%{GREEDYDATA}%{IPTABLES}%{SPACE}'
+    - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{SPACE}%{IPTABLES_HOSTNAME}%{SPACE}%{UDM_LOGS}%{IPTABLES_IP_PAYLOAD}'
     - '%{GREEDYDATA}\[%{UBIQUITI_LABEL}\]%{IPTABLES}%{SPACE}'
     - '%{GREEDYDATA}%{IPTABLES}%{SPACE}'
     pattern_definitions:
-      IPTABLES_HOSTNAME: '%{HOSTNAME:observer.name}%{SPACE}kernel:'
+      IPTABLES_HOSTNAME: '%{HOSTNAME:observer.name}%{SPACE}(%{NOTSPACE}%{SPACE})?kernel:'
       IPTABLES_ACTION: '(:?%{WORD:event.action}:|%{IPTABLES_HOSTNAME}%{SPACE}iptables%{SPACE}%{WORD:event.action}|%{IPTABLES_HOSTNAME})'
       UNSIGNED_INT: '[0-9]+'
       ETHTYPE: (?:[A-Fa-f0-9]{2}):(?:[A-Fa-f0-9]{2})
@@ -49,6 +50,7 @@ processors:
       UBIQUITI_FIELD: '[^-\]]*'
       UBIQUITI_RULESET_NAME: '[^\]]*'
       UBIQUITI_LABEL: '%{UBIQUITI_RULESET_NAME:iptables.ubiquiti.rule_set}-%{UBIQUITI_FIELD:iptables.ubiquiti.rule_number}-%{UBIQUITI_FIELD:event.action}'
+      UDM_LOGS: '(%{UNSIGNED_INT}%{SPACE})?(TTL|TL|L)=(%{UNSIGNED_INT:iptables.ttl:int})%{SPACE}(ID=(%{UNSIGNED_INT:iptables.id:int})%{SPACE})?(DF%{SPACE})?'
 - rename:
     field: message
     target_field: log.original

diff --git a/x-pack/filebeat/module/iptables/log/test/ubiquiti.log b/x-pack/filebeat/module/iptables/log/test/ubiquiti.log
@@ -1,5 +1,11 @@
-Jan  5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520 
-Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0 
-Jan  5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0 
-Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0 
-Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0 
+Jan  5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520
+Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0
+Jan  5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0
+Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0
+Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0
+May  5 20:46:45 My-Office-Gateway user.info kernel: TTL=126 ID=15317 DF PROTO=TCP SPT=59344 DPT=443 WINDOW=8212 RES=0x00 ACK PSH URGP=0
+May  5 20:46:46 My-Office-Gateway user.info kernel:  TTL=126 ID=51392 DF PROTO=TCP SPT=51653 DPT=7914 WINDOW=1024 RES=0x00 ACK PSH URGP=0
+May  5 20:46:46 My-Office-Gateway user.info kernel: L=126 ID=8698 DF PROTO=TCP SPT=88 DPT=51179 WINDOW=2053 RES=0x00 ACK URGP=0
+May  5 20:47:09 My-Office-Gateway user.info kernel: 0 TTL=126 ID=15461 DF PROTO=TCP SPT=59289 DPT=443 WINDOW=8208 RES=0x00 ACK PSH URGP=0
+May  5 20:46:56 My-Office-Gateway user.info kernel: L=126 ID=8702 DF PROTO=TCP SPT=88 DPT=51182 WINDOW=2053 RES=0x00 ACK URGP=0
+May  5 20:45:44 My-Office-Gateway user.info kernel: TL=126 ID=4622 DF PROTO=TCP SPT=389 DPT=49209 WINDOW=8192 RES=0x00 ECE ACK SYN URGP=0
diff --git a/x-pack/filebeat/module/iptables/log/test/ubiquiti.log-expected.json b/x-pack/filebeat/module/iptables/log/test/ubiquiti.log-expected.json
@@ -29,7 +29,7 @@
         "iptables.ubiquiti.rule_set": "LAN_LOCAL",
         "iptables.udp.length": 520,
         "log.offset": 0,
-        "log.original": "Jan  5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520 ",
+        "log.original": "Jan  5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520",
         "network.community_id": "1:3qoibVBmc9hsnHpP4Ms5HO6ls7Q=",
         "network.transport": "udp",
         "network.type": "ipv4",
@@ -85,8 +85,8 @@
         "iptables.ttl": 63,
         "iptables.ubiquiti.rule_number": "2000",
         "iptables.ubiquiti.rule_set": "WAN_OUT",
-        "log.offset": 252,
-        "log.original": "Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0 ",
+        "log.offset": 251,
+        "log.original": "Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0",
         "network.community_id": "1:7bPQdYPL4yePwQJZt0I1dvVXLHc=",
         "network.transport": "tcp",
         "network.type": "ipv4",
@@ -143,8 +143,8 @@
         "iptables.ubiquiti.output_zone": "dest",
         "iptables.ubiquiti.rule_number": "default",
         "iptables.ubiquiti.rule_set": "source-dest",
-        "log.offset": 513,
-        "log.original": "Jan  5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0 ",
+        "log.offset": 511,
+        "log.original": "Jan  5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0",
         "network.community_id": "1:6BwNFzns3BNljtYZJCwhPO5Qoq0=",
         "network.transport": "tcp",
         "network.type": "ipv4",
@@ -201,8 +201,8 @@
         "iptables.ttl": 63,
         "iptables.ubiquiti.rule_number": "2000",
         "iptables.ubiquiti.rule_set": "WAN_OUT",
-        "log.offset": 774,
-        "log.original": "Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0 ",
+        "log.offset": 771,
+        "log.original": "Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0",
         "network.community_id": "1:6BwNFzns3BNljtYZJCwhPO5Qoq0=",
         "network.transport": "tcp",
         "network.type": "ipv4",
@@ -257,8 +257,8 @@
         "iptables.ttl": 63,
         "iptables.ubiquiti.rule_number": "2000",
         "iptables.ubiquiti.rule_set": "WAN_OUT",
-        "log.offset": 1028,
-        "log.original": "Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0 ",
+        "log.offset": 1024,
+        "log.original": "Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0",
         "network.community_id": "1:6BwNFzns3BNljtYZJCwhPO5Qoq0=",
         "network.transport": "tcp",
         "network.type": "ipv4",
@@ -276,5 +276,184 @@
         "tags": [
             "iptables"
         ]
+    },
+    {
+        "destination.port": 443,
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "iptables.log",
+        "event.kind": "event",
+        "event.module": "iptables",
+        "event.timezone": "-02:00",
+        "fileset.name": "log",
+        "input.type": "log",
+        "iptables.id": 15317,
+        "iptables.tcp.flags": [
+            "ACK",
+            "PSH"
+        ],
+        "iptables.tcp.reserved_bits": 0,
+        "iptables.tcp.window": 8212,
+        "iptables.ttl": 126,
+        "log.offset": 1277,
+        "log.original": "May  5 20:46:45 My-Office-Gateway user.info kernel: TTL=126 ID=15317 DF PROTO=TCP SPT=59344 DPT=443 WINDOW=8212 RES=0x00 ACK PSH URGP=0",
+        "network.transport": "tcp",
+        "observer.name": "My-Office-Gateway",
+        "service.type": "iptables",
+        "source.port": 59344,
+        "tags": [
+            "iptables",
+            "forwarded"
+        ]
+    },
+    {
+        "destination.port": 7914,
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "iptables.log",
+        "event.kind": "event",
+        "event.module": "iptables",
+        "event.timezone": "-02:00",
+        "fileset.name": "log",
+        "input.type": "log",
+        "iptables.id": 51392,
+        "iptables.tcp.flags": [
+            "ACK",
+            "PSH"
+        ],
+        "iptables.tcp.reserved_bits": 0,
+        "iptables.tcp.window": 1024,
+        "iptables.ttl": 126,
+        "log.offset": 1413,
+        "log.original": "May  5 20:46:46 My-Office-Gateway user.info kernel:  TTL=126 ID=51392 DF PROTO=TCP SPT=51653 DPT=7914 WINDOW=1024 RES=0x00 ACK PSH URGP=0",
+        "network.transport": "tcp",
+        "observer.name": "My-Office-Gateway",
+        "service.type": "iptables",
+        "source.port": 51653,
+        "tags": [
+            "iptables",
+            "forwarded"
+        ]
+    },
+    {
+        "destination.port": 51179,
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "iptables.log",
+        "event.kind": "event",
+        "event.module": "iptables",
+        "event.timezone": "-02:00",
+        "fileset.name": "log",
+        "input.type": "log",
+        "iptables.id": 8698,
+        "iptables.tcp.flags": [
+            "ACK"
+        ],
+        "iptables.tcp.reserved_bits": 0,
+        "iptables.tcp.window": 2053,
+        "iptables.ttl": 126,
+        "log.offset": 1551,
+        "log.original": "May  5 20:46:46 My-Office-Gateway user.info kernel: L=126 ID=8698 DF PROTO=TCP SPT=88 DPT=51179 WINDOW=2053 RES=0x00 ACK URGP=0",
+        "network.transport": "tcp",
+        "observer.name": "My-Office-Gateway",
+        "service.type": "iptables",
+        "source.port": 88,
+        "tags": [
+            "iptables",
+            "forwarded"
+        ]
+    },
+    {
+        "destination.port": 443,
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "iptables.log",
+        "event.kind": "event",
+        "event.module": "iptables",
+        "event.timezone": "-02:00",
+        "fileset.name": "log",
+        "input.type": "log",
+        "iptables.id": 15461,
+        "iptables.tcp.flags": [
+            "ACK",
+            "PSH"
+        ],
+        "iptables.tcp.reserved_bits": 0,
+        "iptables.tcp.window": 8208,
+        "iptables.ttl": 126,
+        "log.offset": 1679,
+        "log.original": "May  5 20:47:09 My-Office-Gateway user.info kernel: 0 TTL=126 ID=15461 DF PROTO=TCP SPT=59289 DPT=443 WINDOW=8208 RES=0x00 ACK PSH URGP=0",
+        "network.transport": "tcp",
+        "observer.name": "My-Office-Gateway",
+        "service.type": "iptables",
+        "source.port": 59289,
+        "tags": [
+            "iptables",
+            "forwarded"
+        ]
+    },
+    {
+        "destination.port": 51182,
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "iptables.log",
+        "event.kind": "event",
+        "event.module": "iptables",
+        "event.timezone": "-02:00",
+        "fileset.name": "log",
+        "input.type": "log",
+        "iptables.id": 8702,
+        "iptables.tcp.flags": [
+            "ACK"
+        ],
+        "iptables.tcp.reserved_bits": 0,
+        "iptables.tcp.window": 2053,
+        "iptables.ttl": 126,
+        "log.offset": 1817,
+        "log.original": "May  5 20:46:56 My-Office-Gateway user.info kernel: L=126 ID=8702 DF PROTO=TCP SPT=88 DPT=51182 WINDOW=2053 RES=0x00 ACK URGP=0",
+        "network.transport": "tcp",
+        "observer.name": "My-Office-Gateway",
+        "service.type": "iptables",
+        "source.port": 88,
+        "tags": [
+            "iptables",
+            "forwarded"
+        ]
+    },
+    {
+        "destination.port": 49209,
+        "event.category": [
+            "network"
+        ],
+        "event.dataset": "iptables.log",
+        "event.kind": "event",
+        "event.module": "iptables",
+        "event.timezone": "-02:00",
+        "fileset.name": "log",
+        "input.type": "log",
+        "iptables.id": 4622,
+        "iptables.tcp.flags": [
+            "ECE",
+            "ACK",
+            "SYN"
+        ],
+        "iptables.tcp.reserved_bits": 0,
+        "iptables.tcp.window": 8192,
+        "iptables.ttl": 126,
+        "log.offset": 1945,
+        "log.original": "May  5 20:45:44 My-Office-Gateway user.info kernel: TL=126 ID=4622 DF PROTO=TCP SPT=389 DPT=49209 WINDOW=8192 RES=0x00 ECE ACK SYN URGP=0",
+        "network.transport": "tcp",
+        "observer.name": "My-Office-Gateway",
+        "service.type": "iptables",
+        "source.port": 389,
+        "tags": [
+            "iptables",
+            "forwarded"
+        ]
     }
 ]