[Filebeat] Add HIP Match logs to Palo Alto Module (#25686)

Update panw.panos module to parse HIP Match logs.

Also this updates the Global Protect parsing with additional fields per the updated docs, https://docs.paloaltonetworks.com/pan-os/10-0/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/globalprotect-log-fields.html

CHANGELOG.next.asciidoc

@@ -804,6 +804,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Content-Type override to aws-s3 input. {issue}25697[25697] {pull}25772[25772]
 - In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776]
 - Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
+- Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -113994,6 +113994,86 @@ type: keyword
 The type of tunnel (either SSLVPN or IPSec).
 
 
+type: keyword
+
+--
+
+*`panw.panos.connect_method`*::
++
+--
+A string showing the how the GlobalProtect app connects to Gateway
+
+
+type: keyword
+
+--
+
+*`panw.panos.matchname`*::
++
+--
+Name of the HIP object or profile.
+
+
+type: keyword
+
+--
+
+*`panw.panos.matchtype`*::
++
+--
+Whether the hip field represents a HIP object or a HIP profile.
+
+
+type: keyword
+
+--
+
+*`panw.panos.priority`*::
++
+--
+The priority order of the gateway that is based on highest (1), high (2), medium (3), low (4), or lowest (5) to which the GlobalProtect app can connect.
+
+
+type: keyword
+
+--
+
+*`panw.panos.response_time`*::
++
+--
+The SSL response time of the selected gateway that is measured in milliseconds on the endpoint during tunnel setup.
+
+
+type: keyword
+
+--
+
+*`panw.panos.attempted_gateways`*::
++
+--
+The fields that are collected for each gateway connection attempt with the gateway name, SSL response time, and priority
+
+
+type: keyword
+
+--
+
+*`panw.panos.gateway`*::
++
+--
+The name of the gateway that is specified on the portal configuration.
+
+
+type: keyword
+
+--
+
+*`panw.panos.selection_type`*::
++
+--
+The connection method that is selected to connect to the gateway.
+
+
 type: keyword
 
 --

x-pack/filebeat/module/panw/panos/_meta/fields.yml

@@ -145,149 +145,225 @@
 
     - name: virtual_sys
       type: keyword
+      default_field: false
       description: >
         Virtual system instance
 
     - name: client_os_ver
       type: keyword
+      default_field: false
       description: >
         The client device’s OS version.
 
     - name: client_os
       type: keyword
+      default_field: false
       description: >
         The client device’s OS version.
 
     - name: client_ver
       type: keyword
+      default_field: false
       description: >
         The client’s GlobalProtect app version.
 
     - name: stage
       type: keyword
+      default_field: false
       example: before-login
       description: >
         A string showing the stage of the connection
 
     - name: actionflags
       type: keyword
+      default_field: false
       description: >
         A bit field indicating if the log was forwarded to Panorama.
 
     - name: error
       type: keyword
+      default_field: false
       description: >
         A string showing that error that has occurred in any event.
 
     - name: error_code
       type: integer
+      default_field: false
       description: >
         An integer associated with any errors that occurred.
 
     - name: repeatcnt
       type: integer
+      default_field: false
       description: >
         The number of sessions with the same source IP address, destination IP address, application, and subtype that GlobalProtect has detected within the last five seconds.An integer associated with any errors that occurred.
 
     - name: serial_number
       type: keyword
+      default_field: false
       description: >
         The serial number of the user’s machine or device.
 
     - name: auth_method
       type: keyword
+      default_field: false
       example: LDAP
       description: >
         A string showing the authentication type
 
     - name: datasource
       type: keyword
+      default_field: false
       description: >
         Source from which mapping information is collected.
 
     - name: datasourcetype
       type: keyword
+      default_field: false
       description: >
         Mechanism used to identify the IP/User mappings within a data source.
 
     - name: datasourcename
       type: keyword
+      default_field: false
       description: >
         User-ID source that sends the IP (Port)-User Mapping.
 
     - name: factorno
       type: integer
+      default_field: false
       description: >
         Indicates the use of primary authentication (1) or additional factors (2, 3).
 
     - name: factortype
       type: keyword
+      default_field: false
       description: >
         Vendor used to authenticate a user when Multi Factor authentication is present.
 
     - name: factorcompletiontime
       type: date
+      default_field: false
       description: >
         Time the authentication was completed.
 
     - name: ugflags
       type: keyword
+      default_field: false
       description: |
         Displays whether the user group that was found during user group mapping. Supported values are:
         User Group Found—Indicates whether the user could be mapped to a group.
         Duplicate User—Indicates whether duplicate users were found in a user group. Displays N/A if no user group is found.
 
     - name: device_group_hierarchy
       type: group
+      default_field: false
       description: >
         A sequence of identification numbers that indicate the device group’s location within a device group hierarchy. The firewall (or virtual system) generating the log includes the identification number of each ancestor in its device group hierarchy. The shared device group (level 0) is not included in this structure.
         If the log values are 12, 34, 45, 0, it means that the log was generated by a firewall (or virtual system) that belongs to device group 45, and its ancestors are 34, and 12.
       fields:
         - name: level_1
           type: keyword
+          default_field: false
           description: >
             A sequence of identification numbers that indicate the device group’s location within a device group hierarchy. The firewall (or virtual system) generating the log includes the identification number of each ancestor in its device group hierarchy. The shared device group (level 0) is not included in this structure.
             If the log values are 12, 34, 45, 0, it means that the log was generated by a firewall (or virtual system) that belongs to device group 45, and its ancestors are 34, and 12.
 
         - name: level_2
           type: keyword
+          default_field: false
           description: >
             A sequence of identification numbers that indicate the device group’s location within a device group hierarchy. The firewall (or virtual system) generating the log includes the identification number of each ancestor in its device group hierarchy. The shared device group (level 0) is not included in this structure.
             If the log values are 12, 34, 45, 0, it means that the log was generated by a firewall (or virtual system) that belongs to device group 45, and its ancestors are 34, and 12.
 
         - name: level_3
           type: keyword
+          default_field: false
           description: >
             A sequence of identification numbers that indicate the device group’s location within a device group hierarchy. The firewall (or virtual system) generating the log includes the identification number of each ancestor in its device group hierarchy. The shared device group (level 0) is not included in this structure.
             If the log values are 12, 34, 45, 0, it means that the log was generated by a firewall (or virtual system) that belongs to device group 45, and its ancestors are 34, and 12.
 
         - name: level_4
           type: keyword
+          default_field: false
           description: >
             A sequence of identification numbers that indicate the device group’s location within a device group hierarchy. The firewall (or virtual system) generating the log includes the identification number of each ancestor in its device group hierarchy. The shared device group (level 0) is not included in this structure.
             If the log values are 12, 34, 45, 0, it means that the log was generated by a firewall (or virtual system) that belongs to device group 45, and its ancestors are 34, and 12.
 
     - name: timeout
       type: integer
+      default_field: false
       description: >
         Timeout after which the IP/User Mappings are cleared.
 
     - name: vsys_id
       type: keyword
+      default_field: false
       description: >
         A unique identifier for a virtual system on a Palo Alto Networks firewall.
 
     - name: vsys_name
       type: keyword
+      default_field: false
       description: >
         The name of the virtual system associated with the session; only valid on firewalls enabled for multiple virtual systems.
 
     - name: description
       type: keyword
+      default_field: false
       description: >
         Additional information for any event that has occurred.
 
     - name: tunnel_type
       type: keyword
+      default_field: false
       description: >
         The type of tunnel (either SSLVPN or IPSec).
+
+    - name: connect_method
+      type: keyword
+      default_field: false
+      description: >
+        A string showing the how the GlobalProtect app connects to Gateway
+
+    - name: matchname
+      type: keyword
+      default_field: false
+      description: >
+        Name of the HIP object or profile.
+
+    - name: matchtype
+      type: keyword
+      default_field: false
+      description: >
+        Whether the hip field represents a HIP object or a HIP profile.
+
+    - name: priority
+      type: keyword
+      default_field: false
+      description: >
+        The priority order of the gateway that is based on highest (1), high (2), medium (3), low (4), or lowest (5) to which the GlobalProtect app can connect.
+
+    - name: response_time
+      type: keyword
+      default_field: false
+      description: >
+        The SSL response time of the selected gateway that is measured in milliseconds on the endpoint during tunnel setup.
+
+    - name: attempted_gateways
+      type: keyword
+      default_field: false
+      description: >
+        The fields that are collected for each gateway connection attempt with the gateway name, SSL response time, and priority
+
+    - name: gateway
+      type: keyword
+      default_field: false
+      description: >
+        The name of the gateway that is specified on the portal configuration.
+
+    - name: selection_type
+      type: keyword
+      default_field: false
+      description: >
+        The connection method that is selected to connect to the gateway.

x-pack/filebeat/module/panw/panos/config/input.yml

@@ -37,12 +37,12 @@ processors:
       overwrite_keys: true
       omit_empty: true
       mappings:
+        _temp_.ietf_header: 0
         event.created: 1
         observer.serial_number: 2
         panw.panos.type: 3
         panw.panos.sub_type: 4
         _temp_.generated_time: 6
-        _temp_.ietf_header: 0
 
   - extract_array:
       when:
@@ -215,6 +215,11 @@ processors:
         observer.hostname: 33
         panw.panos.sequence_number: 34
         panw.panos.actionflags: 35
+        panw.panos.selection_type: 37
+        panw.panos.response_time: 38
+        panw.panos.priority: 39
+        panw.panos.attempted_gateways: 40
+        panw.panos.gateway: 41
 
   - extract_array:
       when:
@@ -257,6 +262,43 @@ processors:
         source.user.name: 31
         client.user.name: 31
 
+  - extract_array:
+      when:
+        or:
+          - equals:
+              panw.panos.type: HIPMATCH
+          - equals:
+              panw.panos.type: HIP-MATCH
+      field: csv
+      omit_empty: true
+      overwrite_keys: true
+      fail_on_error: false
+      mappings:
+        _temp_.srcuser: 7
+        panw.panos.virtual_sys: 8
+        host.name: 9
+        host.os.full: 10
+        client.ip: 11
+        source.ip: 11
+        source.address: 11
+        panw.panos.matchname: 12
+        panw.panos.repeatcnt: 13
+        panw.panos.matchtype: 14
+        panw.panos.sequence_number: 17
+        panw.panos.actionflags: 18
+        panw.panos.device_group_hierarchy.level_1: 19
+        panw.panos.device_group_hierarchy.level_2: 20
+        panw.panos.device_group_hierarchy.level_3: 21
+        panw.panos.device_group_hierarchy.level_4: 22
+        panw.panos.vsys_name: 23
+        observer.hostname: 24
+        panw.panos.vsys_id: 25
+        _temp_.source_ipv6: 26
+        host.id: 27
+        panw.panos.serial_number: 28
+        host.mac: 29
+
+
   - drop_fields:
       fields:
         - csv