[Filebeat] Adding fixes to the TI module (elastic#24133)

* cleaning up TI module, adding safer config options, updating docs and fixing the MISP tag copy painless script * updating otx pipeline to remove specific null value * fixing grok pattern in MISP to fetch hash values
P1llus · Feb 19, 2021 · f394755 · f394755
1 parent 3319f5b
commit f394755
Show file tree

Hide file tree

Showing 19 changed files with 500 additions and 223 deletions.
diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc
@@ -12,8 +12,8 @@ This file is generated! See scripts/docs_collector.py
 == Threat Intel module
 beta[]
 
-This module is a collection of different threat intelligence sources. The ingested data is meant to be used with [Indicator Match rules]https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule, but is also
-compatible with other features like [Enrich Processors]https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html.
+This module is a collection of different threat intelligence sources. The ingested data is meant to be used with https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule[Indicator Match rules], but is also
+compatible with other features like https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html[Enrich Processors].
 The related threat intel attribute that is meant to be used for matching incoming source data is stored under the `threatintel.indicator.*` fields.
 
 Currently supporting:

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -1996,7 +1996,7 @@ filebeat.modules:
     var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/
 
     # The interval to poll the API for updates.
-    var.interval: 60m
+    var.interval: 10m
 
   abusemalware:
     enabled: true
@@ -2008,7 +2008,7 @@ filebeat.modules:
     var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/
 
     # The interval to poll the API for updates.
-    var.interval: 60m
+    var.interval: 10m
 
   misp:
     enabled: true
@@ -2022,6 +2022,10 @@ filebeat.modules:
     # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI.
     var.api_token: API_KEY
 
+    # Configures the type of SSL verification done, if MISP is running on self signed certificates
+    # then the certificate would either need to be trusted, or verification_mode set to none.
+    #var.ssl.verification_mode: none
+
     # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context.
     # For examples please reference the filebeat module documentation.
     #var.filters:
@@ -2030,10 +2034,10 @@ filebeat.modules:
 
     # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer
     # than the last event that was already ingested.
-    var.first_interval: 24h
+    var.first_interval: 300h
 
     # The interval to poll the API for updates.
-    var.interval: 60m
+    var.interval: 5m
 
   otx:
     enabled: true
@@ -2050,22 +2054,26 @@ filebeat.modules:
     # Optional filters that can be applied to retrieve only specific indicators.
     #var.types: "domain,IPv4,hostname,url,FileHash-SHA256"
 
+    # The timeout of the HTTP client connecting to the OTX API
+    #var.http_client_timeout: 120s
+
     # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module.
-    var.lookback_range: 2h
+    var.lookback_range: 1h
 
     # How far back to look once the beat starts up for the first time, the value has to be in hours.
-    var.first_interval: 24h
+    var.first_interval: 400h
 
     # The interval to poll the API for updates
-    var.interval: 60m
+    var.interval: 5m
 
   anomali:
     enabled: true
 
     # Input used for ingesting threat intel data
     var.input: httpjson
 
-    # The URL used for Threat Intel API calls.
+    # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending
+    # on the type of threat intel source that is needed.
     var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects
 
     # The Username used by anomali Limo, defaults to guest.
@@ -2075,10 +2083,10 @@ filebeat.modules:
     #var.password: guest
 
     # How far back to look once the beat starts up for the first time, the value has to be in hours.
-    var.first_interval: 24h
+    var.first_interval: 400h
 
     # The interval to poll the API for updates
-    var.interval: 60m
+    var.interval: 5m
 
 #---------------------------- Apache Tomcat Module ----------------------------
 - module: tomcat

diff --git a/x-pack/filebeat/module/threatintel/_meta/config.yml b/x-pack/filebeat/module/threatintel/_meta/config.yml
@@ -9,7 +9,7 @@
     var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/
 
     # The interval to poll the API for updates.
-    var.interval: 60m
+    var.interval: 10m
 
   abusemalware:
     enabled: true
@@ -21,7 +21,7 @@
     var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/
 
     # The interval to poll the API for updates.
-    var.interval: 60m
+    var.interval: 10m
 
   misp:
     enabled: true
@@ -35,6 +35,10 @@
     # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI.
     var.api_token: API_KEY
 
+    # Configures the type of SSL verification done, if MISP is running on self signed certificates
+    # then the certificate would either need to be trusted, or verification_mode set to none.
+    #var.ssl.verification_mode: none
+
     # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context.
     # For examples please reference the filebeat module documentation.
     #var.filters:
@@ -43,10 +47,10 @@
 
     # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer
     # than the last event that was already ingested.
-    var.first_interval: 24h
+    var.first_interval: 300h
 
     # The interval to poll the API for updates.
-    var.interval: 60m
+    var.interval: 5m
 
   otx:
     enabled: true
@@ -63,22 +67,26 @@
     # Optional filters that can be applied to retrieve only specific indicators.
     #var.types: "domain,IPv4,hostname,url,FileHash-SHA256"
 
+    # The timeout of the HTTP client connecting to the OTX API
+    #var.http_client_timeout: 120s
+
     # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module.
-    var.lookback_range: 2h
+    var.lookback_range: 1h
 
     # How far back to look once the beat starts up for the first time, the value has to be in hours.
-    var.first_interval: 24h
+    var.first_interval: 400h
 
     # The interval to poll the API for updates
-    var.interval: 60m
+    var.interval: 5m
 
   anomali:
     enabled: true
 
     # Input used for ingesting threat intel data
     var.input: httpjson
 
-    # The URL used for Threat Intel API calls.
+    # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending
+    # on the type of threat intel source that is needed.
     var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects
 
     # The Username used by anomali Limo, defaults to guest.
@@ -88,7 +96,7 @@
     #var.password: guest
 
     # How far back to look once the beat starts up for the first time, the value has to be in hours.
-    var.first_interval: 24h
+    var.first_interval: 400h
 
     # The interval to poll the API for updates
-    var.interval: 60m
+    var.interval: 5m
diff --git a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc
@@ -7,8 +7,8 @@
 == Threat Intel module
 beta[]
 
-This module is a collection of different threat intelligence sources. The ingested data is meant to be used with [Indicator Match rules]https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule, but is also
-compatible with other features like [Enrich Processors]https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html.
+This module is a collection of different threat intelligence sources. The ingested data is meant to be used with https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule[Indicator Match rules], but is also
+compatible with other features like https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html[Enrich Processors].
 The related threat intel attribute that is meant to be used for matching incoming source data is stored under the `threatintel.indicator.*` fields.
 
 Currently supporting:

diff --git a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml
@@ -6,7 +6,7 @@ interval: {{ .interval }}
 
 request.method: GET
 {{ if .ssl }}
-  - request.ssl: {{ .ssl | tojson }}
+request.ssl: {{ .ssl | tojson }}
 {{ end }}
 request.url: {{ .url }}
 request.transforms:
@@ -33,9 +33,11 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 processors:
   - decode_json_fields:
-      document_id: "md5_hash"
       fields: [message]
       target: json
+  - fingerprint:
+      fields: ["json.md5_hash"]
+      target_field: "@metadata._id"
   - add_fields:
       target: ''
       fields: