[Filebeat] Adding fixes to the TI module (#24133)

* cleaning up TI module, adding safer config options, updating docs and fixing the MISP tag copy painless script * updating otx pipeline to remove specific null value * fixing grok pattern in MISP to fetch hash values
elastic · Feb 19, 2021 · f394755 · f394755
1 parent 3319f5b
commit f394755
Show file tree

Hide file tree

Showing 19 changed files with 500 additions and 223 deletions.
diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc
@@ -12,8 +12,8 @@ This file is generated! See scripts/docs_collector.py
 == Threat Intel module
 beta[]
 
-This module is a collection of different threat intelligence sources. The ingested data is meant to be used with [Indicator Match rules]https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule, but is also
-compatible with other features like [Enrich Processors]https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html.
+This module is a collection of different threat intelligence sources. The ingested data is meant to be used with https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule[Indicator Match rules], but is also
+compatible with other features like https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html[Enrich Processors].
 The related threat intel attribute that is meant to be used for matching incoming source data is stored under the `threatintel.indicator.*` fields.
 
 Currently supporting:

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -1996,7 +1996,7 @@ filebeat.modules:
  var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/
 
  # The interval to poll the API for updates.
- var.interval: 60m
+ var.interval: 10m
 
  abusemalware:
  enabled: true
@@ -2008,7 +2008,7 @@ filebeat.modules:
  var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/
 
  # The interval to poll the API for updates.
- var.interval: 60m
+ var.interval: 10m
 
  misp:
  enabled: true
@@ -2022,6 +2022,10 @@ filebeat.modules:
  # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI.
  var.api_token: API_KEY
 
+ # Configures the type of SSL verification done, if MISP is running on self signed certificates
+ # then the certificate would either need to be trusted, or verification_mode set to none.
+ #var.ssl.verification_mode: none
+
  # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context.
  # For examples please reference the filebeat module documentation.
  #var.filters:
@@ -2030,10 +2034,10 @@ filebeat.modules:
 
  # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer
  # than the last event that was already ingested.
- var.first_interval: 24h
+ var.first_interval: 300h
 
  # The interval to poll the API for updates.
- var.interval: 60m
+ var.interval: 5m
 
  otx:
  enabled: true
@@ -2050,22 +2054,26 @@ filebeat.modules:
  # Optional filters that can be applied to retrieve only specific indicators.
  #var.types: "domain,IPv4,hostname,url,FileHash-SHA256"
 
+ # The timeout of the HTTP client connecting to the OTX API
+ #var.http_client_timeout: 120s
+
  # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module.
- var.lookback_range: 2h
+ var.lookback_range: 1h
 
  # How far back to look once the beat starts up for the first time, the value has to be in hours.
- var.first_interval: 24h
+ var.first_interval: 400h
 
  # The interval to poll the API for updates
- var.interval: 60m
+ var.interval: 5m
 
  anomali:
  enabled: true
 
  # Input used for ingesting threat intel data
  var.input: httpjson
 
- # The URL used for Threat Intel API calls.
+ # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending
+ # on the type of threat intel source that is needed.
  var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects
 
  # The Username used by anomali Limo, defaults to guest.
@@ -2075,10 +2083,10 @@ filebeat.modules:
  #var.password: guest
 
  # How far back to look once the beat starts up for the first time, the value has to be in hours.
- var.first_interval: 24h
+ var.first_interval: 400h
 
  # The interval to poll the API for updates
- var.interval: 60m
+ var.interval: 5m
 
 #---------------------------- Apache Tomcat Module ----------------------------
 - module: tomcat

diff --git a/x-pack/filebeat/module/threatintel/_meta/config.yml b/x-pack/filebeat/module/threatintel/_meta/config.yml
@@ -9,7 +9,7 @@
  var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/
 
  # The interval to poll the API for updates.
- var.interval: 60m
+ var.interval: 10m
 
  abusemalware:
  enabled: true
@@ -21,7 +21,7 @@
  var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/
 
  # The interval to poll the API for updates.
- var.interval: 60m
+ var.interval: 10m
 
  misp:
  enabled: true
@@ -35,6 +35,10 @@
  # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI.
  var.api_token: API_KEY
 
+ # Configures the type of SSL verification done, if MISP is running on self signed certificates
+ # then the certificate would either need to be trusted, or verification_mode set to none.
+ #var.ssl.verification_mode: none
+
  # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context.
  # For examples please reference the filebeat module documentation.
  #var.filters:
@@ -43,10 +47,10 @@
 
  # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer
  # than the last event that was already ingested.
- var.first_interval: 24h
+ var.first_interval: 300h
 
  # The interval to poll the API for updates.
- var.interval: 60m
+ var.interval: 5m
 
  otx:
  enabled: true
@@ -63,22 +67,26 @@
  # Optional filters that can be applied to retrieve only specific indicators.
  #var.types: "domain,IPv4,hostname,url,FileHash-SHA256"
 
+ # The timeout of the HTTP client connecting to the OTX API
+ #var.http_client_timeout: 120s
+
  # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module.
- var.lookback_range: 2h
+ var.lookback_range: 1h
 
  # How far back to look once the beat starts up for the first time, the value has to be in hours.
- var.first_interval: 24h
+ var.first_interval: 400h
 
  # The interval to poll the API for updates
- var.interval: 60m
+ var.interval: 5m
 
  anomali:
  enabled: true
 
  # Input used for ingesting threat intel data
  var.input: httpjson
 
- # The URL used for Threat Intel API calls.
+ # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending
+ # on the type of threat intel source that is needed.
  var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects
 
  # The Username used by anomali Limo, defaults to guest.
@@ -88,7 +96,7 @@
  #var.password: guest
 
  # How far back to look once the beat starts up for the first time, the value has to be in hours.
- var.first_interval: 24h
+ var.first_interval: 400h
 
  # The interval to poll the API for updates
- var.interval: 60m
+ var.interval: 5m
diff --git a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc
@@ -7,8 +7,8 @@
 == Threat Intel module
 beta[]
 
-This module is a collection of different threat intelligence sources. The ingested data is meant to be used with [Indicator Match rules]https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule, but is also
-compatible with other features like [Enrich Processors]https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html.
+This module is a collection of different threat intelligence sources. The ingested data is meant to be used with https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule[Indicator Match rules], but is also
+compatible with other features like https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html[Enrich Processors].
 The related threat intel attribute that is meant to be used for matching incoming source data is stored under the `threatintel.indicator.*` fields.
 
 Currently supporting:

diff --git a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml
@@ -6,7 +6,7 @@ interval: {{ .interval }}
 
 request.method: GET
 {{ if .ssl }}
- - request.ssl: {{ .ssl | tojson }}
+request.ssl: {{ .ssl | tojson }}
 {{ end }}
 request.url: {{ .url }}
 request.transforms:
@@ -33,9 +33,11 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 processors:
  - decode_json_fields:
- document_id: "md5_hash"
  fields: [message]
  target: json
+ - fingerprint:
+ fields: ["json.md5_hash"]
+ target_field: "@metadata._id"
  - add_fields:
  target: ''
  fields: