From 369f0c2bee99f7c47cd9aad27628e67ee13d63a3 Mon Sep 17 00:00:00 2001 From: content-bot <55035720+content-bot@users.noreply.github.com> Date: Sun, 25 Jun 2023 15:21:38 +0300 Subject: [PATCH] Added Opensearch support for Elasticsearch feed (#27623) * Added Opensearch support for Elasticsearch feed (#27333) * Added Opensearch support for Elasticsearch feed * Updated release notes and readme * Updated release notes and readme * Revert changes to poetry lock * Update docker image. Remove tested on version x.x in README. * Fix flake8 errors. * Update docker comment in RN. * Trying to add opensearchpy dependency for test * fix flake8 errors. * Adding known_words section * Moving opensearch-py to dev-packages section. * Update docker image * Align yml docker with RN docker. --------- Co-authored-by: Danny_Fried * Updating docker image. --------- Co-authored-by: anilagr <40182783+anilagr@users.noreply.github.com> Co-authored-by: Danny_Fried --- Packs/FeedElasticsearch/.pack-ignore | 3 + .../FeedElasticsearch/FeedElasticsearch.py | 13 +- .../FeedElasticsearch/FeedElasticsearch.yml | 11 +- .../FeedElasticsearch_test.py | 7 +- .../Integrations/FeedElasticsearch/Pipfile | 1 + .../Integrations/FeedElasticsearch/README.md | 308 +++--------------- Packs/FeedElasticsearch/ReleaseNotes/1_1_0.md | 6 + Packs/FeedElasticsearch/pack_metadata.json | 2 +- 8 files changed, 74 insertions(+), 277 deletions(-) create mode 100644 Packs/FeedElasticsearch/ReleaseNotes/1_1_0.md diff --git a/Packs/FeedElasticsearch/.pack-ignore b/Packs/FeedElasticsearch/.pack-ignore index e69de29bb2d1..b5afabe852ab 100644 --- a/Packs/FeedElasticsearch/.pack-ignore +++ b/Packs/FeedElasticsearch/.pack-ignore @@ -0,0 +1,3 @@ +[known_words] +Elasticsearch +Opensearch \ No newline at end of file diff --git a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.py b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.py index 428a68dfee86..bfdc65574a04 100644 --- a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.py +++ b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.py @@ -3,9 +3,6 @@ from CommonServerUserPython import * '''IMPORTS''' -from elasticsearch import Elasticsearch, RequestsHttpConnection -from elasticsearch_dsl import Search -from elasticsearch_dsl.query import QueryString import requests import warnings import urllib3 @@ -33,6 +30,16 @@ FEED_TYPE_CORTEX = 'Cortex XSOAR Feed' FEED_TYPE_CORTEX_MT = 'Cortex XSOAR MT Shared Feed' +ELASTIC_SEARCH_CLIENT = demisto.params().get('client_type') +if ELASTIC_SEARCH_CLIENT == 'OpenSearch': + from opensearchpy import OpenSearch as Elasticsearch, RequestsHttpConnection + from opensearch_dsl import Search + from opensearch_dsl.query import QueryString +else: + from elasticsearch import Elasticsearch, RequestsHttpConnection + from elasticsearch_dsl import Search + from elasticsearch_dsl.query import QueryString + class ElasticsearchClient: def __init__(self, insecure=None, server=None, username=None, password=None, api_key=None, api_id=None, diff --git a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml index a29b1b44dbf3..51f61d49c92b 100644 --- a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml +++ b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml @@ -12,6 +12,15 @@ configuration: name: credentials required: false type: 9 +- additionalinfo: In some hosted ElasticSearch environments, the standard ElasticSearch client is not supported. If you encounter any related client issues, please consider using the OpenSearch client type. + defaultvalue: OpenSearch + display: Client type + name: client_type + options: + - ElasticSearch + - OpenSearch + required: false + type: 15 - defaultvalue: 'false' display: Trust any certificate (not secure) name: insecure @@ -163,7 +172,7 @@ script: description: Gets indicators available in the configured Elasticsearch database. execution: false name: es-get-indicators - dockerimage: demisto/py3-tools:1.0.0.45904 + dockerimage: demisto/py3-tools:1.0.0.64131 feed: true isfetch: false longRunning: false diff --git a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch_test.py b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch_test.py index c1ccfd085be1..a337bb55d3c6 100644 --- a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch_test.py +++ b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch_test.py @@ -1,3 +1,6 @@ +from CommonServerPython import * + + class MockHit: def __init__(self, hit_val): self._hit_val = hit_val @@ -171,7 +174,9 @@ def test_hit_to_indicator(): assert ioc[CUSTOM_TYPE_KEY] == '' -def test_extract_indicators_from_insight_hit(mocker): +def test_extract_indicators_from_insight_hit2(mocker): + params: dict = {'client_type': 'OpenSearch'} + mocker.patch.object(demisto, 'params', return_value=params) import FeedElasticsearch as esf mocker.patch.object(esf, 'hit_to_indicator', return_value=dict(PARSED_INSIGHT_HIT)) ioc_lst, ioc_enrch_lst = esf.extract_indicators_from_insight_hit(PARSED_INSIGHT_HIT, ['tag1', 'tag2'], 'AMBER') diff --git a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/Pipfile b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/Pipfile index 5ae3da54e6d4..e4447edf393e 100644 --- a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/Pipfile +++ b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/Pipfile @@ -9,6 +9,7 @@ pytest = "*" pytest-mock = "*" requests-mock = "*" pytest-asyncio = "*" +opensearch-py = "*" [packages] elasticsearch = "*" diff --git a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/README.md b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/README.md index f63f1a40afa4..9e19202e455e 100644 --- a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/README.md +++ b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/README.md @@ -6,295 +6,61 @@ Fetch indicators stored in an Elasticsearch database. 2. The Cortex XSOAR MT Shared Feed contains indicators shared by a tenant account in a multi-tenant environment. 3. The Generic Feed contains a feed in a format specified by the user. -## Configure ElasticsearchFeed on Cortex XSOAR ---- - -1. Navigate to __Settings__ > __Integrations__ > __Servers & Services__. -2. Search for SharedTenantElasticsearchFeed. -3. Click __Add instance__ to create and configure a new integration instance. - * __Server URL__: Elasticsearch database URL. - * __Name__: Used for authentication via Username + Password or API ID + API Key (If you wish to use API Key authorization enter **_api_key_id:** followed by your API key ID). - * __Password__: Used for authentication via Username + Password or API ID + API Key (If you wish to use API Key authorization enter your API key). - * __Trust any certificate (not secure)__: Ignore HTTPS certificates. - * __Use system proxy settings__: Enable/Disable - * __Feed Type__: Choose the feed type saved into the Elasticsearch database. Cortex XSOAR Feed are indicators saved by Cortex XSOAR in an Elasticsearch - configured enviornment. Cortex XSOAR MT Shared Feed are indicators shared by a - tenant in a MT env. Generic Feed is a feed in a format dictated by the user - * __Fetch indicators__: Enable/Disable - * __First Fetch Time__: Determine how far to look back for fetched indicators (<number> <time unit>, e.g., 12 hours, 7 days). - * __Indicator Reputation__: Indicators from this integration instance will be marked with this reputation. - * __Source Reliability__: Reliability of the source providing the intelligence data. - * __Traffic Light Protocol Color__: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed. More information about the protocol can be found at https://us-cert.cisa.gov/tlp. - * __Indicator Value Field__: Source field that contains the indicator value in the index. - * __Indicator Type Field__: Source field that contains the indicator type in the index. - * __Indicator Type__: Default indicator type used in case no "Indicator Type Field" was provided - * __Index From Which To Fetch Indicators__: Multiple indices may be used by separating them with a comma. If none is provided, will search in all indices - * __Time Field Type__: Time field type used in the database. - * __Index Time Field__: Used for sorting sort and limiting data. If left empty, no sorting will be done. - * __Query__: Elasticsearch query to be executed when fetching indicators from Elasticsearch. -4. Click __Test__ to validate the URLs, token, and connection. -## Fetched Incidents Data ---- +## Configure Elasticsearch Feed on Cortex XSOAR + +1. Navigate to **Settings** > **Integrations** > **Servers & Services**. +2. Search for Elasticsearch Feed. +3. Click **Add instance** to create and configure a new integration instance. + + | **Parameter** | **Description** | **Required** | + | --- | --- | --- | + | Server URL | | True | + | Name (see ?->Authentication) | Provide Username \+ Passoword instead of API key \+ API ID | False | + | Password | | False | + | Client type | In some hosted ElasticSearch environments, the standard ElasticSearch client is not supported. If you encounter any related client issues, please consider using the OpenSearch client type. | False | + | Trust any certificate (not secure) | | False | + | Use system proxy settings | | False | + | Feed Type | The Cortex XSOAR Feed contains system indicators saved in an Elasticsearch index. The Cortex XSOAR MT Shared Feed contains indicators shared by a tenant account in a multi-tenant environment. Generic Feed contains a feed in a format specified by the user | False | + | Fetch indicators | | False | + | First Fetch Time | Determine how far to look back for fetched indicators \(<number> <time unit>, e.g., 12 hours, 7 days\). | False | + | Indicator Reputation | Indicators from this integration instance will be marked with this reputation | False | + | Source Reliability | Reliability of the source providing the intelligence data | True | + | Traffic Light Protocol Color | The Traffic Light Protocol \(TLP\) designation to apply to indicators fetched from the feed | False | + | | | False | + | | | False | + | Feed Fetch Interval | | False | + | Tags | Supports CSV values. | False | + | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | + | Indicator Value Field | Source field that contains the indicator value in the index | False | + | Indicator Type Field | Source field that contains the indicator type in the index | False | + | Indicator Type | Default indicator type used in case no "Indicator Type Field" was provided | False | + | Index from Which To Fetch Indicators | A comma-separated list of indexes. If empty, searches all indexes. | False | + | Time Field Type | | False | + | Index Time Field | Used for sorting and limiting data. If empty, results are not sorted. | False | + | Query | Elasticsearch query to execute when fetching indicators from Elasticsearch | False | + +4. Click **Test** to validate the URLs, token, and connection. ## Commands ---- + You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook. After you successfully execute a command, a DBot message appears in the War Room with the command details. -1. get-shared-indicators -### 1. get-shared-indicators ---- -Gets indicators shared with this tenant (MT only). -##### Base Command - -`get-shared-indicators` -##### Input - -| **Argument Name** | **Description** | **Required** | -| --- | --- | --- | -| limit | The maximum number of indicators to fetch. | Required | - - -##### Context Output - -| **Path** | **Type** | **Description** | -| --- | --- | --- | -| ElasticsearchFeed.SharedIndicators.Indicators | Unknown | Indicators shared from other tenants without enrichments. | -| ElasticsearchFeed.SharedIndicators.Enrichments | Unknown | Enrichment indicators shared from other tenants. | - - -##### Command Example -```!get-shared-indicators``` - -##### Context Example -``` -{ - "ElasticsearchFeed.SharedIndicators": { - "Indicators": [ - { - "comment": "", - "version": 2, - "sequenceNumber": 26, - "sortValues": null, - "modified": "2020-02-18T11:43:44.200258Z", - "lastSeen": "2020-02-18T10:39:05.230163+02:00", - "id": "e086aa137fa19f67d27b39d0eca18610", - "deletedFeedFetchTime": "0001-01-01T00:00:00Z", - "rawJSON": { - "comment": "", - "version": 2, - "sequenceNumber": 26, - "sortValues": null, - "modified": "2020-02-18T11:43:44.200258Z", - "lastSeen": "2020-02-18T10:39:05.230163+02:00", - "id": "e086aa137fa19f67d27b39d0eca18610", - "deletedFeedFetchTime": "0001-01-01T00:00:00Z", - "investigationsCount": 0, - "primaryTerm": 2, - "score": 0, - "investigationIDs": [], - "type": "IP", - "isShared": true, - "rawName": "1.1.1.1", - "modifiedTime": "0001-01-01T00:00:00Z", - "lastSeenEntryID": "API", - "CustomFields": { - "internal": false - }, - "firstSeen": "2020-02-18T10:39:05.230163+02:00", - "name": "1.1.1.1", - "account": "TestAccount-456", - "lastReputationRun": "0001-01-01T00:00:00Z", - "manualSetTime": "0001-01-01T00:00:00Z", - "firstSeenEntryID": "API", - "expirationStatus": "active", - "value": "1.1.1.1", - "expirationSource": { - "expirationInterval": 10080, - "source": "indicatorType", - "brand": "", - "instance": "", - "setTime": "2020-02-18T10:39:05.230168+02:00", - "user": "", - "expirationPolicy": "indicatorType", - "moduleId": "" - }, - "isIoc": true, - "expiration": "0001-01-01T00:00:00Z", - "context": null, - "createdTime": "2020-02-18T10:39:05.230184+02:00", - "manuallyEditedFields": [ - "indicator_type" - ], - "calculatedTime": "2020-02-18T10:39:05.230163+02:00", - "manualExpirationTime": "0001-01-01T00:00:00Z" - }, - "investigationsCount": 0, - "primaryTerm": 2, - "score": 0, - "investigationIDs": [], - "type": "IP", - "isShared": true, - "rawName": "1.1.1.1", - "modifiedTime": "0001-01-01T00:00:00Z", - "lastSeenEntryID": "API", - "CustomFields": { - "internal": false - }, - "firstSeen": "2020-02-18T10:39:05.230163+02:00", - "name": "1.1.1.1", - "account": "TestAccount-456", - "lastReputationRun": "0001-01-01T00:00:00Z", - "manualSetTime": "0001-01-01T00:00:00Z", - "firstSeenEntryID": "API", - "expirationStatus": "active", - "value": "1.1.1.1", - "expirationSource": { - "expirationInterval": 10080, - "source": "indicatorType", - "brand": "", - "instance": "", - "setTime": "2020-02-18T10:39:05.230168+02:00", - "user": "", - "expirationPolicy": "indicatorType", - "moduleId": "" - }, - "isIoc": true, - "expiration": "0001-01-01T00:00:00Z", - "context": null, - "createdTime": "2020-02-18T10:39:05.230184+02:00", - "manuallyEditedFields": [ - "indicator_type" - ], - "calculatedTime": "2020-02-18T10:39:05.230163+02:00", - "manualExpirationTime": "0001-01-01T00:00:00Z" - }, - { - "comment": "", - "version": 2, - "sequenceNumber": 25, - "sortValues": null, - "modified": "2020-02-18T11:43:44.200288Z", - "lastSeen": "2020-02-18T10:41:22.268124+02:00", - "id": "5b8656aafcb40bb58caf1d17ef8506a9", - "deletedFeedFetchTime": "0001-01-01T00:00:00Z", - "rawJSON": { - "comment": "", - "version": 2, - "sequenceNumber": 25, - "sortValues": null, - "modified": "2020-02-18T11:43:44.200288Z", - "lastSeen": "2020-02-18T10:41:22.268124+02:00", - "id": "5b8656aafcb40bb58caf1d17ef8506a9", - "deletedFeedFetchTime": "0001-01-01T00:00:00Z", - "investigationsCount": 0, - "primaryTerm": 2, - "score": 0, - "investigationIDs": [], - "type": "IP", - "isShared": true, - "rawName": "2.2.2.2", - "modifiedTime": "0001-01-01T00:00:00Z", - "lastSeenEntryID": "API", - "CustomFields": { - "internal": false - }, - "firstSeen": "2020-02-18T10:41:22.268124+02:00", - "name": "2.2.2.2", - "account": "TestAccount-456", - "lastReputationRun": "0001-01-01T00:00:00Z", - "manualSetTime": "0001-01-01T00:00:00Z", - "firstSeenEntryID": "API", - "expirationStatus": "active", - "value": "2.2.2.2", - "expirationSource": { - "expirationInterval": 10080, - "source": "indicatorType", - "brand": "", - "instance": "", - "setTime": "2020-02-18T10:41:22.268125+02:00", - "user": "", - "expirationPolicy": "indicatorType", - "moduleId": "" - }, - "isIoc": true, - "expiration": "0001-01-01T00:00:00Z", - "context": null, - "createdTime": "2020-02-18T10:41:22.268133+02:00", - "manuallyEditedFields": [ - "indicator_type" - ], - "calculatedTime": "2020-02-18T10:41:22.268124+02:00", - "manualExpirationTime": "0001-01-01T00:00:00Z" - }, - "investigationsCount": 0, - "primaryTerm": 2, - "score": 0, - "investigationIDs": [], - "type": "IP", - "isShared": true, - "rawName": "2.2.2.2", - "modifiedTime": "0001-01-01T00:00:00Z", - "lastSeenEntryID": "API", - "CustomFields": { - "internal": false - }, - "firstSeen": "2020-02-18T10:41:22.268124+02:00", - "name": "2.2.2.2", - "account": "TestAccount-456", - "lastReputationRun": "0001-01-01T00:00:00Z", - "manualSetTime": "0001-01-01T00:00:00Z", - "firstSeenEntryID": "API", - "expirationStatus": "active", - "value": "2.2.2.2", - "expirationSource": { - "expirationInterval": 10080, - "source": "indicatorType", - "brand": "", - "instance": "", - "setTime": "2020-02-18T10:41:22.268125+02:00", - "user": "", - "expirationPolicy": "indicatorType", - "moduleId": "" - }, - "isIoc": true, - "expiration": "0001-01-01T00:00:00Z", - "context": null, - "createdTime": "2020-02-18T10:41:22.268133+02:00", - "manuallyEditedFields": [ - "indicator_type" - ], - "calculatedTime": "2020-02-18T10:41:22.268124+02:00", - "manualExpirationTime": "0001-01-01T00:00:00Z" - } - ], - "Enrichments": [] - } -} -``` - -##### Human Readable Output -### Indicators -|name| -|---| -| 1.1.1.1 | -| 2.2.2.2 | - ### es-get-indicators + *** Gets indicators available in the configured Elasticsearch database. - #### Base Command `es-get-indicators` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | limit | The maximum number of indicators to fetch. The default is 50. Default is 50. | Required | - #### Context Output There is no context output for this command. diff --git a/Packs/FeedElasticsearch/ReleaseNotes/1_1_0.md b/Packs/FeedElasticsearch/ReleaseNotes/1_1_0.md new file mode 100644 index 000000000000..b2fe29fbdd6e --- /dev/null +++ b/Packs/FeedElasticsearch/ReleaseNotes/1_1_0.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### Elasticsearch Feed +- Updated the Docker image to: *demisto/py3-tools:1.0.0.64131*. +- Added support for the Opensearch client. \ No newline at end of file diff --git a/Packs/FeedElasticsearch/pack_metadata.json b/Packs/FeedElasticsearch/pack_metadata.json index 8fa06f902333..97a185773e14 100644 --- a/Packs/FeedElasticsearch/pack_metadata.json +++ b/Packs/FeedElasticsearch/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Elasticsearch Feed", "description": "Indicators feed from Elasticsearch database", "support": "xsoar", - "currentVersion": "1.0.32", + "currentVersion": "1.1.0", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",