From ebba5b0c21a87fafaf66d72ab018cfe24866da66 Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh Date: Wed, 19 Jul 2023 20:13:12 +0530 Subject: [PATCH 1/8] Vectra UDI connector Adding Vectra UDI connector --- stix_shifter_modules/vectra/README.md | 439 ++++++ stix_shifter_modules/vectra/__init__.py | 0 .../vectra/configuration/config.json | 30 + .../vectra/configuration/lang_en.json | 19 + stix_shifter_modules/vectra/entry_point.py | 12 + stix_shifter_modules/vectra/requirements.txt | 1 + .../vectra/stix_translation/__init__.py | 0 .../stix_translation/json/config_map.json | 84 ++ .../stix_translation/json/from_stix_map.json | 299 ++++ .../stix_translation/json/operators.json | 15 + .../json/stix_2_1/from_stix_map.json | 299 ++++ .../json/stix_2_1/to_stix_map.json | 1206 +++++++++++++++++ .../stix_translation/json/to_stix_map.json | 1206 +++++++++++++++++ .../stix_translation/query_constructor.py | 678 +++++++++ .../stix_translation/query_translator.py | 24 + .../vectra/stix_translation/transformers.py | 69 + .../vectra/stix_transmission/__init__.py | 0 .../vectra/stix_transmission/api_client.py | 33 + .../vectra/stix_transmission/connector.py | 215 +++ .../vectra/stix_transmission/error_mapper.py | 34 + .../test_vectra_json_to_stix.py | 362 +++++ .../test_vectra_stix_to_query.py | 509 +++++++ .../test/stix_transmission/test_vectra.py | 463 +++++++ .../vectra/vectra_supported_stix.md | 385 ++++++ 24 files changed, 6382 insertions(+) create mode 100644 stix_shifter_modules/vectra/README.md create mode 100644 stix_shifter_modules/vectra/__init__.py create mode 100644 stix_shifter_modules/vectra/configuration/config.json create mode 100644 stix_shifter_modules/vectra/configuration/lang_en.json create mode 100644 stix_shifter_modules/vectra/entry_point.py create mode 100644 stix_shifter_modules/vectra/requirements.txt create mode 100644 stix_shifter_modules/vectra/stix_translation/__init__.py create mode 100644 stix_shifter_modules/vectra/stix_translation/json/config_map.json create mode 100644 stix_shifter_modules/vectra/stix_translation/json/from_stix_map.json create mode 100644 stix_shifter_modules/vectra/stix_translation/json/operators.json create mode 100644 stix_shifter_modules/vectra/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json create mode 100644 stix_shifter_modules/vectra/stix_translation/query_constructor.py create mode 100644 stix_shifter_modules/vectra/stix_translation/query_translator.py create mode 100644 stix_shifter_modules/vectra/stix_translation/transformers.py create mode 100644 stix_shifter_modules/vectra/stix_transmission/__init__.py create mode 100644 stix_shifter_modules/vectra/stix_transmission/api_client.py create mode 100644 stix_shifter_modules/vectra/stix_transmission/connector.py create mode 100644 stix_shifter_modules/vectra/stix_transmission/error_mapper.py create mode 100644 stix_shifter_modules/vectra/test/stix_translation/test_vectra_json_to_stix.py create mode 100644 stix_shifter_modules/vectra/test/stix_translation/test_vectra_stix_to_query.py create mode 100644 stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py create mode 100644 stix_shifter_modules/vectra/vectra_supported_stix.md diff --git a/stix_shifter_modules/vectra/README.md b/stix_shifter_modules/vectra/README.md new file mode 100644 index 000000000..ddc37e43b --- /dev/null +++ b/stix_shifter_modules/vectra/README.md @@ -0,0 +1,439 @@ +# Vectra NDR Connector + +**Table of Contents** + +- [Vectra NDR API Endpoints](#Vectra NDR-api-endpoints) +- [Pattern expression with STIX and CUSTOM attributes - Single Observation](#single-observation) +- [Pattern expression with STIX and CUSTOM attributes - Multiple Observation](#multiple-observation) +- [STIX Execute Query](#stix-execute-query) +- [Limitations](#limitations) +- [Observations](#observations) +- [References](#references) + +### Vectra NDR API Endpoints + + | Connector Method | Vectra NDR API Endpoint | Method | + |-----|------| ------| + | Ping Endpoint | https://< server >/api/v2.4/health/connectivity | GET | + | Results Endpoint | https://< server >/api/v2.4/search/detections | GET | + +### Format for calling stix-shifter from the command line +``` +python main.py `` `` `` `` +``` +### Pattern expression with STIX and CUSTOM attributes + +#### Single Observation + +#### STIX Translate query to fetch the Hidden HTTP Tunnel detection on a specific ipaddress +```shell +translate vectra query {} "[ipv4-addr:value='1.1.1.1' AND x-ibm-finding:name='Hidden HTTP Tunnel'] START t'2023-06-01T00:00:00.000Z' STOP t'2023-06-12T00:00:00.000Z'" +``` +#### STIX Translate query - output +```json +{ + "queries": [ + "query_string=(detection.detection:\"Hidden HTTP Tunnel\" AND (detection.src_ip:\"1.1.1.1\" OR detection.grouped_details.dst_ips:\"1.1.1.1\" OR detection.grouped_details.dst_hosts.dst_ip:\"1.1.1.1\" OR detection.grouped_details.origin_ip:\"1.1.1.1\" OR detection.grouped_details.sessions.dst_ip:\"1.1.1.1\" OR detection.grouped_details.subnet:\"1.1.1.1\" OR detection.grouped_details.event.dst_ip:\"1.1.1.1\" OR detection.grouped_details.event.dst_ips:\"1.1.1.1\" OR detection.grouped_details.events.sessions.dst_ip:\"1.1.1.1\" OR detection.grouped_details.connection_events.target_host.ip:\"1.1.1.1\") AND (detection.last_timestamp:[2023-06-01T0000 to 2023-06-12T0000]))" + ] +} + +``` + +#### STIX Transmit results + +```shell +transmit +vectra +"{\"host\":\"instance.vectra.com\"}" +"{\"auth\":{\"apitoken\": \"xxxx\"}}" +results +"[query_string=(detection.detection_type:\"Hidden HTTP Tunnel\" AND (detection.src_ip:\"1.1.1.1\" OR detection.grouped_details.dst_ips:\"1.1.1.1\" OR detection.grouped_details.dst_hosts.dst_ip:\"1.1.1.1\" OR detection.grouped_details.origin_ip:\"1.1.1.1\" OR detection.grouped_details.sessions.dst_ip:\"1.1.1.1\" OR detection.grouped_details.subnet:\"1.1.1.1\" OR detection.grouped_details.events.dst_ip:\"1.1.1.1\" OR detection.grouped_details.events.dst_ips:\"1.1.1.1\" OR detection.grouped_details.events.sessions.dst_ip:\"1.1.1.1\" OR detection.grouped_details.connection_events.target_host.ip:\"1.1.1.1\") AND (detection.last_timestamp:[2023-04-01T0000 to 2023-06-12T0000]))]" +0 +1 +``` + +#### STIX Transmit results - output +```json +{ + "success": true, + "data": [ + { + "id": 13660, + "category": "COMMAND & CONTROL", + "detection": "Hidden HTTP Tunnel", + "detection_category": "COMMAND & CONTROL", + "detection_type": "Hidden HTTP Tunnel", + "custom_detection": null, + "description": null, + "src_ip": "1.1.1.1", + "state": "inactive", + "certainty": 0, + "threat": 0, + "created_timestamp": "2023-04-04T02:25:19Z", + "first_timestamp": "2023-04-04T02:12:42Z", + "last_timestamp": "2023-04-04T02:23:32Z", + "targets_key_asset": false, + "is_targeting_key_asset": false, + "src_account": null, + "src_host": { + "id": 872, + "ip": "1.1.1.1", + "name": "VMAL #2 windows 1.1.1.1 (higaki-ha11)", + "is_key_asset": false, + "groups": [ + { + "id": 145, + "name": "Super Test domain group", + "description": "created during API testing", + "last_modified": "2022-08-03T15:33:04Z", + "last_modified_by": "reliaquest", + "type": "host" + }, + { + "id": 144, + "name": "Partner VLAB - User Devices", + "description": "", + "last_modified": "2022-01-27T12:05:24Z", + "last_modified_by": "user (Removed)", + "type": "ip" + } + ], + "threat": 82, + "certainty": 71 + }, + "note": null, + "note_modified_by": null, + "note_modified_timestamp": null, + "sensor": "eti2pc2s", + "sensor_name": "Vec2c610896102c466a28f49a", + "tags": [], + "triage_rule_id": null, + "assigned_to": "vectra_sir_admin", + "assigned_date": "2022-12-14T06:59:22Z", + "groups": [ + { + "id": 144, + "name": "Partner VLAB - User Devices", + "description": "", + "type": "ip", + "last_modified": "2022-01-27T12:05:24Z", + "last_modified_by": "user" + } + ], + "is_marked_custom": false, + "is_custom_model": false, + "src_linked_account": null, + "grouped_details": [ + { + "description": null, + "dst_geo": null, + "dst_geo_lat": null, + "dst_geo_lon": null, + "first_timestamp": "2023-04-04T02:12:42Z", + "last_timestamp": "2023-04-04T02:23:32Z", + "flex_json": {}, + "detection_guid": null, + "distilled_context": {}, + "sequence_id": null, + "is_host_detail": true, + "is_account_detail": false, + "account_uid": null, + "account_detection": null, + "host_detection": 13660, + "accounts": [], + "target_domains": [], + "dst_ports": [ + 80 + ], + "protocol": "tcp", + "bytes_received": 336025, + "bytes_sent": 2699, + "dst_ips": [ + "2.2.2.2" + ], + "src_ip": "1.1.1.1", + "num_sessions": 197 + } + ], + "summary": { + "bytes_received": 336025, + "bytes_sent": 2699, + "dst_ips": [ + "2.2.2.2" + ], + "dst_ports": [ + 80 + ], + "num_sessions": 197 + }, + "campaign_summaries": [], + "is_triaged": false, + "filtered_by_ai": false, + "filtered_by_user": false, + "filtered_by_rule": false, + "_doc_modified_ts": "2023-06-08T06:21:44.371724" + } + ] +} +``` + + +#### STIX Translate results + +```json +{ + "type": "bundle", + "id": "bundle--828a180e-a942-4102-b62a-6e524f62c97d", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "vectra", + "identity_class": "events", + "created": "2022-04-11T16:11:11.878Z", + "modified": "2022-04-11T16:11:11.878Z" + }, + { + "id": "observed-data--d72f6317-6771-4980-a33d-e773bb4cb94c", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2023-06-08T07:55:56.262Z", + "modified": "2023-06-08T07:55:56.262Z", + "objects": { + "0": { + "type": "x-ibm-finding", + "alert_id": 13660, + "ttp_tagging_refs": [ + "1" + ], + "name": "Hidden HTTP Tunnel", + "finding_type": "alert", + "src_ip_ref": "2", + "x_state": "inactive", + "confidence": 0, + "severity": 0, + "time_observed": "2023-04-04T02:25:19Z", + "start": "2023-04-04T02:12:42Z", + "end": "2023-04-04T02:23:32Z", + "x_sensor_name": "Vec2c610896102c466a28f49a", + "x_assigned_to": "vectra_sir_admin", + "x_assigned_date": "2022-12-14T06:59:22Z", + "ioc_refs": [ + "5" + ], + "x_dst_ports": [ + 80 + ], + "event_count": 197, + "x_is_triaged": false + }, + "1": { + "type": "x-ibm-ttp-tagging", + "kill_chain_phases": { + "kill_chain_name": "mitre-attack", + "phase_name": "COMMAND & CONTROL" + }, + "name": "Hidden HTTP Tunnel", + "confidence": 0.0 + }, + "2": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "3": { + "type": "x-oca-asset", + "ip_refs": [ + "2" + ], + "device_id": 872, + "hostname": "VMAL #2 windows 1.1.1.1 (higaki-ha11)", + "x_is_key_asset": false, + "x_threat": 82, + "x_certainty": 71 + }, + "4": { + "type": "network-traffic", + "start": "2023-04-04T02:12:42Z", + "end": "2023-04-04T02:23:32Z", + "dst_port": 80, + "protocols": [ + "tcp" + ], + "dst_byte_count": 336025, + "src_byte_count": 2699, + "dst_ref": "5", + "src_ref": "2", + "x_num_sessions": 197 + }, + "5": { + "type": "ipv4-addr", + "value": "2.2.2.2" + } + }, + "first_observed": "2023-06-08T07:55:56.262Z", + "last_observed": "2023-06-08T07:55:56.262Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} + +``` + +#### Multiple Observation + +```shell +translate vectra query {} "([x-ibm-finding:confidence>20 AND x-sql-request-info:response_code=404] AND [x-ibm-finding:severity>20 AND x-sql-request-info:user_agent LIKE 'Mozilla']) START t'2023-04-01T00:00:00.000Z' STOP t'2023-06-12T00:00:00.000Z'" +``` + +#### STIX Multiple observation - output +```json +{ + "queries": [ + "query_string=(detection.grouped_details.targets.events.response_code:\"404\" AND detection.certainty:>\"20\" AND (detection.last_timestamp:[2023-04-01T0000 to 2023-06-12T0000])) OR (detection.grouped_details.targets.events.user_agent:Mozilla AND detection.threat:>\"20\" AND (detection.last_timestamp:[2023-04-01T0000 to 2023-06-12T0000]))" + ] +} +``` + +### STIX Execute query +```shell +execute +vectra +vectra +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"Vectra NDR\",\"identity_class\":\"system\",\"created\":\"2023-02-23T13:22:50.336Z\",\"modified\":\"2022-02-23T13:22:50.336Z\"}" +"{\"host\":\"xyz\"}" +"{\"auth\":{\"api_token\": \"xxx\"}}" +"([x-ibm-finding:confidence>20 AND x-sql-request-info:response_code=404] AND [x-ibm-finding:severity>20 AND x-sql-request-info:user_agent LIKE 'Mozilla']) START t'2023-04-01T00:00:00.000Z' STOP t'2023-06-12T00:00:00.000Z'" +``` + +#### STIX Execute query - output +```json +{ + "type": "bundle", + "id": "bundle--a819ae5e-d4ab-451c-b460-eee6ccf972aa", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "vectra", + "identity_class": "events", + "created": "2022-04-11T16:11:11.878Z", + "modified": "2022-04-11T16:11:11.878Z" + }, + { + "id": "observed-data--d873c3cd-60fb-4ae8-9434-9bb15eda17e5", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2023-06-08T09:43:45.729Z", + "modified": "2023-06-08T09:43:45.729Z", + "objects": { + "0": { + "type": "x-ibm-finding", + "alert_id": 13733, + "ttp_tagging_refs": [ + "1" + ], + "name": "SQL Injection Activity", + "finding_type": "alert", + "src_ip_ref": "2", + "x_state": "active", + "confidence": 89, + "severity": 47, + "time_observed": "2023-06-06T10:12:30Z", + "start": "2023-06-06T10:11:12Z", + "end": "2023-06-06T10:51:20Z", + "x_sensor_name": "Vec2c610896a94728f49a", + "ioc_refs": [ + "5" + ], + "event_count": 2, + "x_is_triaged": false + }, + "1": { + "type": "x-ibm-ttp-tagging", + "kill_chain_phases": { + "kill_chain_name": "mitre-attack", + "phase_name": "LATERAL MOVEMENT" + }, + "name": "SQL Injection Activity", + "confidence": 0.89 + }, + "2": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "3": { + "type": "x-oca-asset", + "ip_refs": [ + "2" + ], + "device_id": 1710, + "hostname": "dogrady150", + "x_is_key_asset": false, + "x_threat": 84, + "x_certainty": 82 + }, + "4": { + "type": "network-traffic", + "start": "2023-06-06T10:11:12Z", + "end": "2023-06-06T10:51:20Z", + "dst_ref": "5", + "src_ref": "2", + "x_sql_request_info_refs": [ + "6", + "7" + ], + "dst_byte_count": 2766, + "src_byte_count": 598, + "protocols": [ + "tcp" + ] + }, + "5": { + "type": "ipv4-addr", + "value": "2.2.2.2" + }, + "6": { + "type": "x-sql-request-info", + "http_segment": "/testing.asp?id=1+and+1=%28select%20%27MROwxCvP%27%2b%40%40servername%2b%27DEXMymnrh%27%29;--", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36 Edg/97.0.1072.69", + "sql_fragment": "1 and 1=(select 'MROwxCvP'+@@servername+'DEXMymnrh');--", + "response_code": "404", + "bytes_received": 1383, + "last_seen": "2023-06-06T10:51:20Z" + }, + "7": { + "type": "x-sql-request-info", + "http_segment": "/testing.asp?id=1+and+1=%28select%20%27UJCfFWifx%27%2b%40%40servername%2b%27vIHAyYiH%27%29;--", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36 Edg/97.0.1072.69", + "sql_fragment": "1 and 1=(select 'UJCfFWifx'+@@servername+'vIHAyYiH');--", + "response_code": "404", + "bytes_received": 1383, + "last_seen": "2023-06-06T10:11:12Z" + } + }, + "first_observed": "2023-06-08T09:43:45.729Z", + "last_observed": "2023-06-08T09:43:45.729Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +### Limitations +- Detections are stored in Detect for Network and searchable for 180 days. +- It is observed that Vectra API returns 'Query too long' error if translated query string length is more than MAX_QUERY_LENGTH. To avoid this error, query is split at 'OR' conditions and multiple requests are made to vectra to get the result. However, the part of query joined by 'AND' operator won't be split and 'Query too long' error is returned if the length of that part is beyond MAX_QUERY_LENGTH. +- Operator 'NOT' is applicable to string type fields only. +- Operators 'LIKE' and 'MATCH' doesn't work if space(s) are present in the field value. + +### Observations +- Invalid query error is returned if special character(s) is used in field value for '=' operator. To avoid this use 'LIKE' operator without special character(s) for this field. Example; For query '[x-ldap-event:request = '(&(objectClass=group))']' use '[x-ldap-event:request LIKE 'objectClass=group']'. +- UTC Format of any timestamp field value in query is converted to vectra timestamp format (YYYY-MM-DDTHHMM) during translation thereby dropping millisecond part of UTC format. +- Query on 'grouped-details:subnet' field works for the network ID part of the field only. For example to create query on "subnet": "10.250.50.0/24" following query shoud be used [x-grouped-details:subnet = '10.250.50'] + +### References +- [Vectra REST API Guide v2.4](https://support.vectra.ai/s/article/KB-VS-1626) +- [Advanced Search Reference Guide](https://support.vectra.ai/s/article/KB-VS-1116) +- [Understanding Vectra AI](https://support.vectra.ai/s/article/KB-VS-1285) +- [Detection and Campaign lifespan and retention periods](https://support.vectra.ai/s/article/KB-VS-1099) diff --git a/stix_shifter_modules/vectra/__init__.py b/stix_shifter_modules/vectra/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/vectra/configuration/config.json b/stix_shifter_modules/vectra/configuration/config.json new file mode 100644 index 000000000..d15c190bf --- /dev/null +++ b/stix_shifter_modules/vectra/configuration/config.json @@ -0,0 +1,30 @@ +{ + "connection": { + "type": { + "displayName": "Vectra", + "group": "vectra" + }, + "host": { + "type": "text", + "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$" + }, + "help": { + "type": "link", + "default": "data-sources.html" + }, + "options": { + "type": "fields", + "result_limit": { + "max": 10000 + } + } + }, + "configuration": { + "auth": { + "type" : "fields", + "api_token": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/vectra/configuration/lang_en.json b/stix_shifter_modules/vectra/configuration/lang_en.json new file mode 100644 index 000000000..2f5708bd1 --- /dev/null +++ b/stix_shifter_modules/vectra/configuration/lang_en.json @@ -0,0 +1,19 @@ +{ + "connection": { + "host": { + "label": "Management IP address or Hostname", + "description": "Specify the IP address or hostname of the data source" + }, + "help": { + "label": "Need additional help?", + "description": "More details on the data source setting can be found in the specified link" + } + }, + "configuration": { + "auth": { + "api_token": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/vectra/entry_point.py b/stix_shifter_modules/vectra/entry_point.py new file mode 100644 index 000000000..9039155cc --- /dev/null +++ b/stix_shifter_modules/vectra/entry_point.py @@ -0,0 +1,12 @@ +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint + +class EntryPoint(BaseEntryPoint): + + # python main.py translate vectra query '{}' "[ipv4-addr:value = '127.0.0.1']" + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + self.set_async(False) + if connection: + self.setup_transmission_basic(connection, configuration) + self.setup_translation_simple(dialect_default='default') diff --git a/stix_shifter_modules/vectra/requirements.txt b/stix_shifter_modules/vectra/requirements.txt new file mode 100644 index 000000000..9e8d893e6 --- /dev/null +++ b/stix_shifter_modules/vectra/requirements.txt @@ -0,0 +1 @@ +pyparsing==3.0.9 \ No newline at end of file diff --git a/stix_shifter_modules/vectra/stix_translation/__init__.py b/stix_shifter_modules/vectra/stix_translation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/vectra/stix_translation/json/config_map.json b/stix_shifter_modules/vectra/stix_translation/json/config_map.json new file mode 100644 index 000000000..65171fea9 --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/json/config_map.json @@ -0,0 +1,84 @@ +{ + "int_supported_fields": [ + "detection.grouped_details.connection_events.dst_port", + "detection.grouped_details.num_events", + "detection.certainty", + "detection.grouped_details.events.target_summary.dst_port", + "detection.grouped_details.count", + "detection.grouped_details.sessions.bytes_sent", + "detection.grouped_details.events.count", + "detection.id", + "detection.grouped_details.events.dst_ports", + "detection.grouped_details.events.sessions.dst_port", + "detection.grouped_details.duration", + "detection.grouped_details.num_response_objects", + "detection.grouped_details.dst_ports", + "detection.grouped_details.events.num_response_objects", + "detection.grouped_details.connection_events.total_bytes_sent", + "detection.grouped_details.src_account.privilege_level", + "detection.grouped_details.anomalous_profiles.count", + "detection.grouped_details.events.bytes_sent", + "detection.grouped_details.src_host.privilege_level", + "detection.grouped_details.connection_events.total_bytes_rcvd", + "detection.grouped_details.num_services_high_privilege", + "detection.grouped_details.bytes_received", + "detection.grouped_details.targets.events.bytes_received", + "detection.grouped_details.events.sessions.duration", + "detection.grouped_details.dst_hosts.dst_port", + "detection.grouped_details.num_attempts", + "detection.grouped_details.num_services_requested", + "detection.grouped_details.bytes_sent", + "detection.grouped_details.events.bytes_received", + "detection.grouped_details.num_sessions", + "detection.grouped_details.sessions.dst_port", + "detection.grouped_details.service_accessed.privilege_level", + "detection.grouped_details.events.duration", + "detection.grouped_details.num_accounts", + "detection.grouped_details.origin_port", + "detection.grouped_details.sessions.bytes_received", + "detection.grouped_details.dst_geo_lat", + "detection.grouped_details.origin_geo_lat", + "detection.grouped_details.dst_geo_lon", + "detection.grouped_details.origin_geo_lon", + "detection.threat", + "detection.summary.num_sessions", + "detection.summary.num_attempts", + "detection.summary.num_successes", + "detection.summary.dst_ports", + "detection.summary.matches", + "detection.src_host.id", + "detection.src_host.threat", + "detection.src_host.certainty", + "detection.grouped_details.connection_events.duration_int" + ], + "timestamp_supported_fields": [ + "detection.first_timestamp", + "detection.grouped_details.first_timestamp", + "detection.grouped_details.sessions.first_timestamp", + "detection.grouped_details.events.first_timestamp", + "detection.grouped_details.events.sessions.first_timestamp", + "detection.grouped_details.events.target_summary.first_timestamp", + "detection.grouped_details.connection_events.first_timestamp", + "detection.last_timestamp", + "detection.grouped_details.last_timestamp", + "detection.grouped_details.dst_hosts.last_timestamp", + "detection.grouped_details.sessions.last_timestamp", + "detection.grouped_details.events.last_seen", + "detection.grouped_details.events.last_timestamp", + "detection.grouped_details.events.target_summary.last_timestamp", + "detection.grouped_details.connection_events.last_timestamp", + "detection.created_timestamp", + "detection.assigned_date", + "detection.grouped_details.first_seen", + "detection.grouped_details.last_seen", + "detection.grouped_details.targets.events.last_seen", + "detection.grouped_details.anomalous_profiles.first_timestamp", + "detection.grouped_details.anomalous_profiles.last_timestamp" + ], + "boolean_supported_fields": [ + "detection.is_triaged", + "detection.src_host.is_key_asset", + "detection.grouped_details.events.is_normally_accessed_by_rdp", + "detection.grouped_details.connection_events.is_external" + ] +} \ No newline at end of file diff --git a/stix_shifter_modules/vectra/stix_translation/json/from_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/from_stix_map.json new file mode 100644 index 000000000..97ced88e4 --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/json/from_stix_map.json @@ -0,0 +1,299 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "detection.src_ip", + "detection.grouped_details.dst_ips", + "detection.grouped_details.dst_hosts.dst_ip", + "detection.grouped_details.normal_admin_hosts.ip", + "detection.grouped_details.dst_hosts.ip", + "detection.grouped_details.origin_ip", + "detection.grouped_details.sessions.dst_ip", + "detection.grouped_details.events.dst_ip", + "detection.grouped_details.events.dst_ips", + "detection.grouped_details.events.sessions.dst_ip", + "detection.grouped_details.connection_events.target_host.ip" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "detection.src_ip", + "detection.grouped_details.dst_ips", + "detection.grouped_details.dst_hosts.dst_ip", + "detection.grouped_details.normal_admin_hosts.ip", + "detection.grouped_details.dst_hosts.ip", + "detection.grouped_details.origin_ip", + "detection.grouped_details.sessions.dst_ip", + "detection.grouped_details.events.dst_ip", + "detection.grouped_details.events.dst_ips", + "detection.grouped_details.events.sessions.dst_ip", + "detection.grouped_details.connection_events.target_host.ip" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "detection.grouped_details.target_domains", + "detection.grouped_details.origin_domain", + "detection.grouped_details.events.target_domains", + "detection.grouped_details.connection_events.target_host.dst_dns" + ] + } + }, + "network-traffic": { + "fields": { + "dst_port": [ + "detection.grouped_details.dst_ports", + "detection.grouped_details.dst_hosts.dst_port", + "detection.grouped_details.origin_port", + "detection.grouped_details.sessions.dst_port", + "detection.grouped_details.events.dst_ports", + "detection.grouped_details.events.sessions.dst_port", + "detection.grouped_details.events.target_summary.dst_port", + "detection.grouped_details.connection_events.dst_port" + ], + "src_port": ["detection.grouped_details.src_port"], + "src_ref.value": ["detection.src_ip"], + "dst_ref.value": [ + "detection.grouped_details.dst_ips", + "detection.grouped_details.dst_hosts.dst_ip", + "detection.grouped_details.sessions.dst_ip", + "detection.grouped_details.origin_ip", + "detection.grouped_details.events.sessions.dst_ip", + "detection.grouped_details.connection_events.target_host.ip" + ], + "protocols[*]": [ + "detection.grouped_details.protocol", + "detection.grouped_details.app_protocol", + "detection.grouped_details.dst_protocol", + "detection.grouped_details.origin_protocol", + "detection.grouped_details.sessions.protocol", + "detection.grouped_details.sessions.app_protocol", + "detection.grouped_details.events.protocol", + "detection.grouped_details.events.sessions.app_protocol", + "detection.grouped_details.events.sessions.protocol", + "detection.grouped_details.events.target_summary.app_protocol", + "detection.grouped_details.events.target_summary.protocol", + "detection.grouped_details.connection_events.protocol" + ], + "src_byte_count": [ + "detection.grouped_details.bytes_sent", + "detection.grouped_details.sessions.bytes_sent", + "detection.grouped_details.events.bytes_sent", + "detection.grouped_details.connection_events.total_bytes_sent" + ], + "dst_byte_count": [ + "detection.grouped_details.bytes_received", + "detection.grouped_details.sessions.bytes_received", + "detection.grouped_details.events.bytes_received", + "detection.grouped_details.events.sessions.bytes_received", + "detection.grouped_details.connection_events.total_bytes_rcvd" + ], + "start": [ + "detection.first_timestamp", + "detection.grouped_details.first_timestamp", + "detection.grouped_details.sessions.first_timestamp", + "detection.grouped_details.events.first_timestamp", + "detection.grouped_details.events.sessions.first_timestamp", + "detection.grouped_details.events.target_summary.first_timestamp", + "detection.grouped_details.connection_events.first_timestamp" + ], + "end": [ + "detection.last_timestamp", + "detection.grouped_details.last_timestamp", + "detection.grouped_details.dst_hosts.last_timestamp", + "detection.grouped_details.sessions.last_timestamp", + "detection.grouped_details.events.last_seen", + "detection.grouped_details.events.last_timestamp", + "detection.grouped_details.events.target_summary.last_timestamp", + "detection.grouped_details.connection_events.last_timestamp" + ], + "x_count": ["detection.grouped_details.events.count"], + "x_dst_country": ["detection.grouped_details.events.dst_country"], + "x_num_accounts": ["detection.grouped_details.num_accounts"], + "x_reason": ["detection.grouped_details.reason"], + "x_num_attempts": ["detection.grouped_details.num_attempts"], + "x_tunnel_type": ["detection.grouped_details.sessions.tunnel_type"], + "x_num_sessions": ["detection.grouped_details.num_sessions"], + "x_user_agent": ["detection.grouped_details.user_agent"], + "x_dst_geo_latitude": [ + "detection.grouped_details.dst_geo_lat", + "detection.grouped_details.origin_geo_lat" + ], + "x_dst_geo_longitude": [ + "detection.grouped_details.dst_geo_lon", + "detection.grouped_details.origin_geo_lon" + ], + "x_dst_geo": [ + "detection.grouped_details.dst_geo", + "detection.grouped_details.origin_geo" + ], + "x_num_response_objects": ["detection.grouped_details.num_response_objects"], + "x_client_name": ["detection.grouped_details.client_name"], + "x_client_token": ["detection.grouped_details.client_token"], + "x_is_normally_accessed_by_rdp": ["detection.grouped_details.events.is_normally_accessed_by_rdp"], + "x_rpc_uuid": ["detection.grouped_details.uuid"], + "x_nt_referrer": ["detection.grouped_details.events.referrer"], + "x_num_events": ["detection.grouped_details.num_events"], + "x_time_duration": [ + "detection.grouped_details.duration", + "detection.grouped_details.events.duration", + "detection.grouped_details.events.sessions.duration", + "detection.grouped_details.connection_events.duration_int" + ], + "x_status_code": ["detection.grouped_details.status_code"], + "x_named_pipe": ["detection.grouped_details.named_pipe"], + "x_uri": ["detection.grouped_details.uri"], + "x_src_session_uid": ["detection.grouped_details.metadata.orig_sluid"], + "x_executed_functions": ["detection.grouped_details.executed_functions"], + "x_event_type": ["detection.grouped_details.events.event_type"], + "x_error_code": ["detection.grouped_details.events.error_code"], + "x_target_domain_refs[*].value": ["detection.grouped_details.events.target_domains"], + "x_is_external": ["detection.grouped_details.connection_events.is_external"], + "x_request_uri": ["detection.grouped_details.uri"], + "x_period_identified": ["detection.grouped_details.period_identified"], + "x_smb_share": ["detection.grouped_details.share"], + "x_account_uid": ["detection.grouped_details.account_uid"], + "extensions.'http-request-ext'.request_method": ["detection.grouped_details.events.http_method"], + "extensions.'http-request-ext'.x_response_code": ["detection.grouped_details.events.response_code"], + "extensions.'http-request-ext'.request_header.'User-Agent'": ["detection.grouped_details.user_agent"] + } + }, + "user-account": { + "fields": { + "user_id": [ + "detection.grouped_details.dst_accounts.uid", + "detection.grouped_details.src_account.name", + "detection.grouped_details.normal_users", + "detection.summary.accounts" + ], + "account_login": ["detection.grouped_details.src_account.name"], + "x_privilege_category": ["detection.grouped_details.src_account.privilege_category"], + "x_privilege_level": ["detection.grouped_details.src_account.privilege_level"] + } + }, + "x-ibm-finding": { + "fields": { + "name": ["detection.detection_type"], + "alert_id": ["detection.id"], + "description": ["detection.description", "detection.summary.description"], + "x_num_sessions": ["detection.grouped_details.num_sessions"], + "severity": ["detection.threat"], + "confidence": ["detection.certainty"], + "start": ["detection.first_timestamp"], + "end": ["detection.last_timestamp"], + "time_observed": ["detection.created_timestamp"], + "event_count": ["detection.summary.num_sessions", "detection.summary.num_attempts"], + "x_state": ["detection.state"], + "x_num_successes": ["detection.summary.num_successes"], + "x_assigned_to": [ + "detection.assigned_to" + ], + "x_assigned_date": [ + "detection.assigned_date" + ], + "x_sensor_name": ["detection.sensor_name"], + "x_is_triaged": ["detection.is_triaged"], + "src_ip_ref": ["detection.src_ip"], + "x_dst_ports": ["detection.summary.dst_ports"], + "x_account_refs.user_id": ["detection.summary.accounts"], + "x_shares": ["detection.summary.shares"], + "x_probable_owner": ["detection.summary.probable_owner"], + "x_matches": ["detection.summary.matches"] + } + }, + "x-ibm-ttp-tagging": { + "fields": { + "name": ["detection.detection_type"], + "confidence": ["detection.certainty"], + "kill_chain_phases.phase_name": ["detection.detection_category"] + } + }, + "x-oca-asset": { + "fields": { + "hostname": ["detection.src_host.name"], + "device_id": ["detection.src_host.id"], + "x_is_key_asset": ["detection.src_host.is_key_asset"], + "ip_refs[*].value": ["detection.src_ip"], + "x_threat": ["detection.src_host.threat"], + "x_certainty": ["detection.src_host.certainty"], + "x_privilege_category": ["detection.grouped_details.src_host.privilege_category"], + "x_privilege_level": ["detection.grouped_details.src_host.privilege_level"] + } + }, + "x-grouped-details": { + "fields": { + "first_seen": ["detection.grouped_details.first_seen"], + "last_seen": ["detection.grouped_details.last_seen"], + "detection_source": ["detection.grouped_details.detection_source"], + "detection_slug": ["detection.grouped_details.detection_slug"], + "account_ref.user_id": ["detection.grouped_details.src_account.name"], + "ja3_hashes": ["detection.grouped_details.ja3_hashes"], + "ja3s_hashes": ["detection.grouped_details.ja3s_hashes"], + "x_num_sessions": ["detection.grouped_details.num_sessions"], + "start": ["detection.grouped_details.first_timestamp"], + "end": ["detection.grouped_details.last_timestamp"], + "count": ["detection.grouped_details.count"], + "client_name": ["detection.grouped_details.client_name"], + "client_token": ["detection.grouped_details.client_token"], + "dst_byte_count": ["detection.grouped_details.bytes_received"], + "src_byte_count": ["detection.grouped_details.bytes_sent"], + "subnet": ["detection.grouped_details.subnet"], + "rpc_function_uuid": ["detection.grouped_details.uuid"], + "num_services_requested": ["detection.grouped_details.num_services_requested"], + "num_services_high_privilege": ["detection.grouped_details.num_services_high_privilege"], + "service_privilege": ["detection.grouped_details.service_privilege"] + } + }, + "x-service-accessed-info": { + "fields": { + "name": ["detection.grouped_details.service_accessed.name"], + "privilege_category": ["detection.grouped_details.service_accessed.privilege_category"], + "privilege_level": ["detection.grouped_details.service_accessed.privilege_level"] + } + }, + "x-ldap-event": { + "fields": { + "base_object": ["detection.grouped_details.events.base_object"], + "request": ["detection.grouped_details.events.request"], + "response_code": ["detection.grouped_details.events.response_code"], + "num_response_objects": ["detection.grouped_details.events.num_response_objects"], + "last_timestamp": ["detection.grouped_details.events.last_timestamp"] + } + }, + "x-sql-request-info": { + "fields": { + "http_segment": ["detection.grouped_details.targets.events.http_segment"], + "user_agent": ["detection.grouped_details.targets.events.user_agent"], + "sql_fragment": ["detection.grouped_details.targets.events.sql_fragment"], + "response_code": ["detection.grouped_details.targets.events.response_code"], + "bytes_received": ["detection.grouped_details.targets.events.bytes_received"], + "last_seen": ["detection.grouped_details.targets.events.last_seen"] + } + }, + "x-anomalous-rpc": { + "fields": { + "function_call": ["detection.grouped_details.anomalous_profiles.function_call"], + "rpc_function_uuid": ["detection.grouped_details.anomalous_profiles.function_uuid"], + "count": ["detection.grouped_details.anomalous_profiles.count"], + "start": ["detection.grouped_details.anomalous_profiles.first_timestamp"], + "end": ["detection.grouped_details.anomalous_profiles.last_timestamp"] + } + }, + "x-services-requested": { + "fields": { + "service": ["detection.grouped_details.services_requested.service"] + } + }, + "x-new-host-info": { + "fields": { + "artifact": ["detection.grouped_details.artifact"], + "via": ["detection.grouped_details.via"], + "role": ["detection.grouped_details.role"], + "end": ["detection.grouped_details.last_timestamp"] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/vectra/stix_translation/json/operators.json b/stix_shifter_modules/vectra/stix_translation/json/operators.json new file mode 100644 index 000000000..6053cff6b --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/json/operators.json @@ -0,0 +1,15 @@ +{ + "ComparisonExpressionOperators.And": "AND", + "ComparisonExpressionOperators.Or": "OR", + "ComparisonComparators.GreaterThan": ":>", + "ComparisonComparators.LessThan": ":<", + "ComparisonComparators.GreaterThanOrEqual": ":>=", + "ComparisonComparators.LessThanOrEqual": ":<=", + "ComparisonComparators.Equal": ":", + "ComparisonComparators.Like": ":", + "ComparisonComparators.In": ":", + "ComparisonComparators.Matches": ":*", + "ComparisonComparators.NotEqual": ":", + "ObservationOperators.Or": "OR", + "ObservationOperators.And": "OR" +} diff --git a/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..53002daf2 --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,299 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "detection.src_ip", + "detection.grouped_details.dst_ips", + "detection.grouped_details.dst_hosts.dst_ip", + "detection.grouped_details.normal_admin_hosts.ip", + "detection.grouped_details.dst_hosts.ip", + "detection.grouped_details.origin_ip", + "detection.grouped_details.sessions.dst_ip", + "detection.grouped_details.events.dst_ip", + "detection.grouped_details.events.dst_ips", + "detection.grouped_details.events.sessions.dst_ip", + "detection.grouped_details.connection_events.target_host.ip" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "detection.src_ip", + "detection.grouped_details.dst_ips", + "detection.grouped_details.dst_hosts.dst_ip", + "detection.grouped_details.normal_admin_hosts.ip", + "detection.grouped_details.dst_hosts.ip", + "detection.grouped_details.origin_ip", + "detection.grouped_details.sessions.dst_ip", + "detection.grouped_details.events.dst_ip", + "detection.grouped_details.events.dst_ips", + "detection.grouped_details.events.sessions.dst_ip", + "detection.grouped_details.connection_events.target_host.ip" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "detection.grouped_details.target_domains", + "detection.grouped_details.origin_domain", + "detection.grouped_details.events.target_domains", + "detection.grouped_details.connection_events.target_host.dst_dns" + ] + } + }, + "network-traffic": { + "fields": { + "dst_port": [ + "detection.grouped_details.dst_ports", + "detection.grouped_details.dst_hosts.dst_port", + "detection.grouped_details.origin_port", + "detection.grouped_details.sessions.dst_port", + "detection.grouped_details.events.dst_ports", + "detection.grouped_details.events.sessions.dst_port", + "detection.grouped_details.events.target_summary.dst_port", + "detection.grouped_details.connection_events.dst_port" + ], + "src_port": ["detection.grouped_details.src_port"], + "src_ref.value": ["detection.src_ip"], + "dst_ref.value": [ + "detection.grouped_details.dst_ips", + "detection.grouped_details.dst_hosts.dst_ip", + "detection.grouped_details.sessions.dst_ip", + "detection.grouped_details.origin_ip", + "detection.grouped_details.events.sessions.dst_ip", + "detection.grouped_details.connection_events.target_host.ip" + ], + "protocols[*]": [ + "detection.grouped_details.protocol", + "detection.grouped_details.app_protocol", + "detection.grouped_details.dst_protocol", + "detection.grouped_details.origin_protocol", + "detection.grouped_details.sessions.protocol", + "detection.grouped_details.sessions.app_protocol", + "detection.grouped_details.events.protocol", + "detection.grouped_details.events.sessions.app_protocol", + "detection.grouped_details.events.sessions.protocol", + "detection.grouped_details.events.target_summary.app_protocol", + "detection.grouped_details.events.target_summary.protocol", + "detection.grouped_details.connection_events.protocol" + ], + "src_byte_count": [ + "detection.grouped_details.bytes_sent", + "detection.grouped_details.sessions.bytes_sent", + "detection.grouped_details.events.bytes_sent", + "detection.grouped_details.connection_events.total_bytes_sent" + ], + "dst_byte_count": [ + "detection.grouped_details.bytes_received", + "detection.grouped_details.sessions.bytes_received", + "detection.grouped_details.events.bytes_received", + "detection.grouped_details.events.sessions.bytes_received", + "detection.grouped_details.connection_events.total_bytes_rcvd" + ], + "start": [ + "detection.first_timestamp", + "detection.grouped_details.first_timestamp", + "detection.grouped_details.sessions.first_timestamp", + "detection.grouped_details.events.first_timestamp", + "detection.grouped_details.events.sessions.first_timestamp", + "detection.grouped_details.events.target_summary.first_timestamp", + "detection.grouped_details.connection_events.first_timestamp" + ], + "end": [ + "detection.last_timestamp", + "detection.grouped_details.last_timestamp", + "detection.grouped_details.dst_hosts.last_timestamp", + "detection.grouped_details.sessions.last_timestamp", + "detection.grouped_details.events.last_seen", + "detection.grouped_details.events.last_timestamp", + "detection.grouped_details.events.target_summary.last_timestamp", + "detection.grouped_details.connection_events.last_timestamp" + ], + "x_count": ["detection.grouped_details.events.count"], + "x_dst_country": ["detection.grouped_details.events.dst_country"], + "x_num_accounts": ["detection.grouped_details.num_accounts"], + "x_reason": ["detection.grouped_details.reason"], + "x_num_attempts": ["detection.grouped_details.num_attempts"], + "x_tunnel_type": ["detection.grouped_details.sessions.tunnel_type"], + "x_num_sessions": ["detection.grouped_details.num_sessions"], + "x_user_agent": ["detection.grouped_details.user_agent"], + "x_dst_geo_latitude": [ + "detection.grouped_details.dst_geo_lat", + "detection.grouped_details.origin_geo_lat" + ], + "x_dst_geo_longitude": [ + "detection.grouped_details.dst_geo_lon", + "detection.grouped_details.origin_geo_lon" + ], + "x_dst_geo": [ + "detection.grouped_details.dst_geo", + "detection.grouped_details.origin_geo" + ], + "x_num_response_objects": ["detection.grouped_details.num_response_objects"], + "x_client_name": ["detection.grouped_details.client_name"], + "x_client_token": ["detection.grouped_details.client_token"], + "x_is_normally_accessed_by_rdp": ["detection.grouped_details.events.is_normally_accessed_by_rdp"], + "x_rpc_uuid": ["detection.grouped_details.uuid"], + "x_nt_referrer": ["detection.grouped_details.events.referrer"], + "x_num_events": ["detection.grouped_details.num_events"], + "x_time_duration": [ + "detection.grouped_details.duration", + "detection.grouped_details.events.duration", + "detection.grouped_details.events.sessions.duration", + "detection.grouped_details.connection_events.duration_int" + ], + "x_status_code": ["detection.grouped_details.status_code"], + "x_named_pipe": ["detection.grouped_details.named_pipe"], + "x_uri": ["detection.grouped_details.uri"], + "x_src_session_uid": ["detection.grouped_details.metadata.orig_sluid"], + "x_executed_functions": ["detection.grouped_details.executed_functions"], + "x_event_type": ["detection.grouped_details.events.event_type"], + "x_error_code": ["detection.grouped_details.events.error_code"], + "x_target_domain_refs[*].value": ["detection.grouped_details.events.target_domains"], + "x_is_external": ["detection.grouped_details.connection_events.is_external"], + "x_request_uri": ["detection.grouped_details.uri"], + "x_period_identified": ["detection.grouped_details.period_identified"], + "x_smb_share": ["detection.grouped_details.share"], + "x_account_uid": ["detection.grouped_details.account_uid"], + "extensions.'http-request-ext'.request_method": ["detection.grouped_details.events.http_method"], + "extensions.'http-request-ext'.x_response_code": ["detection.grouped_details.events.response_code"], + "extensions.'http-request-ext'.request_header.'User-Agent'": ["detection.grouped_details.user_agent"] + } + }, + "user-account": { + "fields": { + "user_id": [ + "detection.grouped_details.dst_accounts.uid", + "detection.grouped_details.src_account.name", + "detection.grouped_details.normal_users", + "detection.summary.accounts" + ], + "account_login": ["detection.grouped_details.src_account.name"], + "x_privilege_category": ["detection.grouped_details.src_account.privilege_category"], + "x_privilege_level": ["detection.grouped_details.src_account.privilege_level"] + } + }, + "x-ibm-finding": { + "fields": { + "name": ["detection.detection_type"], + "alert_id": ["detection.id"], + "description": ["detection.description", "detection.summary.description"], + "x_num_sessions": ["detection.grouped_details.num_sessions"], + "x_severity": ["detection.threat"], + "confidence": ["detection.certainty"], + "start": ["detection.first_timestamp"], + "end": ["detection.last_timestamp"], + "time_observed": ["detection.created_timestamp"], + "event_count": ["detection.summary.num_sessions", "detection.summary.num_attempts"], + "x_state": ["detection.state"], + "x_num_successes": ["detection.summary.num_successes"], + "x_assigned_to": [ + "detection.assigned_to" + ], + "x_assigned_date": [ + "detection.assigned_date" + ], + "x_sensor_name": ["detection.sensor_name"], + "x_is_triaged": ["detection.is_triaged"], + "src_ip_ref": ["detection.src_ip"], + "x_dst_ports": ["detection.summary.dst_ports"], + "x_account_refs.user_id": ["detection.summary.accounts"], + "x_shares": ["detection.summary.shares"], + "x_probable_owner": ["detection.summary.probable_owner"], + "x_matches": ["detection.summary.matches"] + } + }, + "x-ibm-ttp-tagging": { + "fields": { + "name": ["detection.detection_type"], + "confidence": ["detection.certainty"], + "kill_chain_phases.phase_name": ["detection.detection_category"] + } + }, + "x-oca-asset": { + "fields": { + "hostname": ["detection.src_host.name"], + "device_id": ["detection.src_host.id"], + "x_is_key_asset": ["detection.src_host.is_key_asset"], + "ip_refs[*].value": ["detection.src_ip"], + "x_threat": ["detection.src_host.threat"], + "x_certainty": ["detection.src_host.certainty"], + "x_privilege_category": ["detection.grouped_details.src_host.privilege_category"], + "x_privilege_level": ["detection.grouped_details.src_host.privilege_level"] + } + }, + "x-grouped-details": { + "fields": { + "first_seen": ["detection.grouped_details.first_seen"], + "last_seen": ["detection.grouped_details.last_seen"], + "detection_source": ["detection.grouped_details.detection_source"], + "detection_slug": ["detection.grouped_details.detection_slug"], + "account_ref.user_id": ["detection.grouped_details.src_account.name"], + "ja3_hashes": ["detection.grouped_details.ja3_hashes"], + "ja3s_hashes": ["detection.grouped_details.ja3s_hashes"], + "x_num_sessions": ["detection.grouped_details.num_sessions"], + "start": ["detection.grouped_details.first_timestamp"], + "end": ["detection.grouped_details.last_timestamp"], + "count": ["detection.grouped_details.count"], + "client_name": ["detection.grouped_details.client_name"], + "client_token": ["detection.grouped_details.client_token"], + "dst_byte_count": ["detection.grouped_details.bytes_received"], + "src_byte_count": ["detection.grouped_details.bytes_sent"], + "subnet": ["detection.grouped_details.subnet"], + "rpc_function_uuid": ["detection.grouped_details.uuid"], + "num_services_requested": ["detection.grouped_details.num_services_requested"], + "num_services_high_privilege": ["detection.grouped_details.num_services_high_privilege"], + "service_privilege": ["detection.grouped_details.service_privilege"] + } + }, + "x-service-accessed-info": { + "fields": { + "name": ["detection.grouped_details.service_accessed.name"], + "privilege_category": ["detection.grouped_details.service_accessed.privilege_category"], + "privilege_level": ["detection.grouped_details.service_accessed.privilege_level"] + } + }, + "x-ldap-event": { + "fields": { + "base_object": ["detection.grouped_details.events.base_object"], + "request": ["detection.grouped_details.events.request"], + "response_code": ["detection.grouped_details.events.response_code"], + "num_response_objects": ["detection.grouped_details.events.num_response_objects"], + "last_timestamp": ["detection.grouped_details.events.last_timestamp"] + } + }, + "x-sql-request-info": { + "fields": { + "http_segment": ["detection.grouped_details.targets.events.http_segment"], + "user_agent": ["detection.grouped_details.targets.events.user_agent"], + "sql_fragment": ["detection.grouped_details.targets.events.sql_fragment"], + "response_code": ["detection.grouped_details.targets.events.response_code"], + "bytes_received": ["detection.grouped_details.targets.events.bytes_received"], + "last_seen": ["detection.grouped_details.targets.events.last_seen"] + } + }, + "x-anomalous-rpc": { + "fields": { + "function_call": ["detection.grouped_details.anomalous_profiles.function_call"], + "rpc_function_uuid": ["detection.grouped_details.anomalous_profiles.function_uuid"], + "count": ["detection.grouped_details.anomalous_profiles.count"], + "start": ["detection.grouped_details.anomalous_profiles.first_timestamp"], + "end": ["detection.grouped_details.anomalous_profiles.last_timestamp"] + } + }, + "x-services-requested": { + "fields": { + "service": ["detection.grouped_details.services_requested.service"] + } + }, + "x-new-host-info": { + "fields": { + "artifact": ["detection.grouped_details.artifact"], + "via": ["detection.grouped_details.via"], + "role": ["detection.grouped_details.role"], + "end": ["detection.grouped_details.last_timestamp"] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..a664dde41 --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,1206 @@ +{ + "src_ip": [ + { + "key": "ipv4-addr.value", + "object": "source_ip" + }, + { + "key": "ipv6-addr.value", + "object": "source_ip" + }, + { + "key": "x-ibm-finding.src_ip_ref", + "object": "detection", + "references": "source_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "source_ip" + ], + "group": true + } + ], + "detection_type": [ + { + "key": "x-ibm-finding.name", + "object": "detection" + }, + { + "key": "x-ibm-ttp-tagging.name", + "object": "ttp_finding" + }, + { + "key": "x-ibm-finding.finding_type", + "object": "detection", + "value": "alert" + } + ], + "id": [ + { + "key": "x-ibm-finding.alert_id", + "object": "detection" + } + ], + "description": { + "key": "x-ibm-finding.description", + "object": "detection" + }, + "summary": { + "description": { + "key": "x-ibm-finding.description", + "object": "detection" + }, + "num_sessions": { + "key": "x-ibm-finding.event_count", + "object": "detection" + }, + "num_attempts": { + "key": "x-ibm-finding.event_count", + "object": "detection" + }, + "num_successes": { + "key": "x-ibm-finding.x_num_successes", + "object": "detection" + }, + "dst_ports": { + "key": "x-ibm-finding.x_dst_ports", + "object": "detection" + }, + "accounts": [ + { + "key": "user-account.user_id", + "object": "account", + "unwrap": true + }, + { + "key": "x-ibm-finding.x_account_refs", + "object": "detection", + "references": [ + "account" + ], + "group": true + } + ], + "shares": { + "key": "x-ibm-finding.x_shares", + "object": "detection" + }, + "probable_owner": { + "key": "x-ibm-finding.x_probable_owner", + "object": "detection" + }, + "matches": { + "key": "x-ibm-finding.x_matches", + "object": "detection" + } + }, + "threat": [ + { + "key": "x-ibm-finding.x_severity", + "object": "detection" + } + ], + "certainty": [ + { + "key": "x-ibm-finding.confidence", + "object": "detection" + }, + { + "key": "x-ibm-ttp-tagging.confidence", + "object": "ttp_finding", + "transformer": "ConvertToReal" + } + ], + "first_timestamp": { + "key": "x-ibm-finding.start", + "object": "detection" + }, + "last_timestamp": { + "key": "x-ibm-finding.end", + "object": "detection" + }, + "created_timestamp": { + "key": "x-ibm-finding.time_observed", + "object": "detection" + }, + "state": { + "key": "x-ibm-finding.x_state", + "object": "detection" + }, + "assigned_to": { + "key": "x-ibm-finding.x_assigned_to", + "object": "detection" + }, + "assigned_date": { + "key": "x-ibm-finding.x_assigned_date", + "object": "detection" + }, + "src_host": { + "name": { + "key": "x-oca-asset.hostname", + "object": "host" + }, + "id": { + "key": "x-oca-asset.device_id", + "object": "host" + }, + "is_key_asset": { + "key": "x-oca-asset.x_is_key_asset", + "object": "host" + }, + "threat": { + "key": "x-oca-asset.x_threat", + "object": "host" + }, + "certainty": { + "key": "x-oca-asset.x_certainty", + "object": "host" + } + }, + "detection_category": [ + { + "key": "x-ibm-ttp-tagging.kill_chain_phases", + "object": "ttp_finding", + "transformer": "ChainNameValue" + }, + { + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "detection", + "references": [ + "ttp_finding" + ] + } + ], + "sensor_name": { + "key": "x-ibm-finding.x_sensor_name", + "object": "detection" + }, + "is_triaged": { + "key": "x-ibm-finding.x_is_triaged", + "object": "detection" + }, + "grouped_details": { + "protocol": { + "key": "network-traffic.protocols", + "object": "nt_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "app_protocol": { + "key": "network-traffic.protocols", + "object": "nt_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "dst_protocol": { + "key": "network-traffic.protocols", + "object": "nt_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "dst_ports": { + "key": "network-traffic.dst_port", + "object": "nt_session", + "transformer": "ListToValue" + }, + "dst_ips": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt_session", + "references": "dst_ip" + } + ], + "dst_hosts": { + "ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + } + ], + "groupdstReference": { + "key": "network-traffic.dst_ref", + "object": "nt_session", + "references": "dst_ip", + "group_ref": true + } + }, + "normal_admin_hosts": { + "ip": [ + { + "key": "ipv4-addr.value", + "object": "normal_host", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "normal_host", + "unwrap": true, + "transformer": "FilterIPv4List" + } + ], + "groupNormalHostReference": { + "key": "network-traffic.x_normal_admin_host_refs", + "object": "nt_session", + "references": "normal_host", + "group_ref": true + } + }, + "src_port": { + "key": "network-traffic.src_port", + "object": "nt_session" + }, + "bytes_received": { + "key": "network-traffic.dst_byte_count", + "object": "nt_session", + "transformer": "ToInteger" + }, + "bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "nt_session", + "transformer": "ToInteger" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "nt_session" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "nt_session" + }, + "duration": { + "key": "network-traffic.x_time_duration", + "object": "nt_session" + }, + "dst_geo": { + "key": "network-traffic.x_dst_geo", + "object": "nt_session" + }, + "dst_geo_lat": { + "key": "network-traffic.x_dst_geo_latitude", + "object": "nt_session" + }, + "dst_geo_lon": { + "key": "network-traffic.x_dst_geo_longitude", + "object": "nt_session" + }, + "target_domains": [ + { + "key": "domain-name.value", + "object": "target_domain", + "unwrap": true, + "transformer": "VerifyDomainValue" + } + ], + "reason": { + "key": "network-traffic.x_reason_message", + "object": "nt_session" + }, + "num_attempts": { + "key": "network-traffic.x_num_attempts", + "object": "nt_session" + }, + "num_successes": { + "key": "network-traffic.x_num_successes", + "object": "nt_session" + }, + "user_agent": { + "key": "network-traffic.x_user_agent", + "object": "nt_session" + }, + "status_code" : { + "key": "network-traffic.x_status_code", + "object": "nt_session" + }, + "uri" : { + "key": "network-traffic.x_request_uri", + "object": "nt_session" + }, + "metadata" : { + "orig_sluid": { + "key": "network-traffic.x_src_session_uid", + "object": "nt_session" + } + }, + "origin_domain": { + "key": "domain-name.value", + "object": "origin_domain" + }, + "origin_ip": [ + { + "key": "ipv4-addr.value", + "object": "origin_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "origin_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "network-traffic.dst_ref", + "object": "origin_nt", + "references": "origin_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "origin_nt", + "references": "source_ip" + }, + { + "key": "domain-name.resolves_to_refs", + "object": "origin_domain", + "references": [ + "origin_ip" + ] + } + ], + "origin_port": { + "key": "network-traffic.dst_port", + "object": "origin_nt", + "transformer": "ListToValue" + }, + "origin_protocol": { + "key": "network-traffic.protocols", + "object": "origin_nt", + "group": true, + "transformer": "ToLowercaseArray" + }, + "origin_geo_lat": { + "key": "network-traffic.x_dst_geo_latitude", + "object": "origin_nt" + }, + "origin_geo_lon": { + "key": "network-traffic.x_dst_geo_longitude", + "object": "origin_nt" + }, + "origin_geo": { + "key": "network-traffic.x_dst_geo", + "object": "origin_nt" + }, + "num_accounts": { + "key": "network-traffic.x_num_accounts", + "object": "nt_session" + }, + "num_response_objects": { + "key": "network-traffic.x_num_response_objects", + "object": "nt_session" + }, + "client_name": { + "key": "network-traffic.x_client_name", + "object": "nt_session" + }, + "client_token": { + "key": "network-traffic.x_client_token", + "object": "nt_session" + }, + "uuid": { + "key": "network-traffic.x_rpc_uuid", + "object": "nt_session" + }, + "named_pipe": { + "key": "network-traffic.x_named_pipe", + "object": "nt_session" + }, + "executed_functions": { + "key": "network-traffic.x_executed_functions", + "object": "nt_session" + }, + "normal_users": [ + { + "key": "user-account.user_id", + "object": "user", + "unwrap": true + }, + { + "key": "network-traffic.x_normal_user_refs", + "object": "nt_session", + "references": ["user"] + } + ], + "num_events": { + "key": "network-traffic.x_num_events", + "object": "nt_session" + }, + "num_sessions": { + "key": "network-traffic.x_num_sessions", + "object": "nt_session" + }, + "first_seen": { + "key": "x-grouped-details.first_seen", + "object": "group_detail" + }, + "last_seen": { + "key": "x-grouped-details.last_seen", + "object": "group_detail" + }, + "detection_source": { + "key": "x-grouped-details.detection_source", + "object": "group_detail" + }, + "detection_slug": { + "key": "x-grouped-details.detection_slug", + "object": "group_detail" + }, + "period_identified": { + "key": "network-traffic.x_period_identified", + "object": "nt_session" + }, + "share": { + "key": "network-traffic.x_smb_share", + "object": "nt_session" + }, + "account_uid" : { + "key": "network-traffic.x_account_uid", + "object": "nt_session" + }, + "src_account" : { + "name": [ + { + "key": "user-account.user_id", + "object": "user_account" + }, + { + "key": "user-account.account_login", + "object": "user_account" + }, + { + "key": "x-grouped-details.account_ref", + "object": "group_detail", + "references": "user_account" + }, + { + "key": "x-ibm-finding.src_os_user_ref", + "object": "detection", + "references": "user_account" + } + ], + "privilege_category": { + "key": "user-account.x_privilege_category", + "object": "user_account" + }, + "privilege_level": { + "key": "user-account.x_privilege_level", + "object": "user_account" + } + }, + "service_accessed": { + "name": [ + { + "key": "x-service-accessed-info.name", + "object": "service_info" + }, + { + "key": "x-grouped-details.service_accessed_info_ref", + "object": "group_detail", + "references": "service_info" + } + ], + "privilege_category": { + "key": "x-service-accessed-info.privilege_category", + "object": "service_info" + }, + "privilege_level": { + "key": "x-service-accessed-info.privilege_level", + "object": "service_info" + } + }, + "src_host":{ + "privilege_category" : { + "key": "x-oca-asset.x_privilege_category", + "object": "host" + }, + "privilege_level" : { + "key": "x-oca-asset.x_privilege_level", + "object": "host" + } + }, + "anomalous_profiles": { + "function_call": { + "key": "x-anomalous-rpc.function_call", + "object": "a_profile" + }, + "function_uuid": { + "key": "x-anomalous-rpc.rpc_function_uuid", + "object": "a_profile" + }, + "count": { + "key": "x-anomalous-rpc.count", + "object": "a_profile" + }, + "first_timestamp": { + "key": "x-anomalous-rpc.start", + "object": "a_profile" + }, + "last_timestamp": { + "key": "x-anomalous-rpc.end", + "object": "a_profile" + } + }, + "events": { + "base_object": { + "key": "x-ldap-event.base_object", + "object": "event" + }, + "request": { + "key": "x-ldap-event.request", + "object": "event" + }, + "response_code": { + "key": "x-ldap-event.response_code", + "object": "event" + }, + "num_response_objects": { + "key": "x-ldap-event.num_response_objects", + "object": "event" + }, + "last_timestamp": { + "key": "x-ldap-event.last_timestamp", + "object": "event" + } + }, + "targets": { + "events": { + "http_segment": { + "key": "x-sql-request-info.http_segment", + "object": "sql_request" + }, + "user_agent": { + "key": "x-sql-request-info.user_agent", + "object": "sql_request" + }, + "sql_fragment": { + "key": "x-sql-request-info.sql_fragment", + "object": "sql_request" + }, + "response_code": { + "key": "x-sql-request-info.response_code", + "object": "sql_request" + }, + "bytes_received": { + "key": "x-sql-request-info.bytes_received", + "object": "sql_request" + }, + "last_seen": { + "key": "x-sql-request-info.last_seen", + "object": "sql_request" + } + } + }, + "groupProfileReference": { + "key": "network-traffic.x_anomalous_rpc_refs", + "object": "nt_session", + "references": [ + "a_profile" + ], + "group_ref": true + }, + "groupEventReference": { + "key": "x-network-traffic.x_ldap_event_refs", + "object": "nt_session", + "references": [ + "event" + ], + "group_ref": true + }, + "groupSQLReferences": { + "key": "network-traffic.x_sql_request_info_refs", + "object": "nt_session", + "references": [ + "sql_request" + ], + "group_ref": true + }, + "groupIocReference": { + "key": "x-ibm-finding.ioc_refs", + "object": "detection", + "references": [ + "source_ip", + "dst_ip", + "origin_ip", + "target_domain", + "normal_host" + ], + "group_ref": true + } + }, + "grouped_details_ex": { + "num_sessions": { + "key": "x-grouped-details.num_sessions", + "object": "group_detail" + }, + "ja3_hashes": { + "key": "x-grouped-details.ja3_hashes", + "object": "group_detail" + }, + "ja3s_hashes": { + "key": "x-grouped-details.ja3s_hashes", + "object": "group_detail" + }, + "first_timestamp": { + "key": "x-grouped-details.start", + "object": "group_detail" + }, + "last_timestamp": { + "key": "x-grouped-details.end", + "object": "group_detail" + }, + "count": { + "key": "x-grouped-details.count", + "object": "group_detail" + }, + "client_name": { + "key": "x-grouped-details.client_name", + "object": "group_detail" + }, + "client_token": { + "key": "x-grouped-details.client_token", + "object": "group_detail" + }, + "bytes_received": { + "key": "x-grouped-details.dst_byte_count", + "object": "group_detail", + "transformer": "ToInteger" + }, + "bytes_sent": { + "key": "x-grouped-details.src_byte_count", + "object": "group_detail", + "transformer": "ToInteger" + }, + "subnet": { + "key": "x-grouped-details.subnet", + "object": "group_detail" + }, + "uuid": { + "key": "x-grouped-details.rpc_function_uuid", + "object": "group_detail" + }, + "num_services_requested": { + "key": "x-grouped-details.num_services_requested", + "object": "group_detail" + }, + "num_services_high_privilege": { + "key": "x-grouped-details.num_services_high_privilege", + "object": "group_detail" + }, + "service_privilege": { + "key": "x-grouped-details.service_privilege", + "object": "group_detail" + }, + "services_requested": { + "service": { + "key": "x-services-requested.service", + "object": "service" + }, + "privilege": { + "key": "x-services-requested.privilege", + "object": "service" + }, + "groupServiceReference": { + "key": "x-grouped-details.service_refs", + "object": "group_detail", + "references": [ + "service" + ], + "group_ref": true + } + }, + "dst_accounts": { + "uid": { + "key": "user-account.user_id", + "object": "dst_account" + }, + "groupServiceReference": { + "key": "x-grouped-details.dst_account_refs", + "object": "group_detail", + "references": [ + "dst_account" + ], + "group_ref": true + } + }, + "dst_hosts": { + "protocol": { + "key": "network-traffic.protocols", + "object": "nt_host", + "group": true, + "transformer": "ToLowercaseArray" + }, + "dst_port": { + "key": "network-traffic.dst_port", + "object": "nt_host", + "transformer": "ListToValue" + }, + "dst_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt_host", + "references": "dst_ip" + } + ], + "last_timestamp": { + "key": "network-traffic.end", + "object": "nt_host" + }, + "group_nt_Reference": { + "key": "x-grouped-details.host_network_refs", + "object": "group_detail", + "references": [ + "nt_host" + ], + "group_ref": true + } + }, + "events": { + "bytes_received": { + "key": "network-traffic.dst_byte_count", + "object": "event" + }, + "bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "event" + }, + "count": { + "key": "network-traffic.x_count", + "object": "event" + }, + "dst_country": { + "key": "network-traffic.x_dst_country", + "object": "event" + }, + "dst_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "event", + "references": "dst_ip" + } + ], + "dst_ips": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "event", + "references": "dst_ip" + } + ], + "dst_ports": { + "key": "network-traffic.dst_port", + "object": "event", + "transformer": "ListToValue" + }, + "duration": { + "key": "network-traffic.x_time_duration", + "object": "event" + }, + "error_code": { + "key": "network-traffic.x_error_code", + "object": "event" + }, + "event_type": { + "key": "network-traffic.x_event_type", + "object": "event" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "event" + }, + "http_method": { + "key": "network-traffic.extensions.http-request-ext.request_method", + "object": "event" + }, + "is_normally_accessed_by_rdp": { + "key": "network-traffic.x_is_normally_accessed_by_rdp", + "object": "event" + }, + "last_seen": { + "key": "network-traffic.end", + "object": "event" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "event" + }, + "num_attempts": { + "key": "network-traffic.x_num_attempts", + "object": "event" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "event", + "group": true, + "transformer": "ToLowercaseArray" + }, + "referrer": { + "key": "network-traffic.x_nt_referrer", + "object": "event" + }, + "request": { + "key": "network-traffic.extensions.http-request-ext.request_value", + "object": "event" + }, + "response_code": { + "key": "network-traffic.extensions.http-request-ext.x_response_code", + "object": "event" + }, + "target_domains": [ + { + "key": "domain-name.value", + "object": "target_domain", + "unwrap": true + }, + { + "key": "network-traffic.x_target_domain_refs", + "object": "event", + "references": ["target_domain"] + } + ], + "sessions": { + "app_protocol": { + "key": "network-traffic.protocols", + "object": "event_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "bytes_received": { + "key": "network-traffic.dst_byte_count", + "object": "event_session" + }, + "dst_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "event_session", + "references": "dst_ip" + } + ], + "dst_port": { + "key": "network-traffic.dst_port", + "object": "event_session" + }, + "duration": { + "key": "network-traffic.x_time_duration", + "object": "event_session" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "event_session" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "event_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "groupSessionReference": { + "key": "x-network-traffic.x_session_refs", + "object": "event", + "references": [ + "event_session" + ], + "group_ref": true + } + }, + "target_summary": { + "app_protocol": { + "key": "network-traffic.protocols", + "object": "event", + "group": true, + "transformer": "ToLowercaseArray" + }, + "bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "event" + }, + "dst_port": { + "key": "network-traffic.dst_port", + "object": "event" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "event" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "event" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "event", + "group": true, + "transformer": "ToLowercaseArray" + } + }, + "url": { + "key": "url.value", + "object": "url" + }, + "user_agent": { + "key": "network-traffic.extensions.http-request-ext.request_header.User-Agent", + "object": "event_session" + }, + "groupEventReference": { + "key": "x-grouped-details.event_refs", + "object": "group_detail", + "references": [ + "event" + ], + "group_ref": true + } + }, + "sessions": { + "tunnel_type": { + "key": "network-traffic.x_tunnel_type", + "object": "session" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "app_protocol": { + "key": "network-traffic.protocols", + "object": "session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "dst_port": { + "key": "network-traffic.dst_port", + "object": "session", + "transformer": "ListToValue" + }, + "dst_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "network-traffic.dst_ref", + "object": "session", + "references": "dst_ip" + } + ], + "bytes_received": { + "key": "network-traffic.dst_byte_count", + "object": "session", + "transformer": "ToInteger" + }, + "bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "session", + "transformer": "ToInteger" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "session" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "session" + }, + "dst_geo": { + "key": "network-traffic.x_dst_geo", + "object": "session" + }, + "dst_geo_lat": { + "key": "network-traffic.x_dst_geo_latitude", + "object": "session" + }, + "dst_geo_lon": { + "key": "network-traffic.x_dst_geo_longitude", + "object": "session" + }, + "groupSessionReference": { + "key": "x-grouped-details.session_refs", + "object": "group_detail", + "references": [ + "session" + ], + "group_ref": true + } + }, + "connection_events": { + "dst_port": { + "key": "network-traffic.dst_port", + "object": "con_event", + "transformer": "ListToValue" + }, + "duration_int": { + "key": "network-traffic.x_time_duration", + "object": "con_event" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "con_event" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "con_event" + }, + "is_external": { + "key": "network-traffic.x_is_external", + "object": "con_event" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "con_event", + "group": true, + "transformer": "ToLowercaseArray" + }, + "target_host": { + "dst_dns": [ + { + "key": "domain-name.value", + "object": "dns_name" + }, + { + "key": "domain-name.resolves_to_refs", + "object": "dns_name", + "references": ["dst_ip"] + } + ], + "ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "network-traffic.dst_ref", + "object": "con_event", + "references": "dst_ip" + } + ] + }, + "total_bytes_rcvd": { + "key": "network-traffic.dst_byte_count", + "object": "con_event", + "transformer": "ToInteger" + }, + "total_bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "con_event", + "transformer": "ToInteger" + }, + "groupConEventsReference": { + "key": "x-grouped-details.connection_event_refs", + "object": "group_detail", + "references": [ + "con_event" + ], + "group_ref": true + } + }, + "groupIOCReference": { + "key": "x-ibm-finding.ioc_refs", + "object": "detection", + "references": [ + "source_ip", + "target_domain", + "dst_ip" + ], + "group_ref": true + } + }, + "grouped_details_info": { + "artifact": { + "key": "x-new-host-info.artifact", + "object": "new_host" + }, + "via": { + "key": "x-new-host-info.via", + "object": "new_host" + }, + "role": { + "key": "x-new-host-info.role", + "object": "new_host" + }, + "last_timestamp": { + "key": "x-new-host-info.end", + "object": "new_host" + }, + "groupNewHostReferences": { + "key": "x-ibm-finding.x_new_host_info_refs", + "object": "detection", + "references": [ + "new_host" + ], + "group_ref": true + }, + "groupIocReference": { + "key": "x-ibm-finding.ioc_refs", + "object": "detection", + "references": [ + "source_ip" + ], + "group_ref": true + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json new file mode 100644 index 000000000..2dfb1d994 --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json @@ -0,0 +1,1206 @@ +{ + "src_ip": [ + { + "key": "ipv4-addr.value", + "object": "source_ip" + }, + { + "key": "ipv6-addr.value", + "object": "source_ip" + }, + { + "key": "x-ibm-finding.src_ip_ref", + "object": "detection", + "references": "source_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "source_ip" + ], + "group": true + } + ], + "detection_type": [ + { + "key": "x-ibm-finding.name", + "object": "detection" + }, + { + "key": "x-ibm-ttp-tagging.name", + "object": "ttp_finding" + }, + { + "key": "x-ibm-finding.finding_type", + "object": "detection", + "value": "alert" + } + ], + "id": [ + { + "key": "x-ibm-finding.alert_id", + "object": "detection" + } + ], + "description": { + "key": "x-ibm-finding.description", + "object": "detection" + }, + "summary": { + "description": { + "key": "x-ibm-finding.description", + "object": "detection" + }, + "num_sessions": { + "key": "x-ibm-finding.event_count", + "object": "detection" + }, + "num_attempts": { + "key": "x-ibm-finding.event_count", + "object": "detection" + }, + "num_successes": { + "key": "x-ibm-finding.x_num_successes", + "object": "detection" + }, + "dst_ports": { + "key": "x-ibm-finding.x_dst_ports", + "object": "detection" + }, + "accounts": [ + { + "key": "user-account.user_id", + "object": "account", + "unwrap": true + }, + { + "key": "x-ibm-finding.x_account_refs", + "object": "detection", + "references": [ + "account" + ], + "group": true + } + ], + "shares": { + "key": "x-ibm-finding.x_shares", + "object": "detection" + }, + "probable_owner": { + "key": "x-ibm-finding.x_probable_owner", + "object": "detection" + }, + "matches": { + "key": "x-ibm-finding.x_matches", + "object": "detection" + } + }, + "threat": [ + { + "key": "x-ibm-finding.severity", + "object": "detection" + } + ], + "certainty": [ + { + "key": "x-ibm-finding.confidence", + "object": "detection" + }, + { + "key": "x-ibm-ttp-tagging.confidence", + "object": "ttp_finding", + "transformer": "ConvertToReal" + } + ], + "first_timestamp": { + "key": "x-ibm-finding.start", + "object": "detection" + }, + "last_timestamp": { + "key": "x-ibm-finding.end", + "object": "detection" + }, + "created_timestamp": { + "key": "x-ibm-finding.time_observed", + "object": "detection" + }, + "state": { + "key": "x-ibm-finding.x_state", + "object": "detection" + }, + "assigned_to": { + "key": "x-ibm-finding.x_assigned_to", + "object": "detection" + }, + "assigned_date": { + "key": "x-ibm-finding.x_assigned_date", + "object": "detection" + }, + "src_host": { + "name": { + "key": "x-oca-asset.hostname", + "object": "host" + }, + "id": { + "key": "x-oca-asset.device_id", + "object": "host" + }, + "is_key_asset": { + "key": "x-oca-asset.x_is_key_asset", + "object": "host" + }, + "threat": { + "key": "x-oca-asset.x_threat", + "object": "host" + }, + "certainty": { + "key": "x-oca-asset.x_certainty", + "object": "host" + } + }, + "detection_category": [ + { + "key": "x-ibm-ttp-tagging.kill_chain_phases", + "object": "ttp_finding", + "transformer": "ChainNameValue" + }, + { + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "detection", + "references": [ + "ttp_finding" + ] + } + ], + "sensor_name": { + "key": "x-ibm-finding.x_sensor_name", + "object": "detection" + }, + "is_triaged": { + "key": "x-ibm-finding.x_is_triaged", + "object": "detection" + }, + "grouped_details": { + "protocol": { + "key": "network-traffic.protocols", + "object": "nt_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "app_protocol": { + "key": "network-traffic.protocols", + "object": "nt_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "dst_protocol": { + "key": "network-traffic.protocols", + "object": "nt_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "dst_ports": { + "key": "network-traffic.dst_port", + "object": "nt_session", + "transformer": "ListToValue" + }, + "dst_ips": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt_session", + "references": "dst_ip" + } + ], + "dst_hosts": { + "ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + } + ], + "groupdstReference": { + "key": "network-traffic.dst_ref", + "object": "nt_session", + "references": "dst_ip", + "group_ref": true + } + }, + "normal_admin_hosts": { + "ip": [ + { + "key": "ipv4-addr.value", + "object": "normal_host", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "normal_host", + "unwrap": true, + "transformer": "FilterIPv4List" + } + ], + "groupNormalHostReference": { + "key": "network-traffic.x_normal_admin_host_refs", + "object": "nt_session", + "references": "normal_host", + "group_ref": true + } + }, + "src_port": { + "key": "network-traffic.src_port", + "object": "nt_session" + }, + "bytes_received": { + "key": "network-traffic.dst_byte_count", + "object": "nt_session", + "transformer": "ToInteger" + }, + "bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "nt_session", + "transformer": "ToInteger" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "nt_session" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "nt_session" + }, + "duration": { + "key": "network-traffic.x_time_duration", + "object": "nt_session" + }, + "dst_geo": { + "key": "network-traffic.x_dst_geo", + "object": "nt_session" + }, + "dst_geo_lat": { + "key": "network-traffic.x_dst_geo_latitude", + "object": "nt_session" + }, + "dst_geo_lon": { + "key": "network-traffic.x_dst_geo_longitude", + "object": "nt_session" + }, + "target_domains": [ + { + "key": "domain-name.value", + "object": "target_domain", + "unwrap": true, + "transformer": "VerifyDomainValue" + } + ], + "reason": { + "key": "network-traffic.x_reason_message", + "object": "nt_session" + }, + "num_attempts": { + "key": "network-traffic.x_num_attempts", + "object": "nt_session" + }, + "num_successes": { + "key": "network-traffic.x_num_successes", + "object": "nt_session" + }, + "user_agent": { + "key": "network-traffic.x_user_agent", + "object": "nt_session" + }, + "status_code" : { + "key": "network-traffic.x_status_code", + "object": "nt_session" + }, + "uri" : { + "key": "network-traffic.x_request_uri", + "object": "nt_session" + }, + "metadata" : { + "orig_sluid": { + "key": "network-traffic.x_src_session_uid", + "object": "nt_session" + } + }, + "origin_domain": { + "key": "domain-name.value", + "object": "origin_domain" + }, + "origin_ip": [ + { + "key": "ipv4-addr.value", + "object": "origin_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "origin_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "network-traffic.dst_ref", + "object": "origin_nt", + "references": "origin_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "origin_nt", + "references": "source_ip" + }, + { + "key": "domain-name.resolves_to_refs", + "object": "origin_domain", + "references": [ + "origin_ip" + ] + } + ], + "origin_port": { + "key": "network-traffic.dst_port", + "object": "origin_nt", + "transformer": "ListToValue" + }, + "origin_protocol": { + "key": "network-traffic.protocols", + "object": "origin_nt", + "group": true, + "transformer": "ToLowercaseArray" + }, + "origin_geo_lat": { + "key": "network-traffic.x_dst_geo_latitude", + "object": "origin_nt" + }, + "origin_geo_lon": { + "key": "network-traffic.x_dst_geo_longitude", + "object": "origin_nt" + }, + "origin_geo": { + "key": "network-traffic.x_dst_geo", + "object": "origin_nt" + }, + "num_accounts": { + "key": "network-traffic.x_num_accounts", + "object": "nt_session" + }, + "num_response_objects": { + "key": "network-traffic.x_num_response_objects", + "object": "nt_session" + }, + "client_name": { + "key": "network-traffic.x_client_name", + "object": "nt_session" + }, + "client_token": { + "key": "network-traffic.x_client_token", + "object": "nt_session" + }, + "uuid": { + "key": "network-traffic.x_rpc_uuid", + "object": "nt_session" + }, + "named_pipe": { + "key": "network-traffic.x_named_pipe", + "object": "nt_session" + }, + "executed_functions": { + "key": "network-traffic.x_executed_functions", + "object": "nt_session" + }, + "normal_users": [ + { + "key": "user-account.user_id", + "object": "user", + "unwrap": true + }, + { + "key": "network-traffic.x_normal_user_refs", + "object": "nt_session", + "references": ["user"] + } + ], + "num_events": { + "key": "network-traffic.x_num_events", + "object": "nt_session" + }, + "num_sessions": { + "key": "network-traffic.x_num_sessions", + "object": "nt_session" + }, + "first_seen": { + "key": "x-grouped-details.first_seen", + "object": "group_detail" + }, + "last_seen": { + "key": "x-grouped-details.last_seen", + "object": "group_detail" + }, + "detection_source": { + "key": "x-grouped-details.detection_source", + "object": "group_detail" + }, + "detection_slug": { + "key": "x-grouped-details.detection_slug", + "object": "group_detail" + }, + "period_identified": { + "key": "network-traffic.x_period_identified", + "object": "nt_session" + }, + "share": { + "key": "network-traffic.x_smb_share", + "object": "nt_session" + }, + "account_uid" : { + "key": "network-traffic.x_account_uid", + "object": "nt_session" + }, + "src_account" : { + "name": [ + { + "key": "user-account.user_id", + "object": "user_account" + }, + { + "key": "user-account.account_login", + "object": "user_account" + }, + { + "key": "x-grouped-details.account_ref", + "object": "group_detail", + "references": "user_account" + }, + { + "key": "x-ibm-finding.src_os_user_ref", + "object": "detection", + "references": "user_account" + } + ], + "privilege_category": { + "key": "user-account.x_privilege_category", + "object": "user_account" + }, + "privilege_level": { + "key": "user-account.x_privilege_level", + "object": "user_account" + } + }, + "service_accessed": { + "name": [ + { + "key": "x-service-accessed-info.name", + "object": "service_info" + }, + { + "key": "x-grouped-details.service_accessed_info_ref", + "object": "group_detail", + "references": "service_info" + } + ], + "privilege_category": { + "key": "x-service-accessed-info.privilege_category", + "object": "service_info" + }, + "privilege_level": { + "key": "x-service-accessed-info.privilege_level", + "object": "service_info" + } + }, + "src_host":{ + "privilege_category" : { + "key": "x-oca-asset.x_privilege_category", + "object": "host" + }, + "privilege_level" : { + "key": "x-oca-asset.x_privilege_level", + "object": "host" + } + }, + "anomalous_profiles": { + "function_call": { + "key": "x-anomalous-rpc.function_call", + "object": "a_profile" + }, + "function_uuid": { + "key": "x-anomalous-rpc.rpc_function_uuid", + "object": "a_profile" + }, + "count": { + "key": "x-anomalous-rpc.count", + "object": "a_profile" + }, + "first_timestamp": { + "key": "x-anomalous-rpc.start", + "object": "a_profile" + }, + "last_timestamp": { + "key": "x-anomalous-rpc.end", + "object": "a_profile" + } + }, + "events": { + "base_object": { + "key": "x-ldap-event.base_object", + "object": "event" + }, + "request": { + "key": "x-ldap-event.request", + "object": "event" + }, + "response_code": { + "key": "x-ldap-event.response_code", + "object": "event" + }, + "num_response_objects": { + "key": "x-ldap-event.num_response_objects", + "object": "event" + }, + "last_timestamp": { + "key": "x-ldap-event.last_timestamp", + "object": "event" + } + }, + "targets": { + "events": { + "http_segment": { + "key": "x-sql-request-info.http_segment", + "object": "sql_request" + }, + "user_agent": { + "key": "x-sql-request-info.user_agent", + "object": "sql_request" + }, + "sql_fragment": { + "key": "x-sql-request-info.sql_fragment", + "object": "sql_request" + }, + "response_code": { + "key": "x-sql-request-info.response_code", + "object": "sql_request" + }, + "bytes_received": { + "key": "x-sql-request-info.bytes_received", + "object": "sql_request" + }, + "last_seen": { + "key": "x-sql-request-info.last_seen", + "object": "sql_request" + } + } + }, + "groupProfileReference": { + "key": "network-traffic.x_anomalous_rpc_refs", + "object": "nt_session", + "references": [ + "a_profile" + ], + "group_ref": true + }, + "groupEventReference": { + "key": "x-network-traffic.x_ldap_event_refs", + "object": "nt_session", + "references": [ + "event" + ], + "group_ref": true + }, + "groupSQLReferences": { + "key": "network-traffic.x_sql_request_info_refs", + "object": "nt_session", + "references": [ + "sql_request" + ], + "group_ref": true + }, + "groupIocReference": { + "key": "x-ibm-finding.ioc_refs", + "object": "detection", + "references": [ + "source_ip", + "dst_ip", + "origin_ip", + "target_domain", + "normal_host" + ], + "group_ref": true + } + }, + "grouped_details_ex": { + "num_sessions": { + "key": "x-grouped-details.num_sessions", + "object": "group_detail" + }, + "ja3_hashes": { + "key": "x-grouped-details.ja3_hashes", + "object": "group_detail" + }, + "ja3s_hashes": { + "key": "x-grouped-details.ja3s_hashes", + "object": "group_detail" + }, + "first_timestamp": { + "key": "x-grouped-details.start", + "object": "group_detail" + }, + "last_timestamp": { + "key": "x-grouped-details.end", + "object": "group_detail" + }, + "count": { + "key": "x-grouped-details.count", + "object": "group_detail" + }, + "client_name": { + "key": "x-grouped-details.client_name", + "object": "group_detail" + }, + "client_token": { + "key": "x-grouped-details.client_token", + "object": "group_detail" + }, + "bytes_received": { + "key": "x-grouped-details.dst_byte_count", + "object": "group_detail", + "transformer": "ToInteger" + }, + "bytes_sent": { + "key": "x-grouped-details.src_byte_count", + "object": "group_detail", + "transformer": "ToInteger" + }, + "subnet": { + "key": "x-grouped-details.subnet", + "object": "group_detail" + }, + "uuid": { + "key": "x-grouped-details.rpc_function_uuid", + "object": "group_detail" + }, + "num_services_requested": { + "key": "x-grouped-details.num_services_requested", + "object": "group_detail" + }, + "num_services_high_privilege": { + "key": "x-grouped-details.num_services_high_privilege", + "object": "group_detail" + }, + "service_privilege": { + "key": "x-grouped-details.service_privilege", + "object": "group_detail" + }, + "services_requested": { + "service": { + "key": "x-services-requested.service", + "object": "service" + }, + "privilege": { + "key": "x-services-requested.privilege", + "object": "service" + }, + "groupServiceReference": { + "key": "x-grouped-details.service_refs", + "object": "group_detail", + "references": [ + "service" + ], + "group_ref": true + } + }, + "dst_accounts": { + "uid": { + "key": "user-account.user_id", + "object": "dst_account" + }, + "groupServiceReference": { + "key": "x-grouped-details.dst_account_refs", + "object": "group_detail", + "references": [ + "dst_account" + ], + "group_ref": true + } + }, + "dst_hosts": { + "protocol": { + "key": "network-traffic.protocols", + "object": "nt_host", + "group": true, + "transformer": "ToLowercaseArray" + }, + "dst_port": { + "key": "network-traffic.dst_port", + "object": "nt_host", + "transformer": "ListToValue" + }, + "dst_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt_host", + "references": "dst_ip" + } + ], + "last_timestamp": { + "key": "network-traffic.end", + "object": "nt_host" + }, + "group_nt_Reference": { + "key": "x-grouped-details.host_network_refs", + "object": "group_detail", + "references": [ + "nt_host" + ], + "group_ref": true + } + }, + "events": { + "bytes_received": { + "key": "network-traffic.dst_byte_count", + "object": "event" + }, + "bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "event" + }, + "count": { + "key": "network-traffic.x_count", + "object": "event" + }, + "dst_country": { + "key": "network-traffic.x_dst_country", + "object": "event" + }, + "dst_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "event", + "references": "dst_ip" + } + ], + "dst_ips": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "event", + "references": "dst_ip" + } + ], + "dst_ports": { + "key": "network-traffic.dst_port", + "object": "event", + "transformer": "ListToValue" + }, + "duration": { + "key": "network-traffic.x_time_duration", + "object": "event" + }, + "error_code": { + "key": "network-traffic.x_error_code", + "object": "event" + }, + "event_type": { + "key": "network-traffic.x_event_type", + "object": "event" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "event" + }, + "http_method": { + "key": "network-traffic.extensions.http-request-ext.request_method", + "object": "event" + }, + "is_normally_accessed_by_rdp": { + "key": "network-traffic.x_is_normally_accessed_by_rdp", + "object": "event" + }, + "last_seen": { + "key": "network-traffic.end", + "object": "event" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "event" + }, + "num_attempts": { + "key": "network-traffic.x_num_attempts", + "object": "event" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "event", + "group": true, + "transformer": "ToLowercaseArray" + }, + "referrer": { + "key": "network-traffic.x_nt_referrer", + "object": "event" + }, + "request": { + "key": "network-traffic.extensions.http-request-ext.request_value", + "object": "event" + }, + "response_code": { + "key": "network-traffic.extensions.http-request-ext.x_response_code", + "object": "event" + }, + "target_domains": [ + { + "key": "domain-name.value", + "object": "target_domain", + "unwrap": true + }, + { + "key": "network-traffic.x_target_domain_refs", + "object": "event", + "references": ["target_domain"] + } + ], + "sessions": { + "app_protocol": { + "key": "network-traffic.protocols", + "object": "event_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "bytes_received": { + "key": "network-traffic.dst_byte_count", + "object": "event_session" + }, + "dst_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "event_session", + "references": "dst_ip" + } + ], + "dst_port": { + "key": "network-traffic.dst_port", + "object": "event_session" + }, + "duration": { + "key": "network-traffic.x_time_duration", + "object": "event_session" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "event_session" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "event_session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "groupSessionReference": { + "key": "x-network-traffic.x_session_refs", + "object": "event", + "references": [ + "event_session" + ], + "group_ref": true + } + }, + "target_summary": { + "app_protocol": { + "key": "network-traffic.protocols", + "object": "event", + "group": true, + "transformer": "ToLowercaseArray" + }, + "bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "event" + }, + "dst_port": { + "key": "network-traffic.dst_port", + "object": "event" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "event" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "event" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "event", + "group": true, + "transformer": "ToLowercaseArray" + } + }, + "url": { + "key": "url.value", + "object": "url" + }, + "user_agent": { + "key": "network-traffic.extensions.http-request-ext.request_header.User-Agent", + "object": "event_session" + }, + "groupEventReference": { + "key": "x-grouped-details.event_refs", + "object": "group_detail", + "references": [ + "event" + ], + "group_ref": true + } + }, + "sessions": { + "tunnel_type": { + "key": "network-traffic.x_tunnel_type", + "object": "session" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "app_protocol": { + "key": "network-traffic.protocols", + "object": "session", + "group": true, + "transformer": "ToLowercaseArray" + }, + "dst_port": { + "key": "network-traffic.dst_port", + "object": "session", + "transformer": "ListToValue" + }, + "dst_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "network-traffic.dst_ref", + "object": "session", + "references": "dst_ip" + } + ], + "bytes_received": { + "key": "network-traffic.dst_byte_count", + "object": "session", + "transformer": "ToInteger" + }, + "bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "session", + "transformer": "ToInteger" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "session" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "session" + }, + "dst_geo": { + "key": "network-traffic.x_dst_geo", + "object": "session" + }, + "dst_geo_lat": { + "key": "network-traffic.x_dst_geo_latitude", + "object": "session" + }, + "dst_geo_lon": { + "key": "network-traffic.x_dst_geo_longitude", + "object": "session" + }, + "groupSessionReference": { + "key": "x-grouped-details.session_refs", + "object": "group_detail", + "references": [ + "session" + ], + "group_ref": true + } + }, + "connection_events": { + "dst_port": { + "key": "network-traffic.dst_port", + "object": "con_event", + "transformer": "ListToValue" + }, + "duration_int": { + "key": "network-traffic.x_time_duration", + "object": "con_event" + }, + "first_timestamp": { + "key": "network-traffic.start", + "object": "con_event" + }, + "last_timestamp": { + "key": "network-traffic.end", + "object": "con_event" + }, + "is_external": { + "key": "network-traffic.x_is_external", + "object": "con_event" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "con_event", + "group": true, + "transformer": "ToLowercaseArray" + }, + "target_host": { + "dst_dns": [ + { + "key": "domain-name.value", + "object": "dns_name" + }, + { + "key": "domain-name.resolves_to_refs", + "object": "dns_name", + "references": ["dst_ip"] + } + ], + "ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "network-traffic.dst_ref", + "object": "con_event", + "references": "dst_ip" + } + ] + }, + "total_bytes_rcvd": { + "key": "network-traffic.dst_byte_count", + "object": "con_event", + "transformer": "ToInteger" + }, + "total_bytes_sent": { + "key": "network-traffic.src_byte_count", + "object": "con_event", + "transformer": "ToInteger" + }, + "groupConEventsReference": { + "key": "x-grouped-details.connection_event_refs", + "object": "group_detail", + "references": [ + "con_event" + ], + "group_ref": true + } + }, + "groupIOCReference": { + "key": "x-ibm-finding.ioc_refs", + "object": "detection", + "references": [ + "source_ip", + "target_domain", + "dst_ip" + ], + "group_ref": true + } + }, + "grouped_details_info": { + "artifact": { + "key": "x-new-host-info.artifact", + "object": "new_host" + }, + "via": { + "key": "x-new-host-info.via", + "object": "new_host" + }, + "role": { + "key": "x-new-host-info.role", + "object": "new_host" + }, + "last_timestamp": { + "key": "x-new-host-info.end", + "object": "new_host" + }, + "groupNewHostReferences": { + "key": "x-ibm-finding.x_new_host_info_refs", + "object": "detection", + "references": [ + "new_host" + ], + "group_ref": true + }, + "groupIocReference": { + "key": "x-ibm-finding.ioc_refs", + "object": "detection", + "references": [ + "source_ip" + ], + "group_ref": true + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/vectra/stix_translation/query_constructor.py b/stix_shifter_modules/vectra/stix_translation/query_constructor.py new file mode 100644 index 000000000..530898b6f --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/query_constructor.py @@ -0,0 +1,678 @@ +import json +import re +from os import path +from pyparsing import nestedExpr, White +from datetime import datetime, timedelta +from stix_shifter_utils.utils import logger +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, \ + ComparisonExpression, ComparisonComparators, Pattern, CombinedComparisonExpression, CombinedObservationExpression + +logger = logger.set_logger(__name__) + +START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)" +STOP_TIME = datetime.utcnow() +DEFAULT_LIMIT = 10000 +CONFIG_MAP_PATH = "json/config_map.json" + +# API query limit is 1024 (MAX_QUERY_LENGTH+TIMESTAMP_LENGTH) +MAX_QUERY_LENGTH = 924 +TIMESTAMP_LENGTH = 100 + + +class FileNotFoundException(Exception): + pass + + +class QueryStringPatternTranslator: + """ + comparator values to match with supported data source operators + """ + + def __init__(self, pattern: Pattern, data_model_mapper, options): + + logger.info("Vectra Connector") + self.dmm = data_model_mapper + self.comparator_lookup = self.dmm.map_comparator() + self.config_map = self.load_json(CONFIG_MAP_PATH) + self.pattern = pattern + self.options = options + self.qualified_queries = [] + self.parse_expression(pattern) + + @staticmethod + def load_json(rel_path_of_file): + """ Consumes a json file and returns a dictionary + :param rel_path_of_file: str + :return: dict """ + _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file + try: + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundException + except FileNotFoundException as e: + raise FileNotFoundError(f'{rel_path_of_file} not found') from e + + def _format_set(self, values, mapped_field_type, expression) -> list: + """ + Formats value in the event of set operation + :param values: list + :param mapped_field_type: str + :param expression: object + :return formatted value + """ + gen = values.element_iterator() + formatted_values = [] + for value in gen: + value = self._check_value_comparator_support(value, expression.comparator, mapped_field_type) + formatted_value = QueryStringPatternTranslator._format_equality( + QueryStringPatternTranslator._format_value_type(value, mapped_field_type)) + formatted_values.append(formatted_value) + return formatted_values + + @staticmethod + def _format_match(value, mapped_field_type) -> str: + """ + Formats value in the event of match operation + :param value, mapped_field_type + :return formatted string type value + """ + if mapped_field_type != "string": + raise NotImplementedError("MATCHES operator is supported only for string type input") + if '^' in value and value.index('^') != 0: + raise NotImplementedError('^ symbol should be at the starting position of the expression') + if '$' in value and value.index('$') != len(value) - 1: + raise NotImplementedError('$ symbol should be at the ending position of the expression') + if ' ' in value: + raise NotImplementedError('MATCHES operator is not supported for value contains spaces') + value = QueryStringPatternTranslator._escape_value(value) + return value + '*' + + @staticmethod + def _format_equality(value) -> str: + """ + Formats value in the event of equality operation + :param value + :return formatted value + """ + return f'\"{str(value)}\"' + + @staticmethod + def _format_like(value, mapped_field_type) -> str: + """ + Formatting value in the event of like operation + :param value,mapped_fields_array + :return: list + """ + if mapped_field_type != "string": + raise NotImplementedError("LIKE operator is supported only for string type input") + if ' ' in value: + raise NotImplementedError('LIKE operator is not supported for value contains spaces') + return QueryStringPatternTranslator._escape_value(value) + + @staticmethod + def _escape_value(value) -> str: + """ + adds escape characters to string and regex value + :param value + :return formatted value + """ + if isinstance(value, str): + value = value.replace('\\', '\\\\').replace('\"', '\\"') + return value + + @staticmethod + def _format_datetime(value) -> str: + """ + format the date + :param: value: str + :return: converted_time: str + """ + try: + time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ' + if re.search(r"\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}Z", str(value)): # without milli seconds + time_pattern = '%Y-%m-%dT%H:%M:%SZ' + converted_time = datetime.strptime(value, time_pattern).strftime('%Y-%m-%dT%H%M') + return converted_time + except ValueError: + pass + raise NotImplementedError(f'cannot format the timestamp {value}') + + @staticmethod + def _parse_time_range(qualifier, time_range) -> list: + """ + Converts qualifier timestamp to custom + return: list of converted custom values + """ + try: + compile_timestamp_regex = re.compile(START_STOP_PATTERN) + converted_timestamp = [] + if qualifier and compile_timestamp_regex.search(qualifier): + time_range_iterator = compile_timestamp_regex.finditer(qualifier) + time_range_list = [QueryStringPatternTranslator._format_datetime(each.group()) + for each in time_range_iterator] + else: + start_time = STOP_TIME - timedelta(minutes=time_range) + time_range_list = [start_time.strftime('%Y-%m-%dT%H%M'), STOP_TIME.strftime('%Y-%m-%dT%H%M')] + for timestamp in time_range_list: + converted_timestamp.append(timestamp) + return converted_timestamp + except (KeyError, IndexError, TypeError) as e: + raise e + + def _get_mapped_field_type(self, mapped_field_array) -> str: + """ + Returns the type of mapped field array + :param mapped_field_array: list + :return: str + """ + mapped_field = mapped_field_array[0] + mapped_field_type = "string" + for key, value in self.config_map.items(): + if mapped_field in value and key in ["int_supported_fields", "timestamp_supported_fields", + "boolean_supported_fields"]: + mapped_field_type = key.split('_')[0] + break + return mapped_field_type + + @staticmethod + def _format_value_type(value, mapped_field_type): + """ + check input value format that matches with the mapped field value type + :param value + :param mapped_field_type: str + :return value + """ + converted_value = str(value) + if mapped_field_type == "int": + if not converted_value.isdigit(): + raise NotImplementedError(f'string type input - {converted_value} is not supported for ' + f'integer type fields') + converted_value = int(value) + + return converted_value + + def _check_value_comparator_support(self, value, comparator, mapped_field_type): + """ + checks the comparator and value support + :param value + :param comparator + :param mapped_field_type: str + :return value: str + """ + operator = self.comparator_lookup[str(comparator)] + + if mapped_field_type != "timestamp" and isinstance(value, str) and \ + comparator in [ComparisonComparators.GreaterThan, + ComparisonComparators.GreaterThanOrEqual, ComparisonComparators.LessThan, + ComparisonComparators.LessThanOrEqual]: + raise NotImplementedError(f'{operator.replace(":<", "<").replace(":>", ">")} operator is not supported for ' + f'string type input') + + if mapped_field_type == "timestamp" and comparator not in [ComparisonComparators.Like, + ComparisonComparators.Matches]: + value = self._format_datetime(str(value)) + + if mapped_field_type == "boolean": + if comparator not in [ComparisonComparators.Equal, ComparisonComparators.NotEqual, + ComparisonComparators.In]: + raise NotImplementedError(f'{operator.replace(":<", "<").replace(":>", ">")} ' + f'operator is not supported for Boolean type input. ' + f'Possible supported operator are [ =, !=, IN, NOT IN ]') + + if not value.lower() in ["true", "false"]: + raise NotImplementedError('Boolean type field allows only true/false') + return value + + def _lookup_comparison_operator(self, expression_operator) -> str: + """ + lookup operators support in gcp chronicle + :param expression_operator:object + :return str + """ + if str(expression_operator) not in self.comparator_lookup: + raise NotImplementedError( + f'Comparison operator {expression_operator.name} unsupported for vectra connector') + + return self.comparator_lookup[str(expression_operator)] + + def _eval_comparison_value(self, expression, mapped_field_type): + """ + Function for parsing comparison expression value + :param expression: expression object + :return: formatted expression value + """ + if (expression.negated or expression.comparator == ComparisonComparators.NotEqual) and \ + mapped_field_type != 'string': + raise NotImplementedError('Not operator is only supported for string type fields') + + if expression.comparator == ComparisonComparators.Like: + value = self._format_like(expression.value, mapped_field_type) + elif expression.comparator == ComparisonComparators.Matches: + value = self._format_match(expression.value, mapped_field_type) + elif expression.comparator == ComparisonComparators.In: + value = self._format_set(expression.value, mapped_field_type, expression) + elif expression.comparator in [ComparisonComparators.GreaterThan, ComparisonComparators.GreaterThanOrEqual, + ComparisonComparators.LessThan, ComparisonComparators.LessThanOrEqual, + ComparisonComparators.Equal, ComparisonComparators.NotEqual]: + value = self._format_value_type(expression.value, mapped_field_type) + value = self._check_value_comparator_support(value, expression.comparator, mapped_field_type) + value = self._format_equality(value) + else: + raise NotImplementedError('Unknown comparator expression operator') + return value + + def _add_qualifier(self, query, qualifier) -> str: + """ + Convert the qualifier into epoch time and + append in the query. + params: query : str + params: qualifier + return: query : str + """ + time_range = QueryStringPatternTranslator._parse_time_range(qualifier, self.options['time_range']) + query = f"(({query}) AND (detection.last_timestamp:[{time_range[0]} to {time_range[1]}]))" + return query + + def _create_parsed_query(self, field_name, value, comparator, is_negated) -> str: + """ + Creates comparison string for each field_name in mapped_field_array + :param field_name: str + :param value + :param comparator: str + :param is_negated: boolean + :return: str + """ + comparison_string = "" + if not isinstance(value, list): + value = [value] + all_mappings = [] + for values in value: + if is_negated: + field_mappings = f'({field_name}:* AND NOT {field_name}{comparator}{values})' + else: + field_mappings = f'{field_name}{comparator}{values}' + all_mappings.append(field_mappings) + if all_mappings: + if len(all_mappings) == 1: + comparison_string += all_mappings[0] + else: + condition_string = " OR " + in_comparison_string = f'{condition_string}'.join(all_mappings) + comparison_string += in_comparison_string + + return comparison_string + + def _parse_mapped_fields(self, formatted_value, mapped_fields_array, expression) -> str: + """ + parse mapped fields into boolean expression + :param formatted_value: str + :param mapped_fields_array: list + :param mapped_field_type:str + :param expression: expression object + :return: str + """ + is_negated = None + comparator = self._lookup_comparison_operator(expression.comparator) + comparison_string = "" + comparison_string_new_count = 0 + if expression.negated or expression.comparator == ComparisonComparators.NotEqual: + is_negated = True + for field_name in mapped_fields_array: + comparison_string_new = self._create_parsed_query(field_name, formatted_value, comparator, is_negated) + if comparison_string_new: + comparison_string_new_count += 1 + if comparison_string: + comparison_string += " OR " + comparison_string += comparison_string_new + + return comparison_string + + def _eval_combined_comparison_exp(self, expression) -> str: + """ + Function for parsing combined comparison expression + :param expression: expression object + """ + operator = self._lookup_comparison_operator(expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + if not expression_01 or not expression_02: + return '' + if isinstance(expression.expr1, CombinedComparisonExpression): + expression_01 = f'{expression_01}' + if isinstance(expression.expr2, CombinedComparisonExpression): + expression_02 = f'{expression_02}' + + query = f'({expression_01}) {operator} ({expression_02})' + return f'{query}' + + def _eval_combined_observation_exp(self, expression, qualifier=None) -> str: + """ + Function for parsing combined observation expression + :param expression: expression object + :param qualifier: qualifier + """ + operator = self._lookup_comparison_operator(expression.operator) + expression_01 = self._parse_expression(expression.expr1, qualifier) + expression_02 = self._parse_expression(expression.expr2, qualifier) + query = '' + if expression_01 and expression_02: + query = f'({expression_01} {operator} {expression_02})' + elif expression_01: + query = f'{expression_01}' + elif expression_02: + query = f'{expression_02}' + return query + + def _parse_expression(self, expression, qualifier=None) -> str: + """ + Formation of vectra query from ANTLR parsing expression + :param expression: expression object, ANTLR parsed expression object + :param qualifier: str, default in None + :return :None or str + """ + if isinstance(expression, ComparisonExpression): # Base Case + stix_objects = expression.object_path.split(':') + mapped_fields_array = self.dmm.map_field(stix_objects[0], stix_objects[1]) + mapped_field_type = self._get_mapped_field_type(mapped_fields_array) + value = self._eval_comparison_value(expression, mapped_field_type) + query = self._parse_mapped_fields(value, mapped_fields_array, expression) + return query + + elif isinstance(expression, CombinedComparisonExpression): + return self._eval_combined_comparison_exp(expression) + + elif isinstance(expression, ObservationExpression): + query = self._parse_expression(expression.comparison_expression) + return self._add_qualifier(query, qualifier) + + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + if isinstance(expression.observation_expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(expression.observation_expression.operator) + expression_01 = self._parse_expression(expression.observation_expression.expr1, expression.qualifier) + expression_02 = self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + query = f'({expression_01} {operator} {expression_02})' + else: + query = self._parse_expression(expression.observation_expression, expression.qualifier) + if qualifier is not None: + query = self._add_qualifier(query, qualifier) + return query + + elif isinstance(expression, CombinedObservationExpression): + return self._eval_combined_observation_exp(expression, qualifier) + elif isinstance(expression, Pattern): + return self._parse_expression(expression.expression) + else: + raise RuntimeError(f'Unknown Recursion Case for expression={expression}, ' + f'type(expression)={type(expression)}') + + def parse_expression(self, pattern: Pattern) -> list: + """ + Formation of vectra query from ANTLR parsing expression. + :param pattern: expression object, ANTLR parsed expression object + """ + query = self._parse_expression(pattern) + if self.options['result_limit'] > DEFAULT_LIMIT: + self.options['result_limit'] = DEFAULT_LIMIT + + vectra_queries = [] + + # Query length exceed the max query limit will split the query + if len(query) > MAX_QUERY_LENGTH: + obj = QuerySeparator() + vectra_queries = obj.split_query(query) + + # change single query into list + if query and not vectra_queries: + vectra_queries = [query] + + for row in vectra_queries: + self.qualified_queries.append("query_string=" + row) + + +def translate_pattern(pattern: Pattern, data_model_mapping, options) -> list: + """ + Conversion of ANTLR pattern to Vectra query + :param pattern: expression object, ANTLR parsed expression object + :param data_model_mapping: DataMapper object, mapping object obtained by parsing json + :param options: dict, time_range defaults to 5 + :return: list, Vectra queries + """ + translated_query_strings = QueryStringPatternTranslator(pattern, data_model_mapping, options) + queries = translated_query_strings.qualified_queries + return queries + + +class QuerySeparator: + """ + split the translated query based on query length + """ + + def split_query(self, query): + """ + split the query based on query length limit. + Between two statements if OR is present it will split if AND is present it wont split. + param query: str + return queries: list + """ + try: + queries = [] + # Based on brackets converting the translated query into nested list + parsed_query = nestedExpr(opener="(", closer=")", ignoreExpr=White('')).parseString(query).asList() + parsed_query = parsed_query[0] + + # Check valid parsing + if len(parsed_query) != 3: + return [query] + + # process the each statement its need to split as separate query or not + statement_1 = self.split_sub_query(parsed_query[0]) + statement_2 = self.split_sub_query(parsed_query[2]) + operator = parsed_query[1] + + if str(statement_1).count('Unable to split') or str(statement_2).count('Unable to split'): + return [query] + + # statement is greater than query length limit and it was split. + if isinstance(statement_1, list) or isinstance(statement_2, list): + + # Try to combine the split query + statement_1 = self.combine_list_query(statement_1) + statement_2 = self.combine_list_query(statement_2) + + # AND operator is processed if only its contains timestamp then will attach the timestamp in each query + queries = self.attach_timestamp(statement_1, statement_2, operator) + + if str(queries).count('Unable to split'): + return [query] + + # statements are lesser than query length limit and it wont split. + elif statement_1 and statement_2: + combined_statement = statement_1 + ' ' + operator + ' ' + statement_2 + if len(combined_statement) > MAX_QUERY_LENGTH: + if operator == 'AND': + queries.append(query) + else: + queries.append(statement_1) + queries.append(statement_2) + else: + queries.append(combined_statement) + else: + queries.append(query) + + return queries + except Exception as e: + logger.info(f'Unable to split the query. Error occurred {str(e)}') + return [query] + + def split_sub_query(self, parsed_query): + """ + Iterate the query from list and based on operator separate the query or combine the query. + param query: str + return queries: list or str + """ + + # iterate other than IN operator + if len(parsed_query) == 3: + statement_1 = parsed_query[0] + statement_2 = parsed_query[2] + operator = parsed_query[1] + queries = [] + query_list = [statement_1, statement_2] + for index, statement in enumerate(query_list, start=0): + if any(isinstance(i, list) for i in statement): + statement = self.split_sub_query(statement) + elif isinstance(statement, list): + statement = ' '.join(map(str, statement)) + if statement and len(statement) > MAX_QUERY_LENGTH: + statement = self.split_query_and_or_operator(statement) + + if index == 0: + statement_1 = statement + else: + statement_2 = statement + + if isinstance(statement_1, list) or isinstance(statement_2, list): + queries = self.attach_timestamp(statement_1, statement_2, operator) + return queries + + combined_query = '(' + statement_1 + ') ' + operator + ' (' + statement_2 + ')' + + if len(combined_query) > MAX_QUERY_LENGTH: + if operator == 'AND': + return 'Unable to split' + queries.append(statement_1) + queries.append(statement_2) + return queries + return combined_query + elif any(isinstance(i, list) for i in parsed_query): + # iterate for IN operator + return self.handle_translated_in_operator(parsed_query) + elif isinstance(parsed_query, list): + combined_query = '(' + ' '.join(map(str, parsed_query)) + ')' + if len(combined_query) > MAX_QUERY_LENGTH: + combined_query = self.split_query_and_or_operator(combined_query) + return combined_query + else: + return '(' + parsed_query + ')' + + @staticmethod + def split_query_and_or_operator(parsed_query, query_len=MAX_QUERY_LENGTH): + """ Split the query based on AND/OR operator. + param query: str, MAX_QUERY_LENGTH: int + return queries: list or str """ + start = 0 + index = 0 + result = [] + combined_query = '' + + while index < len(parsed_query): + if 'OR' in parsed_query[index:len(parsed_query)]: + index = parsed_query[index:len(parsed_query)].index('OR') + index = start + index + 2 + elif 'AND' in parsed_query[index:len(parsed_query)]: + index = parsed_query[index:len(parsed_query)].index('AND') + index = start + index + 3 + else: + index = len(parsed_query) + if len(combined_query + parsed_query[start: index]) > query_len: + if combined_query: + result.append('(' + combined_query.rstrip('OR').rstrip('AND') + ')') + if parsed_query[start: index]: + result.append('(' + parsed_query[start: index].lstrip('(').rstrip(')') + ')') + else: + combined_query += parsed_query[start: index] + combined_query = combined_query.lstrip('(').rstrip(')') + result.append('(' + combined_query + ')') + break + if len(combined_query + parsed_query[start: index]) > query_len and combined_query: + query = combined_query.rstrip('OR').rstrip('AND') + query = query.lstrip('(').rstrip(')').rstrip("'").rstrip(",") + if query: + result.append('(' + query + ')') + combined_query = '' + combined_query += parsed_query[start: index] + start = index + return result + + @staticmethod + def combine_list_query(query_list): + """ Combine the query from list. + param query: list, return queries: list """ + + query = '' + queries = [] + + # exit the process for invalid arguments + if isinstance(query_list, str) or len(query_list) == 1: + return query_list + + for index, each_query in enumerate(query_list, start=0): + if not query: + query = each_query + continue + + combined_query = query + ' OR ' + each_query + if len(combined_query) > MAX_QUERY_LENGTH: + queries.append(query) + query = each_query + if len(query_list) - 1 == index: + queries.append(query) + continue + + query = combined_query + + if len(query_list) - 1 == index: + queries.append(query) + continue + + return queries + + @staticmethod + def attach_timestamp(statement_1, statement_2, operator): + """Attach the timestamp in query""" + queries = [] + + if operator == 'AND': + if statement_2.count('detection.last_timestamp')==1\ + and len(statement_2) < TIMESTAMP_LENGTH: + if not (statement_2.count('(') and statement_2.count(')')): + statement_2 = '(' + statement_2 + ')' + if ') to (' in statement_2: + statement_2 = statement_2.replace(') to (', ' to ') + queries = ['(' + row + ') AND ' + statement_2 for row in statement_1] + else: + return 'Unable to split' + + if operator == 'OR': + queries += statement_1 if isinstance(statement_1, list) else [statement_1] + queries += statement_2 if isinstance(statement_2, list) else [statement_2] + + return queries + + def handle_translated_in_operator(self, parsed_query): + """ Process the translated query having IN operator in STIX pattern. + Params Query: List, return Query: List """ + query = '' + queries = [] + for row in parsed_query: + if isinstance(row, str): + query += row + elif isinstance(row, list): + + combined_query = '(' + ' '.join(map(str, row)) + ')' + + if combined_query.count('NOT') and any(isinstance(i, list) for i in row): + combined_query = self.split_sub_query(row) + + query = query.lstrip('OR').lstrip('AND') + + if isinstance(combined_query, list): + queries += [query + value for value in combined_query] + elif isinstance(combined_query, str) and len(combined_query) > MAX_QUERY_LENGTH: + separated = self.split_query_and_or_operator(combined_query, MAX_QUERY_LENGTH - len(query)) + queries += [query + value for value in separated] + else: + queries += [query + combined_query] + query = '' + return queries diff --git a/stix_shifter_modules/vectra/stix_translation/query_translator.py b/stix_shifter_modules/vectra/stix_translation/query_translator.py new file mode 100644 index 000000000..69e284eab --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/query_translator.py @@ -0,0 +1,24 @@ +import logging + +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator +from . import query_constructor + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file :param antlr_parsing_object: + Antlr parsing objects for the STIX pattern :type antlr_parsing_object: object :param mapping: The mapping + file path to use as instructions on how to transform the given STIX query into another format. This should + default to something if one isn't passed in :type mapping: str (filepath) :return: transformed query string + :rtype: str + """ + + logger.info("Converting STIX2 Pattern to data source query") + + query_string = query_constructor.translate_pattern( + antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/vectra/stix_translation/transformers.py b/stix_shifter_modules/vectra/stix_translation/transformers.py new file mode 100644 index 000000000..a86371349 --- /dev/null +++ b/stix_shifter_modules/vectra/stix_translation/transformers.py @@ -0,0 +1,69 @@ +from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer +from stix_shifter_utils.utils import logger +import re + +LOGGER = logger.set_logger(__name__) + + +class ListToValue(ValueTransformer): + """A value transformer to return first value from the list""" + + @staticmethod + def transform(obj): + try: + if isinstance(obj, list): + obj = obj[0] + except ValueError: + LOGGER.error("Cannot return first value from obj %s", obj) + return obj + + +class ChainNameValue(ValueTransformer): + """A value transformer to add default 'kill_chain_name' as 'mitre-attack' with obj""" + + @staticmethod + def transform(obj): + try: + if obj: + return [{"kill_chain_name": "mitre-attack","phase_name": obj}] + except ValueError: + LOGGER.error("Cannot return obj value %s with default param 'kill_chain_name'", obj) + + return None + + +class ConvertToReal(ValueTransformer): + """Transform obj value between 0 to 1 as a float""" + + @staticmethod + def transform(obj): + try: + if not isinstance(obj, float): + obj = obj * 1.0 + except ValueError: + LOGGER.error('Cannot convert input %s to a float value between 0 to 1', obj) + return obj + + +class VerifyDomainValue(ValueTransformer): + """Transform obj value to domain-name""" + + @staticmethod + def transform(obj): + try: + if obj and isinstance(obj, list): + ip_pattern = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') + domain_pattern = re.compile(r'^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(\.)?)+[a-z0-9][a-z0-9-]{0,' + r'61}[a-z0-9]$') + for index, row in enumerate(obj): + if domain_pattern.search(row): + continue + if ip_pattern.search(row): + obj[index] = ip_pattern.search(row).group() + else: + obj.remove(row) + if not obj: + return None + except ValueError: + LOGGER.error('Invalid domain value %s', obj) + return obj diff --git a/stix_shifter_modules/vectra/stix_transmission/__init__.py b/stix_shifter_modules/vectra/stix_transmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/vectra/stix_transmission/api_client.py b/stix_shifter_modules/vectra/stix_transmission/api_client.py new file mode 100644 index 000000000..982ad6c6f --- /dev/null +++ b/stix_shifter_modules/vectra/stix_transmission/api_client.py @@ -0,0 +1,33 @@ +from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync +from stix_shifter_utils.utils import logger + + +class APIClient: + QUERY_ENDPOINT = "/api/v2.4/search/detections" + PING_ENDPOINT = "/api/v2.4/health/connectivity" + + def __init__(self, connection, configuration): + self.logger = logger.set_logger(__name__) + self.auth = configuration.get('auth') + self.result_limit = connection['options'].get('result_limit') + self.headers = {"Authorization": "Token " + self.auth["api_token"], + 'Content-Type': "application/json", + 'Cache-Control': "no-cache"} + self.client = RestApiClientAsync(connection.get('host'), port=None, headers=self.headers) + + async def ping_data_source(self): + """ + Ping the Data Source + :return: Response object + """ + return await self.client.call_api(self.PING_ENDPOINT, 'GET', headers=self.headers, data=None) + + async def get_search_results(self, query): + """ + Get results from Data Source + :param query: Data Source Query + :return: Response Object + """ + self.logger.debug("query: %s", query) + url = self.QUERY_ENDPOINT + '/?' + query + return await self.client.call_api(url, 'GET', headers=self.headers, data=None) diff --git a/stix_shifter_modules/vectra/stix_transmission/connector.py b/stix_shifter_modules/vectra/stix_transmission/connector.py new file mode 100644 index 000000000..7b960c05b --- /dev/null +++ b/stix_shifter_modules/vectra/stix_transmission/connector.py @@ -0,0 +1,215 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +from .api_client import APIClient +import json + + +class Connector(BaseJsonSyncConnector): + VECTRA_MAX_PAGE_SIZE = 5000 + + def __init__(self, connection, configuration): + self.api_client = APIClient(connection, configuration) + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def ping_connection(self): + return_obj = {} + response_dict = {} + try: + response_wrapper = await self.api_client.ping_data_source() + response_code = response_wrapper.code + if response_code == 200: + return_obj['success'] = True + elif response_code in (401, 403): + response_dict['code'] = response_code + response_dict['message'] = "Invalid Authentication" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + except Exception as ex: + if "timeout_error" in str(ex): + response_dict['code'] = 408 + response_dict['message'] = str(ex) + self.logger.error('error when getting search results: %s', str(ex)) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + async def create_results_connection(self, query, offset, length): + """ + Function to create query connection + :param query: str + :param offset: int + :param length: str + :return: list + """ + response_dict = {} + return_obj = {} + data = [] + try: + offset = int(offset) + length = int(length) + pagination_offset = offset + total_records = length + page = 1 + + # changing offset to page number + if offset > length: + pagination_offset = offset % length + if pagination_offset < 5: + page = round(offset / length) + 1 + else: + page = round(offset / length) + + # set total record count + if self.api_client.result_limit < total_records: + total_records = self.api_client.result_limit + + # set page size + if total_records <= Connector.VECTRA_MAX_PAGE_SIZE: + page_size = total_records + else: + page_size = Connector.VECTRA_MAX_PAGE_SIZE + + query = f"page_size={page_size}&{query}&page={page}" + response_wrapper = await self.api_client.get_search_results(query) + response_code = response_wrapper.code + response = json.loads(response_wrapper.read().decode('utf-8')) + + if response_code == 200: + data += response['results'] + next_url = response.get('next') + while next_url: + if len(data) >= total_records: + break + next_url = next_url.split('/api/v2.4/search/detections/?') + query = next_url[1] + next_response_wrapper = await self.api_client.get_search_results(query) + response_code = next_response_wrapper.code + next_response = json.loads(next_response_wrapper.read().decode('utf-8')) + next_url = next_response.get('next') + if response_code == 200: + data += next_response['results'] + else: + response = next_response + break + + if response_code == 200: + return_obj['success'] = True + return_obj['data'] = self.get_results_data(data[pagination_offset:total_records]) + + # handling error response + if response_code == 200: + pass + elif response_code in (422, 406): + if 'page_size' in str(response) or 'invalid params' in str(response): + response_dict['code'] = response_code + response_dict['message'] = "Invalid Parameters" + else: + response_dict['code'] = 421 + response_dict['message'] = "Invalid Query" + if 'Query too long' in str(response): + response_dict['message'] += "- Query length is too long" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + elif response_code in (401, 403): + response_dict['code'] = response_code + response_dict['message'] = "Invalid Authentication" + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + else: + response_dict['message'] = str(response) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + + except Exception as ex: + if "timeout_error" in str(ex): + response_dict['code'] = 408 + response_dict['message'] = str(ex) + self.logger.error('error when getting search results: %s', str(ex)) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + def set_default_protocol(self, item): + """ + Adds the default value for protocol if it doesn't exist in record. + :param item: dict/list + :return: None + """ + if isinstance(item, list): + for list_item in item: + self.set_default_protocol(list_item) + elif not (item.get('protocol') or item.get("app_protocol") or item.get("dst_protocol")): + item['protocol'] = "tcp" + + @staticmethod + def is_extensible(record) -> bool: + """ + Checks whether the record is extensible for preprocessing. + :param record: dict + :return: bool + """ + if record.get('detection_type') == 'Suspicious LDAP Query': + return False + if record.get('detection_type') in ('RPC Recon', 'Kerberoasting: SPN Sweep'): + return True + if len(record.get('grouped_details', [])) and \ + (record['grouped_details'][0].get('events') or + record['grouped_details'][0].get('sessions') or + record['grouped_details'][0].get('connection_events')): + return True + return False + + @staticmethod + def set_destination_ip(item): + """ set destination ip in each events. + :param item: dict + :return: dict """ + dest_ip = item.get('dst_ips') + if isinstance(dest_ip, str): + item['dest_ip'] = [dest_ip] + + if isinstance(dest_ip, list) and len(dest_ip): + for index, value in enumerate(item['events']): + if len(dest_ip)-1 >= index: + value['dst_ips'] = dest_ip[index] + else: + value['dst_ips'] = dest_ip[0] + return item + + def get_results_data(self, response_dict): + """ + Preprocessing the response based on detection type + :param response_dict: list + :return: list + """ + for record in response_dict: + + detection_type = record.get('detection_type', '') + + if 'Privilege' in detection_type: + # Skip any preprocessing for these detections. + continue + + grouped_details = record.get('grouped_details', []) + if detection_type in ('New Host Role', 'New Host'): + # INFO detection category has a different stix translation mapping than grouped_details. + record['grouped_details_info'] = record.pop('grouped_details') + continue + + if self.is_extensible(record): + # grouped_details_ex has a different stix translation mapping than grouped_details. + record['grouped_details_ex'] = record.pop('grouped_details') + grouped_details = record['grouped_details_ex'] + + # set default protocol + for group in grouped_details: + item = group + if record.get('grouped_details_ex'): + if group.get('events'): + if detection_type == 'Data Smuggler': + group = self.set_destination_ip(group) + item = group.get('events') + elif group.get('sessions'): + item = group.get('sessions') + elif group.get('connection_events'): + item = group.get('connection_events') + elif group.get('dst_hosts'): + item = group.get('dst_hosts') + self.set_default_protocol(item) + return response_dict diff --git a/stix_shifter_modules/vectra/stix_transmission/error_mapper.py b/stix_shifter_modules/vectra/stix_transmission/error_mapper.py new file mode 100644 index 000000000..8e538041a --- /dev/null +++ b/stix_shifter_modules/vectra/stix_transmission/error_mapper.py @@ -0,0 +1,34 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + 401: ErrorCode.TRANSMISSION_AUTH_CREDENTIALS, + 403: ErrorCode.TRANSMISSION_AUTH_CREDENTIALS, + 421: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + 422: ErrorCode.TRANSMISSION_INVALID_PARAMETER, + 408: ErrorCode.TRANSMISSION_CONNECT +} + + +class ErrorMapper: + logger = logger.set_logger(__name__) + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + + @staticmethod + def set_error_code(json_data, return_obj, connector=None): + code = None + try: + code = int(json_data['code']) + except Exception: + pass + + error_code = ErrorMapper.DEFAULT_ERROR + + if code in error_mapping: + error_code = error_mapping.get(code) + + if error_code == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: %s", str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_code, connector=connector) diff --git a/stix_shifter_modules/vectra/test/stix_translation/test_vectra_json_to_stix.py b/stix_shifter_modules/vectra/test/stix_translation/test_vectra_json_to_stix.py new file mode 100644 index 000000000..c86dfe28c --- /dev/null +++ b/stix_shifter_modules/vectra/test/stix_translation/test_vectra_json_to_stix.py @@ -0,0 +1,362 @@ +""" test script to perform unit test case for vectra translate results """ +import unittest +from stix_shifter_modules.vectra.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers + +MODULE = "vectra" +entry_point = EntryPoint() +map_data = entry_point.get_results_translator().map_data +data_source = { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "vectra", + "identity_class": "events" +} +options = {} + +vectra_sample_response = { + 'summary': { + 'dst_ips': ['11.111.11.111'], + 'num_sessions': 4, + 'bytes_sent': 4053918, + 'bytes_received': 3186727, + 'description': 'This host communicated with an external destination using HTTPS where another protocol was ' + 'running over the top of the session. The host appeared to be under the control of the ' + 'external destination. ' + }, + 'id': 10130, + 'category': 'COMMAND & CONTROL', + 'detection': 'Hidden HTTPS Tunnel', + 'detection_category': 'COMMAND & CONTROL', + 'detection_type': 'Hidden HTTPS Tunnel', + 'custom_detection': None, + 'description': None, + 'src_ip': '11.111.11.111', + 'state': 'inactive', + 'certainty': 0, + 'threat': 0, + 'created_timestamp': '2022-12-22T07:43:52Z', + 'first_timestamp': '2022-12-22T07:33:38Z', + 'last_timestamp': '2022-12-27T06:44:32Z', + 'targets_key_asset': False, + 'is_targeting_key_asset': False, + 'src_account': None, + 'src_host': { + 'id': 872, + 'ip': '11.111.11.111', + 'name': 'VMAL #2 windows 11.111.11.111 (higaki-ha11)', + 'is_key_asset': False, + 'groups': [{ + 'id': 145, + 'name': 'Super Test domain group', + 'description': 'created during API testing', + 'last_modified': '2022-08-03T15:33:04Z', + 'last_modified_by': 'reliaquest', + 'type': 'host' + }, { + 'id': 144, + 'name': 'Partner VLAB - User Devices', + 'description': '', + 'last_modified': '2022-01-27T12:05:24Z', + 'last_modified_by': 'user (Removed)', + 'type': 'ip' + }], + 'threat': 82, + 'certainty': 71 + }, + 'note': None, + 'note_modified_by': None, + 'note_modified_timestamp': None, + 'sensor': 'test', + 'sensor_name': 'test', + 'tags': [], + 'triage_rule_id': None, + 'assigned_to': 'vectra', + 'assigned_date': '2022-12-14T06:59:22Z', + 'groups': [{ + 'id': 144, + 'name': 'Partner VLAB - User Devices', + 'description': '', + 'type': 'ip', + 'last_modified': '2022-01-27T12:05:24Z', + 'last_modified_by': 'user' + }], + 'is_marked_custom': False, + 'is_custom_model': False, + 'src_linked_account': None, + 'campaign_summaries': [], + 'is_triaged': False, + 'filtered_by_ai': False, + 'filtered_by_user': False, + 'filtered_by_rule': False, + '_doc_modified_ts': '2023-06-08T06:20:58.684936', + 'grouped_details_ex': [{ + 'external_target': { + 'ip': '11.111.11.111', + 'name': '' + }, + 'num_sessions': 4, + 'bytes_received': 3186727, + 'bytes_sent': 4053918, + 'ja3_hashes': [''], + 'ja3s_hashes': [''], + 'sessions': [{ + 'tunnel_type': 'Long TCP session - Command line', + 'protocol': 'tcp', + 'app_protocol': 'https', + 'dst_port': 222, + 'dst_ip': '11.111.11.111', + 'bytes_received': 361298, + 'bytes_sent': 309439, + 'first_timestamp': '2022-12-27T06:33:33Z', + 'last_timestamp': '2022-12-27T06:44:32Z', + 'dst_geo': None, + 'dst_geo_lat': None, + 'dst_geo_lon': None + }, + { + 'tunnel_type': 'Long TCP session - Command line', + 'protocol': 'tcp', + 'app_protocol': 'https', + 'dst_port': [222], + 'dst_ip': '11.111.11.111', + 'bytes_received': 329386, + 'bytes_sent': 217025, + 'first_timestamp': '2022-12-27T06:14:33Z', + 'last_timestamp': '2022-12-27T06:24:33Z', + 'dst_geo': None, + 'dst_geo_lat': None, + 'dst_geo_lon': None + }], + 'first_timestamp': '2022-12-22T07:33:38Z', + 'last_timestamp': '2022-12-27T06:44:32Z', + 'dst_ips': ['11.111.11.111'], + 'dst_ports': [222], + 'target_domains': [''] + }] +} + +vectra_sample_grouped_response = { + "id": 8111, + "category": "COMMAND & CONTROL", + "detection": "PowershellEmpire", + "detection_category": "COMMAND & CONTROL", + "detection_type": "PowershellEmpire", + "custom_detection": None, + "description": "c3483e00759211eda7e9ab6654adc2f1", + "src_ip": "11.111.11.111", + "state": "inactive", + "certainty": 0, + "threat": 0, + "created_timestamp": "2022-12-06T21:11:20Z", + "first_timestamp": "2022-12-06T19:57:59Z", + "last_timestamp": "2022-12-07T07:09:39Z", + "targets_key_asset": False, + "is_targeting_key_asset": False, + "src_account": None, + "src_host": { + "id": 1543, + "ip": "11.111.11.111", + "name": "VMAL #2 windows 11.111.11.111 (kgalloway141)", + "is_key_asset": False, + "groups": [ + { + "id": 144, + "name": "Partner VLAB - User Devices", + "description": "", + "last_modified": "2022-01-27T12:05:24Z", + "last_modified_by": "user (Removed)", + "type": "ip" + } + ], + "threat": 0, + "certainty": 0 + }, + "note": None, + "note_modified_by": None, + "note_modified_timestamp": None, + "sensor": "None", + "sensor_name": "test", + "tags": [], + "triage_rule_id": None, + "assigned_to": "test", + "assigned_date": "2022-12-14T06:58:53Z", + "groups": [ + { + "id": 144, + "name": "Partner VLAB - User Devices", + "description": "", + "type": "ip", + "last_modified": "2022-01-27T12:05:24Z", + "last_modified_by": "user" + } + ], + "is_marked_custom": False, + "is_custom_model": True, + "src_linked_account": None, + "grouped_details": [ + { + "description": "PowershellEmpire", + "dst_geo": None, + "dst_geo_lat": None, + "dst_geo_lon": None, + "first_timestamp": "2022-12-07T07:09:39Z", + "last_timestamp": "2022-12-07T07:09:39Z", + "count": 1, + "reason": None, + "identity": None, + "flex_json": {}, + "detection_guid": None, + "distilled_context": {}, + "sequence_id": None, + "is_host_detail": True, + "is_account_detail": False, + "account_uid": None, + "account_detection": None, + "host_detection": 8111, + "src_ip": "11.111.11.111", + "user_agent": "\"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\"", + "status_code": "200", + "uri": "\"/admin/get.php\"", + "metadata": { + "user_agent": "\"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\"", + "status_code": "200", + "uri": "\"/admin/get.php\"", + "resp_h": "11.111.11.111", + "resp_p": 80, + "orig_h": "11.111.11.111", + "orig_ip_bytes": 1427994, + "resp_ip_bytes": 17600003, + "resp_hostname": "VMAL #2 windows 11.111.11.111 (kgalloway141)", + "orig_sluid": "test", + "resp_sluid": None + }, + "accounts": [], + "target_domains": [ + "VMAL #2 windows 11.111.11.111 (kgalloway141)" + ], + "dst_ips": [ + "11.111.11.111" + ], + "dst_ports": [ + 80 + ], + "protocol": None, + "bytes_received": 17600003, + "bytes_sent": 1427994 + } + ], + "summary": { + "custom_model_query": "Search: None\n\nFilters: uri: /admin/get.php", + "bytes_received": 986581476, + "bytes_sent": 85530107, + "matches": 100, + "first_matched": "2022-12-06T22:00:00+00:00", + "last_matched": "2022-12-07T07:09:39+00:00", + "description": "" + }, + "campaign_summaries": [], + "is_triaged": False, + "filtered_by_ai": False, + "filtered_by_user": False, + "filtered_by_rule": False, + "_doc_modified_ts": "2023-05-17T22:35:14.497368" +} + + +class TestVectraResultsToStix(unittest.TestCase): + """ + class to perform unit test case for vectra translate results + """ + + @staticmethod + def get_first(itr, constraint): + """ return the obj in the itr if constraint is true """ + return next((obj for obj in itr if constraint(obj)), None) + + @staticmethod + def get_first_of_type(itr, typ): + """ check whether the object belongs to respective stix object """ + return TestVectraResultsToStix.get_first(itr, lambda o: isinstance(o, dict) and o.get('type') == typ) + + @staticmethod + def get_observed_data_objects(data): + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + return observed_data['objects'] + + def test_ipv4_addr_json_to_stix(self): + """test ipv4-addr stix object properties""" + objects = TestVectraResultsToStix.get_observed_data_objects(vectra_sample_response) + ipv4_obj = TestVectraResultsToStix.get_first_of_type(objects.values(), 'ipv4-addr') + assert (ipv4_obj is not None), 'ipv4 object type not found' + assert ipv4_obj['type'] == 'ipv4-addr' + assert ipv4_obj['value'] == '11.111.11.111' + + def test_network_traffic_json_to_stix(self): + """test network_traffic stix object properties""" + objects = TestVectraResultsToStix.get_observed_data_objects(vectra_sample_response) + network_traffic_obj = TestVectraResultsToStix.get_first_of_type(objects.values(), 'network-traffic') + assert (network_traffic_obj.keys() == {'type', 'x_tunnel_type', 'protocols', 'dst_port', 'dst_ref', + 'dst_byte_count', 'src_byte_count', 'start', 'end'}) + assert (network_traffic_obj is not None), 'network-traffic object type not found' + assert network_traffic_obj['type'] == 'network-traffic' + assert network_traffic_obj['x_tunnel_type'] == 'Long TCP session - Command line' + assert network_traffic_obj['protocols'] == ['tcp', 'https'] + + def test_x_grouped_details(self): + """test x-grouped-details stix object properties""" + objects = TestVectraResultsToStix.get_observed_data_objects(vectra_sample_response) + x_grouped_obj = TestVectraResultsToStix.get_first_of_type(objects.values(), 'x-grouped-details') + assert (x_grouped_obj.keys() == {'type', 'num_sessions', 'dst_byte_count', 'src_byte_count', 'ja3_hashes', + 'ja3s_hashes', 'session_refs', 'start', 'end'}) + assert (x_grouped_obj is not None), 'x-grouped-details object type not found' + assert x_grouped_obj['type'] == 'x-grouped-details' + assert x_grouped_obj['num_sessions'] == 4 + assert x_grouped_obj['dst_byte_count'] == 3186727 + + def test_x_ibm_finding(self): + """test x-ibm-finding stix object properties""" + objects = TestVectraResultsToStix.get_observed_data_objects(vectra_sample_response) + x_ibm_obj = TestVectraResultsToStix.get_first_of_type(objects.values(), 'x-ibm-finding') + assert (x_ibm_obj.keys() == {'type', 'type', 'event_count', 'description', 'alert_id', 'ttp_tagging_refs', + 'name', 'finding_type', 'src_ip_ref', 'x_state', 'confidence', 'severity', + 'time_observed', 'start', 'end', 'x_sensor_name', 'x_assigned_to', + 'x_assigned_date', 'x_is_triaged', 'ioc_refs'}) + assert (x_ibm_obj is not None), 'x-ibm-finding object type not found' + assert x_ibm_obj['type'] == 'x-ibm-finding' + assert x_ibm_obj['alert_id'] == 10130 + + def test_x_oca_asset(self): + """test x-oca-asset stix object properties""" + objects = TestVectraResultsToStix.get_observed_data_objects(vectra_sample_response) + x_oca_obj = TestVectraResultsToStix.get_first_of_type(objects.values(), 'x-oca-asset') + assert (x_oca_obj.keys() == {'type', 'ip_refs', 'device_id', 'hostname', 'x_is_key_asset', 'x_threat', + 'x_certainty'}) + assert (x_oca_obj is not None), 'x-oca-asset object type not found' + assert x_oca_obj['type'] == 'x-oca-asset' + assert x_oca_obj['ip_refs'] == ['2'] + assert x_oca_obj['hostname'] == 'VMAL #2 windows 11.111.11.111 (higaki-ha11)' + + def test_domain_name(self): + """test domain-name stix object properties""" + objects = TestVectraResultsToStix.get_observed_data_objects(vectra_sample_grouped_response) + domain_obj = TestVectraResultsToStix.get_first_of_type(objects.values(), 'domain-name') + assert (domain_obj.keys() == {'type', 'value'}) + assert (domain_obj is not None), 'domain-name object type not found' + assert domain_obj['type'] == 'domain-name' + assert domain_obj['value'] == '11.111.11.111' + + def test_invalid_ipv6(self): + """test invalid ipv6 stix object properties""" + objects = TestVectraResultsToStix.get_observed_data_objects(vectra_sample_grouped_response) + ipv6_obj = TestVectraResultsToStix.get_first_of_type(objects.values(), 'ipv6-addr') + assert (ipv6_obj is None), 'ipv6 object type not found' diff --git a/stix_shifter_modules/vectra/test/stix_translation/test_vectra_stix_to_query.py b/stix_shifter_modules/vectra/test/stix_translation/test_vectra_stix_to_query.py new file mode 100644 index 000000000..2a3be363c --- /dev/null +++ b/stix_shifter_modules/vectra/test/stix_translation/test_vectra_stix_to_query.py @@ -0,0 +1,509 @@ +from stix_shifter.stix_translation import stix_translation +import unittest +import re + +translation = stix_translation.StixTranslation() + + +def _remove_timestamp_from_query(queries): + pattern = r'detection.last_timestamp:\[([^>]*?)]' + if isinstance(queries, list): + return [re.sub(pattern, '', str(query)) for query in queries] + elif isinstance(queries, str): + return re.sub(pattern, '', queries) + + +class TestQueryTranslator(unittest.TestCase): + """ + class to perform unit test case vectra translate query + """ + if __name__ == "__main__": + unittest.main() + + def _test_query_assertions(self, query, queries): + """ + to assert the each query in the list against expected result + """ + self.assertIsInstance(queries, list) + self.assertIsInstance(query, dict) + self.assertIsInstance(query['queries'], list) + for index, each_query in enumerate(query.get('queries'), start=0): + self.assertEqual(each_query, queries[index]) + + def test_equal_operator(self): + stix_pattern = "[x-ibm-finding:confidence = 22]" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.certainty:"22") AND (detection.last_timestamp:[2023-07-05T0729 to ' + '2023-07-05T0734]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_equal_operator(self): + stix_pattern = "[network-traffic:x_rpc_uuid != 'tp']" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.grouped_details.uuid:* AND NOT detection.grouped_details.uuid:"tp")) ' + 'AND (detection.last_timestamp:[2023-07-05T0803 to 2023-07-05T0808]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_like_operator(self): + stix_pattern = "[domain-name:value LIKE 'google.com']" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.grouped_details.target_domains:google.com OR ' + 'detection.grouped_details.origin_domain:google.com OR ' + 'detection.grouped_details.events.target_domains:google.com OR ' + 'detection.grouped_details.connection_events.target_host.dst_dns:google.com) AND (' + 'detection.last_timestamp:[2023-06-08T1159 to 2023-06-08T1204]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_like_operator(self): + stix_pattern = "[x-ibm-finding:x_state NOT LIKE 'unknown']" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.state:* AND NOT detection.state:unknown)) AND (' + 'detection.last_timestamp:[2023-07-05T0805 to 2023-07-05T0810]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_greater_than_operator(self): + stix_pattern = "[network-traffic:dst_port > 2]" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.grouped_details.dst_ports:>"2" OR ' + 'detection.grouped_details.dst_hosts.dst_port:>"2" OR detection.grouped_details.origin_port:>"2" ' + 'OR detection.grouped_details.sessions.dst_port:>"2" OR ' + 'detection.grouped_details.events.dst_ports:>"2" OR ' + 'detection.grouped_details.events.sessions.dst_port:>"2" OR ' + 'detection.grouped_details.events.target_summary.dst_port:>"2" OR ' + 'detection.grouped_details.connection_events.dst_port:>"2") AND (detection.last_timestamp:[' + '2023-06-26T0839 to 2023-06-26T0844]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_lt_operator(self): + stix_pattern = "[network-traffic:src_byte_count < 120]" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.grouped_details.bytes_sent:<"120" OR ' + 'detection.grouped_details.sessions.bytes_sent:<"120" OR ' + 'detection.grouped_details.events.bytes_sent:<"120" OR ' + 'detection.grouped_details.connection_events.total_bytes_sent:<"120") AND (' + 'detection.last_timestamp:[2023-06-08T1202 to 2023-06-08T1207]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_gt_or_equal_operator(self): + stix_pattern = "[network-traffic:dst_byte_count >= 175]" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.grouped_details.bytes_received:>="175" OR ' + 'detection.grouped_details.sessions.bytes_received:>="175" OR ' + 'detection.grouped_details.events.bytes_received:>="175" OR ' + 'detection.grouped_details.events.sessions.bytes_received:>="175" OR ' + 'detection.grouped_details.connection_events.total_bytes_rcvd:>="175") AND (' + 'detection.last_timestamp:[2023-06-08T1152 to 2023-06-08T1157]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_lt_or_equal_operator(self): + stix_pattern = "[network-traffic:dst_port <= 176]" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.grouped_details.dst_ports:<="176" OR ' + 'detection.grouped_details.dst_hosts.dst_port:<="176" OR ' + 'detection.grouped_details.origin_port:<="176" OR ' + 'detection.grouped_details.sessions.dst_port:<="176" OR ' + 'detection.grouped_details.events.dst_ports:<="176" OR ' + 'detection.grouped_details.events.sessions.dst_port:<="176" OR ' + 'detection.grouped_details.events.target_summary.dst_port:<="176" OR ' + 'detection.grouped_details.connection_events.dst_port:<="176") AND (detection.last_timestamp:[' + '2023-06-26T0834 to 2023-06-26T0839]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_matches_operator(self): + stix_pattern = "[network-traffic:protocols[*] MATCHES 'http']" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.grouped_details.protocol:*http* OR ' + 'detection.grouped_details.app_protocol:*http* OR detection.grouped_details.dst_protocol:*http* OR ' + 'detection.grouped_details.origin_protocol:*http* OR ' + 'detection.grouped_details.sessions.protocol:*http* OR ' + 'detection.grouped_details.sessions.app_protocol:*http* OR ' + 'detection.grouped_details.events.protocol:*http* OR ' + 'detection.grouped_details.events.sessions.app_protocol:*http* OR ' + 'detection.grouped_details.events.sessions.protocol:*http* OR ' + 'detection.grouped_details.events.target_summary.app_protocol:*http* OR ' + 'detection.grouped_details.events.target_summary.protocol:*http* OR ' + 'detection.grouped_details.connection_events.protocol:*http*) AND (detection.last_timestamp:[' + '2023-06-08T1212 to 2023-06-08T1217]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_matches_operator(self): + stix_pattern = "[x-ibm-finding:x_sensor_name NOT MATCHES 'qualys']" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.sensor_name:* AND NOT detection.sensor_name:*qualys*)) AND (' + 'detection.last_timestamp:[2023-07-05T0807 to 2023-07-05T0812]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_in_operator(self): + stix_pattern = "[x-ibm-finding:description IN ('vulnerbility','threat')]" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.description:"vulnerbility" OR detection.description:"threat" OR ' + 'detection.summary.description:"vulnerbility" OR detection.summary.description:"threat") AND (' + 'detection.last_timestamp:[2023-07-05T0731 to 2023-07-05T0736]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_not_in_operator(self): + stix_pattern = "[x-ibm-finding:description NOT IN ('attack','cyber')]" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.description:* AND NOT detection.description:"attack") OR (' + 'detection.description:* AND NOT detection.description:"cyber") OR (' + 'detection.summary.description:* AND NOT detection.summary.description:"attack") OR (' + 'detection.summary.description:* AND NOT detection.summary.description:"cyber")) AND (' + 'detection.last_timestamp:[2023-07-05T0804 to 2023-07-05T0809]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combined_comparison_AND_operator(self): + stix_pattern = "[x-ibm-finding:severity = '25' AND x-ibm-finding:time_observed = '2022-05-11T00:00:00Z']" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.created_timestamp:"2022-05-11T0000") AND (detection.threat:"25")) AND (' + 'detection.last_timestamp:[2023-07-05T0710 to 2023-07-05T0715]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combined_comparison_OR_operator(self): + stix_pattern = "[network-traffic:severity < 5 OR network-traffic:x_rpc_uuid != 'operate'] START " \ + "t'2022-05-15T16:43:26.000Z' STOP t'2023-10-25T16:43:26.003Z'" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.grouped_details.uuid:* AND NOT ' + 'detection.grouped_details.uuid:"operate")) AND ())'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_AND_operator(self): + stix_pattern = "[network-traffic:src_ref.value = 192] AND [network-traffic:protocols[*] = 'tcp'] START " \ + "t'2022-05-15T16:43:26.000Z' STOP t'2023-10-25T16:43:26.003Z'" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.src_ip:"192") AND ()) OR ((detection.grouped_details.protocol:"tcp" OR ' + 'detection.grouped_details.app_protocol:"tcp" OR detection.grouped_details.dst_protocol:"tcp" OR ' + 'detection.grouped_details.origin_protocol:"tcp" OR ' + 'detection.grouped_details.sessions.protocol:"tcp" OR ' + 'detection.grouped_details.sessions.app_protocol:"tcp" OR ' + 'detection.grouped_details.events.protocol:"tcp" OR ' + 'detection.grouped_details.events.sessions.app_protocol:"tcp" OR ' + 'detection.grouped_details.events.sessions.protocol:"tcp" OR ' + 'detection.grouped_details.events.target_summary.app_protocol:"tcp" OR ' + 'detection.grouped_details.events.target_summary.protocol:"tcp" OR ' + 'detection.grouped_details.connection_events.protocol:"tcp") AND ()))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_OR_operator(self): + stix_pattern = "[x-ibm-finding:alert_id = '5678'] OR [x-ibm-finding:severity > 5]" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.id:"5678") AND (detection.last_timestamp:[2023-07-05T0739 to ' + '2023-07-05T0744])) OR ((detection.threat:>"5") AND (detection.last_timestamp:[2023-07-05T0739 to ' + '2023-07-05T0744])))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_combined_comparison(self): + stix_pattern = "([network-traffic:protocol[*] MATCHES 't' AND network-traffic:src_ref.value = '1.1.1.1'] OR [" \ + "x-oca-asset:hostname NOT LIKE '11.34.00.23']) START t'2022-11-07T00:00:01Z' STOP " \ + "t'2023-03-06T11:00:00.003Z'" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.src_host.name:* AND NOT detection.src_host.name:11.34.00.23)) AND (' + 'detection.last_timestamp:[2022-11-07T0000 to 2023-03-06T1100]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_qualifire(self): + stix_pattern = "([network-traffic:src_ref.value = '1.1.1.1'] OR [ x-oca-asset:hostname NOT LIKE " \ + "'11.34.00.23']) START t'2022-11-07T00:00:01Z' STOP t'2023-03-06T11:00:00.003Z'" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.src_ip:"1.1.1.1") AND (detection.last_timestamp:[2022-11-07T0000 to ' + '2023-03-06T1100])) OR (((detection.src_host.name:* AND NOT detection.src_host.name:11.34.00.23)) ' + 'AND (detection.last_timestamp:[2022-11-07T0000 to 2023-03-06T1100])))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_lt_operator_with_timestamp(self): + stix_pattern = "[network-traffic:start < '2023-05-30T00:00:01Z'] START t'2022-11-07T00:00:01Z' STOP " \ + "t'2023-03-06T11:00:00.003Z'" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.first_timestamp:<"2023-05-30T0000" OR ' + 'detection.grouped_details.first_timestamp:<"2023-05-30T0000" OR ' + 'detection.grouped_details.sessions.first_timestamp:<"2023-05-30T0000" OR ' + 'detection.grouped_details.events.first_timestamp:<"2023-05-30T0000" OR ' + 'detection.grouped_details.events.sessions.first_timestamp:<"2023-05-30T0000" OR ' + 'detection.grouped_details.events.target_summary.first_timestamp:<"2023-05-30T0000" OR ' + 'detection.grouped_details.connection_events.first_timestamp:<"2023-05-30T0000") AND (' + 'detection.last_timestamp:[2022-11-07T0000 to 2023-03-06T1100]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_boolean_field(self): + stix_pattern = "[x-ibm-finding:x_is_triaged = 'true']" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=((detection.is_triaged:"true") AND (detection.last_timestamp:[2023-07-05T0708 to ' + '2023-07-05T0713]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_split_OR_query(self): + stix_pattern = "[network-traffic:protocols[*] = 'tcp' OR (ipv4-addr:value = '1.1.1.1')] OR [" \ + "ipv4-addr:value = '11.111.111.11'] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-03-25T16:43:26.003Z'" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [ + 'query_string=(detection.src_ip:"1.1.1.1" OR detection.grouped_details.dst_ips:"1.1.1.1" OR ' + 'detection.grouped_details.dst_hosts.dst_ip:"1.1.1.1" OR ' + 'detection.grouped_details.normal_admin_hosts.ip:"1.1.1.1" OR ' + 'detection.grouped_details.dst_hosts.ip:"1.1.1.1" OR detection.grouped_details.origin_ip:"1.1.1.1" OR ' + 'detection.grouped_details.sessions.dst_ip:"1.1.1.1" OR ' + 'detection.grouped_details.events.dst_ip:"1.1.1.1" OR detection.grouped_details.events.dst_ips:"1.1.1.1" ' + 'OR detection.grouped_details.events.sessions.dst_ip:"1.1.1.1" OR ' + 'detection.grouped_details.connection_events.target_host.ip:"1.1.1.1") AND (detection.last_timestamp:[' + '2023-07-06T1315 to 2023-07-06T1320])', + 'query_string=(detection.grouped_details.protocol:"tcp" OR detection.grouped_details.app_protocol:"tcp" ' + 'OR detection.grouped_details.dst_protocol:"tcp" OR detection.grouped_details.origin_protocol:"tcp" OR ' + 'detection.grouped_details.sessions.protocol:"tcp" OR ' + 'detection.grouped_details.sessions.app_protocol:"tcp" OR detection.grouped_details.events.protocol:"tcp" ' + 'OR detection.grouped_details.events.sessions.app_protocol:"tcp" OR ' + 'detection.grouped_details.events.sessions.protocol:"tcp" OR ' + 'detection.grouped_details.events.target_summary.app_protocol:"tcp" OR ' + 'detection.grouped_details.events.target_summary.protocol:"tcp" OR ' + 'detection.grouped_details.connection_events.protocol:"tcp") AND (detection.last_timestamp:[' + '2023-07-06T1315 to 2023-07-06T1320])', + 'query_string=(detection.src_ip:"11.111.111.11" OR detection.grouped_details.dst_ips:"11.111.111.11" OR ' + 'detection.grouped_details.dst_hosts.dst_ip:"11.111.111.11" OR ' + 'detection.grouped_details.normal_admin_hosts.ip:"11.111.111.11" OR ' + 'detection.grouped_details.dst_hosts.ip:"11.111.111.11" OR ' + 'detection.grouped_details.origin_ip:"11.111.111.11" OR ' + 'detection.grouped_details.sessions.dst_ip:"11.111.111.11" OR ' + 'detection.grouped_details.events.dst_ip:"11.111.111.11" OR ' + 'detection.grouped_details.events.dst_ips:"11.111.111.11" OR ' + 'detection.grouped_details.events.sessions.dst_ip:"11.111.111.11" OR ' + 'detection.grouped_details.connection_events.target_host.ip:"11.111.111.11") AND (' + 'detection.last_timestamp:[2023-02-15T1643 to 2023-03-25T1643])'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_split_IN_query(self): + stix_pattern = "[network-traffic:protocols[*] IN ('tcp','udp') OR (user-account:id != 'ADMIN')] START " \ + "t'2023-02-15T16:43:26.000Z' STOP t'2023-03-25T16:43:26.003Z'" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [ + 'query_string=((detection.grouped_details.protocol:"tcp" OR detection.grouped_details.protocol:"udp" OR ' + 'detection.grouped_details.app_protocol:"tcp" OR detection.grouped_details.app_protocol:"udp" OR ' + 'detection.grouped_details.dst_protocol:"tcp" OR detection.grouped_details.dst_protocol:"udp" OR ' + 'detection.grouped_details.origin_protocol:"tcp" OR detection.grouped_details.origin_protocol:"udp" OR ' + 'detection.grouped_details.sessions.protocol:"tcp" OR detection.grouped_details.sessions.protocol:"udp" ' + 'OR detection.grouped_details.sessions.app_protocol:"tcp" OR ' + 'detection.grouped_details.sessions.app_protocol:"udp" OR detection.grouped_details.events.protocol:"tcp" ' + 'OR detection.grouped_details.events.protocol:"udp" OR ' + 'detection.grouped_details.events.sessions.app_protocol:"tcp" OR ' + 'detection.grouped_details.events.sessions.app_protocol:"udp" OR ' + 'detection.grouped_details.events.sessions.protocol:"tcp" )) AND (detection.last_timestamp:[' + '2023-02-15T1643 to 2023-03-25T1643])', + 'query_string=(( detection.grouped_details.events.sessions.protocol:"udp" OR ' + 'detection.grouped_details.events.target_summary.app_protocol:"tcp" OR ' + 'detection.grouped_details.events.target_summary.app_protocol:"udp" OR ' + 'detection.grouped_details.events.target_summary.protocol:"tcp" OR ' + 'detection.grouped_details.events.target_summary.protocol:"udp" OR ' + 'detection.grouped_details.connection_events.protocol:"tcp" OR ' + 'detection.grouped_details.connection_events.protocol:"udp")) AND (detection.last_timestamp:[' + '2023-02-15T1643 to 2023-03-25T1643])'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_split_multiple_operator(self): + stix_pattern = "[ipv4-addr:value != '1.1.1.1' OR (network-traffic:x_time_duration > 4000 AND " \ + "network-traffic:x_time_duration < 2000)] START t'2023-05-15T16:43:26.000Z' STOP " \ + "t'2023-06-25T16:43:26.003Z'" + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = [ + 'query_string=((detection.grouped_details.duration:<"2000" OR ' + 'detection.grouped_details.events.duration:<"2000" OR ' + 'detection.grouped_details.events.sessions.duration:<"2000" OR ' + 'detection.grouped_details.connection_events.duration_int:<"2000") AND (' + 'detection.grouped_details.duration:>"4000" OR detection.grouped_details.events.duration:>"4000" OR ' + 'detection.grouped_details.events.sessions.duration:>"4000" OR ' + 'detection.grouped_details.connection_events.duration_int:>"4000") OR (detection.src_ip:* AND NOT ' + 'detection.src_ip:"1.1.1.1") OR (detection.grouped_details.dst_ips:* AND NOT ' + 'detection.grouped_details.dst_ips:"1.1.1.1") OR (detection.grouped_details.dst_hosts.dst_ip:* AND NOT ' + 'detection.grouped_details.dst_hosts.dst_ip:"1.1.1.1") OR (' + 'detection.grouped_details.normal_admin_hosts.ip:* AND NOT ' + 'detection.grouped_details.normal_admin_hosts.ip:"1.1.1.1")) AND (detection.last_timestamp:[' + '2023-05-15T1643 to 2023-06-25T1643])', + 'query_string=((detection.grouped_details.dst_hosts.ip:* AND NOT ' + 'detection.grouped_details.dst_hosts.ip:"1.1.1.1") OR (detection.grouped_details.origin_ip:* AND NOT ' + 'detection.grouped_details.origin_ip:"1.1.1.1") OR (detection.grouped_details.sessions.dst_ip:* AND NOT ' + 'detection.grouped_details.sessions.dst_ip:"1.1.1.1") OR (detection.grouped_details.events.dst_ip:* AND NOT ' + 'detection.grouped_details.events.dst_ip:"1.1.1.1") OR (detection.grouped_details.events.dst_ips:* AND ' + 'NOT detection.grouped_details.events.dst_ips:"1.1.1.1") OR (' + 'detection.grouped_details.events.sessions.dst_ip:* AND NOT ' + 'detection.grouped_details.events.sessions.dst_ip:"1.1.1.1") OR (' + 'detection.grouped_details.connection_events.target_host.ip:* AND NOT ' + 'detection.grouped_details.connection_events.target_host.ip:"1.1.1.1")) AND (detection.last_timestamp:[' + '2023-05-15T1643 to 2023-06-25T1643])'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_split_contain_AND_query(self): + stix_pattern = "[ipv4-addr:value != '1.1.1.1' AND network-traffic:x_time_duration < 2000] START " \ + "t'2023-05-15T16:43:26.000Z' STOP t'2023-06-25T16:43:26.003Z'" + + query = translation.translate('vectra', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['query_string=(((detection.grouped_details.duration:<"2000" OR ' + 'detection.grouped_details.events.duration:<"2000" OR ' + 'detection.grouped_details.events.sessions.duration:<"2000" OR ' + 'detection.grouped_details.connection_events.duration_int:<"2000") AND ((detection.src_ip:* AND ' + 'NOT detection.src_ip:"1.1.1.1") OR (detection.grouped_details.dst_ips:* AND NOT ' + 'detection.grouped_details.dst_ips:"1.1.1.1") OR (detection.grouped_details.dst_hosts.dst_ip:* AND ' + 'NOT detection.grouped_details.dst_hosts.dst_ip:"1.1.1.1") OR (' + 'detection.grouped_details.normal_admin_hosts.ip:* AND NOT ' + 'detection.grouped_details.normal_admin_hosts.ip:"1.1.1.1") OR (' + 'detection.grouped_details.dst_hosts.ip:* AND NOT ' + 'detection.grouped_details.dst_hosts.ip:"1.1.1.1") OR (detection.grouped_details.origin_ip:* AND ' + 'NOT detection.grouped_details.origin_ip:"1.1.1.1") OR (' + 'detection.grouped_details.sessions.dst_ip:* AND NOT ' + 'detection.grouped_details.sessions.dst_ip:"1.1.1.1") OR (detection.grouped_details.events.dst_ip:* AND ' + 'NOT detection.grouped_details.events.dst_ip:"1.1.1.1") OR (' + 'detection.grouped_details.events.dst_ips:* AND NOT ' + 'detection.grouped_details.events.dst_ips:"1.1.1.1") OR (' + 'detection.grouped_details.events.sessions.dst_ip:* AND NOT ' + 'detection.grouped_details.events.sessions.dst_ip:"1.1.1.1") OR (' + 'detection.grouped_details.connection_events.target_host.ip:* AND NOT ' + 'detection.grouped_details.connection_events.target_host.ip:"1.1.1.1"))) AND (' + 'detection.last_timestamp:[2023-05-15T1643 to 2023-06-25T1643]))'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_invalid_matches_operator(self): + stix_pattern = "[network-traffic:src_byte_count MATCHES '120']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : MATCHES operator is supported only for ' \ + 'string type input' + + def test_invalid_matches_symbol(self): + stix_pattern = "[x-ibm-finding:x_sensor_name NOT MATCHES 'qualys^']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : ^ symbol should be at the starting ' \ + 'position of the expression' + + def test_invalid_matches_contains_dollar(self): + stix_pattern = "[network-traffic:protocols[*] MATCHES '$cd']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : $ symbol should be at the ending ' \ + 'position of the expression' + + def test_like_operator_for_integer_field(self): + stix_pattern = "[network-traffic:dst_port LIKE '22']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : LIKE operator is supported only for ' \ + 'string type input' + + def test_string_input_for_integer_fields(self): + stix_pattern = "[network-traffic:dst_port = 'none' ]" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : string type input - none is not ' \ + 'supported for integer type fields' + + def test_invalid_timestamp_format(self): + stix_pattern = "[network-traffic:start < '2023'] START t'2022-11-07T00:00:01Z' STOP t'2023-03-06T11:00:00.003Z'" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : cannot format the timestamp 2023' + + def test_invalid_boolean_operator(self): + stix_pattern = "[x-ibm-finding:x_is_triaged > 'True']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert 'vectra connector error => wrong parameter : :> operator is not supported for Boolean type input. ' \ + 'Possible supported operator are [ =, !=, IN, NOT IN ]' + + def test_invalid_boolean(self): + stix_pattern = "[x-ibm-finding:x_is_triaged = '10']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : Boolean type field allows only ' \ + 'true/false' + + def test_invalid_timestamp(self): + stix_pattern = "[network-traffic:dst_port = 38] START t'0000-03-01T11:00:00.003Z' STOP " \ + "t'2023-03-13T11:00:00.003Z'" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : cannot format the timestamp ' \ + '0000-03-01T11:00:00.003Z' + + def test_invalid_compare_operator(self): + stix_pattern = "[x-ibm-finding:description > 'important']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : > operator is not supported for string' \ + ' type input' + + def test_invalid_match_operator(self): + stix_pattern = "[user-account:user_id MATCHES 'AR EX*']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : MATCHES operator is not supported for ' \ + 'value contains spaces' + + def test_invalid_like_operator(self): + stix_pattern = "[network-traffic:x_reason LIKE 'LO ']" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : LIKE operator is not supported for ' \ + 'value contains spaces' + + def test_invalid_not_operator(self): + stix_pattern = "[network-traffic:dst_port != 22]" + result = translation.translate('vectra', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'vectra connector error => wrong parameter : Not operator is only supported for ' \ + 'string type fields' diff --git a/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py b/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py new file mode 100644 index 000000000..66d6c75ed --- /dev/null +++ b/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py @@ -0,0 +1,463 @@ +import json +import unittest +from unittest.mock import patch +from stix_shifter.stix_transmission import stix_transmission +from stix_shifter.stix_transmission.stix_transmission import run_in_thread +from stix_shifter_modules.vectra.entry_point import EntryPoint +from tests.utils.async_utils import get_mock_response + + +class VectraMockResponse: + """ class for vectra mock response""" + + def __init__(self, code, data, headers): + self.code = code + self.content = data + self.headers = headers + + def read(self): + return bytearray(self.content, 'utf-8') + + +class TestVectraConnection(unittest.TestCase, object): + mocked_response = { + 'count': 23, + 'results': [{ + 'summary': { + 'dst_ips': ['11.111.11.111'], + 'num_sessions': 16, + 'bytes_sent': 48927848, + 'bytes_received': 638510042, + 'description': 'This host communicated with an external destination using HTTPS.' + }, + 'id': 9124, + 'category': 'COMMAND & CONTROL', + 'detection': 'Hidden HTTPS Tunnel', + 'detection_category': 'COMMAND & CONTROL', + 'detection_type': 'Hidden HTTPS Tunnel', + 'custom_detection': None, + 'description': None, + 'src_ip': '11.111.11.111', + 'state': 'inactive', + 'certainty': 0, + 'threat': 0, + 'created_timestamp': '2022-12-15T09:28:39Z', + 'first_timestamp': '2022-12-15T09:18:32Z', + 'last_timestamp': '2022-12-16T01:33:25Z', + 'targets_key_asset': False, + 'is_targeting_key_asset': False, + 'src_account': None, + 'src_host': { + 'id': 1481, + 'ip': '11.111.11.111', + 'name': 'VMAL #2 windows 11.111.11.111 (fschmidt114)', + 'is_key_asset': False, + 'groups': [{ + 'id': 144, + 'name': 'Partner VLAB - User Devices', + 'description': '', + 'last_modified': '2022-01-27T12:05:24Z', + 'last_modified_by': 'user (Removed)', + 'type': 'ip' + }], + 'threat': 0, + 'certainty': 0 + }, + 'note': None, + 'note_modified_by': None, + 'note_modified_timestamp': None, + 'sensor': 'test', + 'sensor_name': 'test', + 'tags': [], + 'triage_rule_id': None, + 'assigned_to': 'crest', + 'assigned_date': '2023-03-29T15:25:23Z', + 'groups': [{ + 'id': 144, + 'name': 'Partner VLAB - User Devices', + 'description': '', + 'type': 'ip', + 'last_modified': '2022-01-27T12:05:24Z', + 'last_modified_by': 'user' + }], + 'is_marked_custom': False, + 'is_custom_model': False, + 'src_linked_account': None, + 'grouped_details': [{ + 'external_target': { + 'ip': '11.111.11.111', + 'name': '' + }, + 'num_sessions': 16, + 'bytes_received': 638510042, + 'bytes_sent': 48927848, + 'ja3_hashes': [''], + 'ja3s_hashes': [''], + 'sessions': [{ + 'tunnel_type': 'Long TCP session - Command line', + 'dst_port': 222, + 'dst_ip': '11.111.11.111', + 'bytes_received': 40495217, + 'bytes_sent': 4108613, + 'first_timestamp': '2022-12-16T01:22:32Z', + 'last_timestamp': '2022-12-16T01:33:25Z', + 'dst_geo': None, + 'dst_geo_lat': None, + 'dst_geo_lon': None + }, { + 'tunnel_type': 'Long TCP session - Command line', + 'protocol': 'tcp', + 'app_protocol': 'https', + 'dst_port': 222, + 'dst_ip': '11.111.11.111', + 'bytes_received': 40483751, + 'bytes_sent': 4087475, + 'first_timestamp': '2022-12-16T01:09:32Z', + 'last_timestamp': '2022-12-16T01:20:25Z', + 'dst_geo': None, + 'dst_geo_lat': None, + 'dst_geo_lon': None + }], + 'first_timestamp': '2022-12-15T09:18:32Z', + 'last_timestamp': '2022-12-16T01:33:25Z', + 'dst_ips': ['11.111.11.111'], + 'dst_ports': [222], + 'target_domains': [''] + }, {}], + 'campaign_summaries': [], + 'is_triaged': False, + 'filtered_by_ai': False, + 'filtered_by_user': False, + 'filtered_by_rule': False, + '_doc_modified_ts': '2023-06-08T00:21:13.669422' + }, { + 'id': 12052, + 'category': 'EXFILTRATION', + 'detection': 'Data Smuggler', + 'detection_category': 'EXFILTRATION', + 'detection_type': 'Data Smuggler', + 'custom_detection': None, + 'description': None, + 'src_ip': '11.111.11.111', + 'state': 'inactive', + 'certainty': 0, + 'threat': 0, + 'created_timestamp': '2023-01-12T14:14:38Z', + 'first_timestamp': '2023-01-12T13:51:28Z', + 'last_timestamp': '2023-01-12T14:09:10Z', + 'targets_key_asset': False, + 'is_targeting_key_asset': False, + 'src_account': None, + 'src_host': { + 'id': 1586, + 'ip': '11.111.11.111', + 'name': '11.111.11.111', + 'is_key_asset': False, + 'groups': [{ + 'id': 144, + 'name': 'Partner VLAB - User Devices', + 'description': '', + 'last_modified': '2022-01-27T12:05:24Z', + 'last_modified_by': 'user (Removed)', + 'type': 'ip' + }], + 'threat': 0, + 'certainty': 0 + }, + 'note': '2023-01-24 01:52:14 - Vectra SIR Admin (Notes)\nNotes added from ServiceNow for testing.\n\n', + 'note_modified_by': 'crest', + 'note_modified_timestamp': '2023-01-24T09:52:16Z', + 'sensor': 'eti2pc2s', + 'sensor_name': 'Vec2c610896a947c5b5102c466a28f49a', + 'tags': ['test', 'test1'], + 'triage_rule_id': None, + 'assigned_to': 'test', + 'assigned_date': '2023-01-23T06:25:09Z', + 'groups': [{ + 'id': 144, + 'name': 'Partner VLAB - User Devices', + 'description': '', + 'type': 'ip', + 'last_modified': '2022-01-27T12:05:24Z', + 'last_modified_by': 'user' + }], + 'is_marked_custom': False, + 'is_custom_model': False, + 'src_linked_account': None, + 'grouped_details': [{ + 'events': [{ + 'grouping_field': 'multi_fields', + 'multi_fields': '3yO12k8i', + 'id': 1664746, + 'subtype': 'pull', + 'first_timestamp': '2023-01-12T13:51:28Z', + 'last_timestamp': '2023-01-12T14:04:30Z', + 'couch_note_id': ['3yO12k8i'], + 'flex5': [], + 'app_protocol': None, + 'target_domains': ['test.com'], + 'bytes_received': 3250323735, + 'bytes_sent': 428748, + 'proxy_ip': None, + 'sessions': [{ + 'target_host': { + 'id': 325, + 'ip': '11.111.111.11', + 'name': 'VMAL #2 - test', + 'is_key_asset': False + }, + 'dst_ip': '11.111.111.11', + 'dst_port': 444, + 'protocol': 'tcp', + 'app_protocol': 'test', + 'bytes_received': 3249935855, + 'first_timestamp': '2023-01-12T14:02:58Z', + 'duration': 92 + }], + 'origin_ips': ['11.111.111.111'], + 'target_summary': { + 'dst_port': 22, + 'protocol': 'tcp', + 'app_protocol': 'sss', + 'bytes_sent': 3313292524, + 'first_timestamp': '2023-01-12T14:04:05Z', + 'last_timestamp': '2023-01-12T14:09:10Z' + } + }], + 'multi_fields': ['', '10.250.20.137'], + 'first_timestamp': '2023-01-12T14:04:05Z', + 'last_timestamp': '2023-01-12T14:09:10Z', + 'app_protocol': None, + 'dst_ips': ['11.111.11.11'], + 'target_domains': [], + 'bytes_received': 33621147, + 'bytes_sent': 3313292524, + 'proxy_ip': None + }], + 'summary': { + 'dst_ports': [222], + 'protocols': ['tcp'], + 'bytes_sent': 3313292524, + 'dst_ips': ['11.111.11.111'] + }, + 'campaign_summaries': [], + 'is_triaged': False, + 'filtered_by_ai': False, + 'filtered_by_user': False, + 'filtered_by_rule': False, + '_doc_modified_ts': '2023-06-25T22:01:18.599393' + }, + { + 'summary': { + 'artifact': ['VMAL #2 windows 11.111.11.111'], + 'last_timestamp': '2023-01-23T20:34:39Z', + 'description': 'This is the first time this host has been seen on the network.' + }, + 'id': 12058, + 'category': 'INFO', + 'detection': 'New Host', + 'detection_category': 'INFO', + 'detection_type': 'New Host', + 'custom_detection': None, + 'description': None, + 'src_ip': '11.111.11.111', + 'state': 'inactive', + 'certainty': 0, + 'threat': 0, + 'created_timestamp': '2023-01-23T20:40:28Z', + 'first_timestamp': '2023-01-23T20:34:39Z', + 'last_timestamp': '2023-01-23T20:34:39Z', + 'targets_key_asset': False, + 'is_targeting_key_asset': False, + 'src_account': None, + 'src_host': { + 'id': 1609, + 'ip': '11.111.11.111', + 'name': 'VMAL #2 windows 11.111.11.111 (test142)', + 'is_key_asset': False, + 'groups': [{ + 'id': 144, + 'name': 'Partner VLAB - Devices', + 'description': '', + 'last_modified': '2022-01-27T12:05:24Z', + 'last_modified_by': 'user (Removed)', + 'type': 'ip' + }], + 'threat': 0, + 'certainty': 0 + }, + 'note': None, + 'note_modified_by': None, + 'note_modified_timestamp': None, + 'sensor': 'None', + 'sensor_name': 'vlab', + 'tags': [], + 'triage_rule_id': None, + 'assigned_to': None, + 'assigned_date': None, + 'groups': [{ + 'id': 144, + 'name': 'Partner VLAB - User Devices', + 'description': '', + 'type': 'ip', + 'last_modified': '2022-01-27T12:05:24Z', + 'last_modified_by': 'user' + }], + 'is_marked_custom': False, + 'is_custom_model': False, + 'src_linked_account': None, + 'grouped_details': [{ + 'artifact': 'VMAL #2 windows 11.111.11.111 (test142)', + 'via': 'AWS Name', + 'last_timestamp': '2023-01-23T20:34:39Z' + }, { + 'artifact': 'test3', + 'via': 'AWS Resource Name', + 'last_timestamp': '2023-01-23T20:34:39Z' + }, { + 'artifact': 'test142', + 'via': 'Ker', + 'last_timestamp': '2023-01-23T20:34:39Z' + }], + 'campaign_summaries': [], + 'is_triaged': False, + 'filtered_by_ai': False, + 'filtered_by_user': False, + 'filtered_by_rule': False, + '_doc_modified_ts': '2023-06-25T22:01:19.318083' + } + ], + 'previous': None, + 'next': 'https://test1/api/v2.4/search/detections/?page_size=9&query_string=(detection.detection:"Hidden ' + 'HTTPS Tunnel" AND &page=2'} + + def connection(self): + """format for connection""" + return {"host": "hostbla"} + + def configuration(self): + """format for configuration""" + return {"auth": {"api_token": "u"}} + + def test_is_async(self): + """check for synchronous or asynchronous""" + entry_point = EntryPoint(self.connection(), self.configuration()) + check_async = entry_point.is_async() + assert check_async is False + + @patch('stix_shifter_modules.vectra.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_results(self, mock_ping_response): + """test ping connection""" + mock_ping_response.return_value = get_mock_response(200, {}, 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_response = run_in_thread(entry_point.ping_connection) + assert ping_response is not None + assert ping_response['success'] is True + + @patch('stix_shifter_modules.vectra.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_invalid_auth(self, mock_ping_response): + """test ping connection""" + mock_ping_response.return_value = get_mock_response(401, {}, 'byte') + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_response = run_in_thread(entry_point.ping_connection) + assert ping_response is not None + assert ping_response['success'] is False + assert ping_response['code'] == 'authentication_fail' + assert 'vectra connector error => Invalid Authentication' == ping_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_ping_timeout(self, mock_ping_response): + """Test timeout exception for ping""" + mock_ping_response.side_effect = Exception("timeout_error") + transmission = stix_transmission.StixTransmission('vectra', self.connection(), self.configuration()) + result_response = transmission.ping() + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert 'vectra connector error => timeout_error' == result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_invalid_auth(self, mock_results_response): + """Test invalid authentication for results""" + query = 'query_string=(detection.certainty:"22" AND (detection.last_timestamp:[2023-06-07T0640 to ' \ + '2023-06-07T0645]))' + mock_response = json.dumps({"count": 1, "results": [], "previous": None, "next": None}) + mock_results_response.return_value = get_mock_response(401, mock_response, 'byte') + transmission = stix_transmission.StixTransmission('vectra', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert result_response['code'] == 'authentication_fail' + assert 'vectra connector error => Invalid Authentication' == result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_result_timeout(self, mock_result_response): + """Test timeout exception for results""" + mock_result_response.side_effect = Exception("timeout_error") + query = 'query_string=((detection.grouped_details.target_domains:google.com OR ' \ + 'detection.grouped_details.origin_domain:google.com) AND (detection.last_timestamp:[2023-06-07T0808 ' \ + 'to 2023-06-07T0813]))' + transmission = stix_transmission.StixTransmission('vectra', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert 'vectra connector error => timeout_error' in result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_result_invalid_parameters(self, mock_results_response): + """Test invalid parameters for results""" + query = 'query_string=(NOT ((detection.description:"attack" OR detection.description:"cyber") OR (' \ + 'detection.summary.description:"attack" OR detection.summary.description:"cyber")) AND (' \ + 'detection.last_timestamp:[2023-06-07T0952 to 2023-06-07T0957]))' + mock_response = json.dumps({'errors': [{'type': 'InvalidQueryParamsError', 'title': 'Invalid page_size'}]}) + mock_results_response.return_value = get_mock_response(422, mock_response, 'byte') + transmission = stix_transmission.StixTransmission('vectra', self.connection(), self.configuration()) + offset = 16 + length = 10 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert result_response['code'] == 'invalid_parameter' + assert 'vectra connector error => Invalid Parameters' == result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_result_invalid_query(self, mock_results_response): + """Test invalid query for results""" + query = 'query_string=(((detection.description:"vulnerability" OR detection.description:"threat") OR (' \ + 'detection.summary.description:"vulnerbility" OR detection.summary.description:"threat")) AND (' \ + 'detection.last_timestamp:[2023-06-07T1024 to 2023-06-07T1029]))' + mock_response = json.dumps({'errors': [{'title': 'Field "detection1.last_timestamp" not found in "detections" ' + 'index', 'offset': 1, + 'field_name': 'detection1.last_timestamp'}]}) + mock_results_response.return_value = get_mock_response(422, mock_response, 'byte') + transmission = stix_transmission.StixTransmission('vectra', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert result_response['code'] == 'invalid_query' + assert 'vectra connector error => Invalid Query' == result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_result_response(self, mock_results_response): + """Test invalid query for results""" + query = '(detection.detection:"Hidden HTTPS Tunnel" AND (detection.last_timestamp:[2022-11-16T0000 to ' \ + '2023-05-23T0000]))' + mock_response_1 = json.dumps(self.mocked_response) + self.mocked_response['next'] = None + mock_response_2 = json.dumps(self.mocked_response) + mock_results_response.side_effect = [get_mock_response(200, mock_response_1, 'byte'), + get_mock_response(200, mock_response_2, 'byte')] + transmission = stix_transmission.StixTransmission('vectra', self.connection(), self.configuration()) + offset = 20 + length = 10 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert result_response['data'] is not None diff --git a/stix_shifter_modules/vectra/vectra_supported_stix.md b/stix_shifter_modules/vectra/vectra_supported_stix.md new file mode 100644 index 000000000..927b7927f --- /dev/null +++ b/stix_shifter_modules/vectra/vectra_supported_stix.md @@ -0,0 +1,385 @@ +##### Updated on 19/07/23 +## Vectra NDR +### Results STIX Domain Objects +* Identity +* Observed Data +
+### Supported STIX Operators +*Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).* + +| STIX Operator | Data Vectra Operator | +|--|--| +| AND (Comparision) | AND | +| OR (Comparision) | OR | +| = | : | +| != | : | +| IN | : | +| MATCHES | :* | +| > | :> | +| >= | :>= | +| < | :< | +| <= | :<= | +| LIKE | : | +| OR (Observation) | OR | +| AND (Observation) | OR | +|
| | +### Searchable STIX objects and properties +| STIX Object and Property | Mapped Data Source Fields | +|--|--| +| **ipv4-addr**:value | detection.src_ip, detection.grouped_details.dst_ips, detection.grouped_details.dst_hosts.dst_ip, detection.grouped_details.normal_admin_hosts.ip, detection.grouped_details.dst_hosts.ip, detection.grouped_details.origin_ip, detection.grouped_details.sessions.dst_ip, detection.grouped_details.events.dst_ip, detection.grouped_details.events.dst_ips, detection.grouped_details.events.sessions.dst_ip, detection.grouped_details.connection_events.target_host.ip | +| **ipv6-addr**:value | detection.src_ip, detection.grouped_details.dst_ips, detection.grouped_details.dst_hosts.dst_ip, detection.grouped_details.normal_admin_hosts.ip, detection.grouped_details.origin_ip, detection.grouped_details.sessions.dst_ip, detection.grouped_details.events.dst_ip, detection.grouped_details.events.dst_ips, detection.grouped_details.events.sessions.dst_ip, detection.grouped_details.connection_events.target_host.ip | +| **domain-name**:value | detection.grouped_details.target_domains, detection.grouped_details.origin_domain, detection.grouped_details.events.target_domains, detection.grouped_details.connection_events.target_host.dst_dns | +| **network-traffic**:dst_port | detection.grouped_details.dst_ports, detection.grouped_details.dst_hosts.dst_port, detection.grouped_details.origin_port, detection.grouped_details.sessions.dst_port, detection.grouped_details.events.dst_port, detection.grouped_details.events.sessions.dst_port, detection.grouped_details.events.target_summary.dst_port, detection.grouped_details.connection_events.dst_port | +| **network-traffic**:src_port | detection.grouped_details.src_port | +| **network-traffic**:src_ref.value | detection.src_ip | +| **network-traffic**:dst_ref.value | detection.grouped_details.dst_ips, detection.grouped_details.dst_hosts.dst_ip, detection.grouped_details.sessions.dst_ip, detection.grouped_details.origin_ip, detection.grouped_details.events.sessions.dst_ip, detection.grouped_details.connection_events.target_host.ip | +| **network-traffic**:protocols[*] | detection.grouped_details.protocol, detection.grouped_details.app_protocol, detection.grouped_details.dst_protocol, detection.grouped_details.origin_protocol, detection.grouped_details.sessions.protocol, detection.grouped_details.sessions.app_protocol, detection.grouped_details.events.protocol, detection.grouped_details.events.sessions.app_protocol, detection.grouped_details.events.sessions.protocol, detection.grouped_details.events.target_summary.app_protocol, detection.grouped_details.events.target_summary.protocol, detection.grouped_details.connection_events.protocol | +| **network-traffic**:src_byte_count | detection.grouped_details.bytes_sent, detection.grouped_details.sessions.bytes_sent, detection.grouped_details.events.bytes_sent, detection.grouped_details.connection_events.total_bytes_sent | +| **network-traffic**:dst_byte_count | detection.grouped_details.bytes_received, detection.grouped_details.sessions.bytes_received, detection.grouped_details.events.bytes_received, detection.grouped_details.events.sessions.bytes_received, detection.grouped_details.connection_events.total_bytes_rcvd | +| **network-traffic**:start | detection.first_timestamp, detection.grouped_details.first_timestamp, detection.grouped_details.sessions.first_timestamp, detection.grouped_details.events.first_timestamp, detection.grouped_details.events.sessions.first_timestamp, detection.grouped_details.events.target_summary.first_timestamp, detection.grouped_details.connection_events.first_timestamp | +| **network-traffic**:end | detection.last_timestamp, detection.grouped_details.last_timestamp, detection.grouped_details.dst_hosts.last_timestamp, detection.grouped_details.sessions.last_timestamp, detection.grouped_details.events.last_seen, detection.grouped_details.events.last_timestamp, detection.grouped_details.events.target_summary.last_timestamp, detection.grouped_details.connection_events.last_timestamp | +| **network-traffic**:x_count | detection.grouped_details.events.count | +| **network-traffic**:x_dst_country | detection.grouped_details.events.dst_country | +| **network-traffic**:x_num_accounts | detection.grouped_details.num_accounts | +| **network-traffic**:x_reason | detection.grouped_details.reason | +| **network-traffic**:x_num_attempts | detection.grouped_details.num_attempts | +| **network-traffic**:x_tunnel_type | detection.grouped_details.sessions.tunnel_type | +| **network-traffic**:x_num_sessions | detection.grouped_details.num_sessions | +| **network-traffic**:x_user_agent | detection.grouped_details.user_agent | +| **network-traffic**:x_dst_geo_latitude | detection.grouped_details.dst_geo_lat, detection.grouped_details.origin_geo_lat, detection.grouped_details.sessions.dst_geo_lat | +| **network-traffic**:x_dst_geo_longitude | detection.grouped_details.dst_geo_lon, detection.grouped_details.origin_geo_lon, detection.grouped_details.sessions.dst_geo_lon | +| **network-traffic**:x_dst_geo | detection.grouped_details.dst_geo, detection.grouped_details.origin_geo, detection.grouped_details.sessions.dst_geo | +| **network-traffic**:x_num_response_objects | detection.grouped_details.num_response_objects | +| **network-traffic**:x_client_name | detection.grouped_details.client_name | +| **network-traffic**:x_client_token | detection.grouped_details.client_token | +| **network-traffic**:x_is_normally_accessed_by_rdp | detection.grouped_details.events.is_normally_accessed_by_rdp | +| **network-traffic**:x_rpc_uuid | detection.grouped_details.uuid | +| **network-traffic**:x_nt_referrer | detection.grouped_details.events.referrer | +| **network-traffic**:x_num_events | detection.grouped_details.num_events | +| **network-traffic**:x_time_duration | detection.grouped_details.duration, detection.grouped_details.events.duration, detection.grouped_details.events.sessions.duration, detection.grouped_details.connection_events.duration | +| **network-traffic**:x_status_code | detection.grouped_details.status_code | +| **network-traffic**:x_named_pipe | detection.grouped_details.named_pipe | +| **network-traffic**:x_uri | detection.grouped_details.uri | +| **network-traffic**:x_src_session_uid | detection.grouped_details.metadata.orig_sluid | +| **network-traffic**:x_executed_functions | detection.grouped_details.executed_functions | +| **network-traffic**:x_event_type | detection.grouped_details.events.event_type | +| **network-traffic**:x_error_code | detection.grouped_details.events.error_code | +| **network-traffic**:x_target_domain_refs[*].value | detection.grouped_details.events.target_domains | +| **network-traffic**:x_is_external | detection.grouped_details.connection_events.is_external | +| **network-traffic**:x_request_uri | detection.grouped_details.uri | +| **network-traffic**:x_period_identified | detection.grouped_details.period_identified | +| **network-traffic**:x_smb_share | detection.grouped_details.share | +| **network-traffic**:x_account_uid | detection.grouped_details.account_uid | +| **network-traffic**:extensions.'http-request-ext'.request_method | detection.grouped_details.events.http_method | +| **network-traffic**:extensions.'http-request-ext'.x_response_code | detection.grouped_details.events.response_code | +| **network-traffic**:extensions.'http-request-ext'.request_header.'User-Agent' | detection.grouped_details.user_agent | +| **user-account**:user_id | detection.grouped_details.dst_accounts.uid, detection.grouped_details.src_account.name, detection.grouped_details.normal_users, detection.summary.accounts | +| **user-account**:account_login | detection.grouped_details.src_account.name | +| **user-account**:x_privilege_category | detection.grouped_details.src_account.privilege_category | +| **user-account**:x_privilege_level | detection.grouped_details.src_account.privilege_level | +| **x-ibm-finding**:name | detection.detection_type | +| **x-ibm-finding**:alert_id | detection.id | +| **x-ibm-finding**:description | detection.description, detection.summary.description | +| **x-ibm-finding**:x_num_sessions | detection.grouped_details.num_sessions | +| **x-ibm-finding**:severity | detection.threat | +| **x-ibm-finding**:confidence | detection.certainty | +| **x-ibm-finding**:start | detection.first_timestamp | +| **x-ibm-finding**:end | detection.last_timestamp | +| **x-ibm-finding**:time_observed | detection.created_timestamp | +| **x-ibm-finding**:event_count | detection.summary.num_sessions, detection.summary.num_attempts | +| **x-ibm-finding**:x_state | detection.state | +| **x-ibm-finding**:x_num_successes | detection.summary.num_successes | +| **x-ibm-finding**:x_assigned_to | detection.assigned_to | +| **x-ibm-finding**:x_assigned_date | detection.assigned_date | +| **x-ibm-finding**:x_sensor_name | detection.sensor_name | +| **x-ibm-finding**:x_is_triaged | detection.is_triaged | +| **x-ibm-finding**:src_ip_ref | detection.src_ip | +| **x-ibm-finding**:x_dst_ports | detection.summary.dst_ports | +| **x-ibm-finding**:x_account_refs.user_id | detection.summary.accounts | +| **x-ibm-finding**:x_shares | detection.summary.shares | +| **x-ibm-finding**:x_probable_owner | detection.summary.probable_owner | +| **x-ibm-finding**:x_matches | detection.summary.matches | +| **x-ibm-ttp-tagging**:name | detection.detection_category | +| **x-ibm-ttp-tagging**:confidence | detection.certainty | +| **x-ibm-ttp-tagging**:kill_chain_phases.phase_name | detection.detection_category | +| **x-oca-asset**:hostname | detection.src_host.name | +| **x-oca-asset**:device_id | detection.src_host.id | +| **x-oca-asset**:x_is_key_asset | detection.src_host.is_key_asset | +| **x-oca-asset**:ip_refs[*].value | detection.src_ip | +| **x-oca-asset**:x_threat | detection.src_host.threat | +| **x-oca-asset**:x_certainty | detection.src_host.certainty | +| **x-oca-asset**:x_privilege_category | detection.grouped_details.src_host.privilege_category | +| **x-oca-asset**:x_privilege_level | detection.grouped_details.src_host.privilege_level | +| **x-grouped-details**:first_seen | detection.grouped_details.first_seen | +| **x-grouped-details**:last_seen | detection.grouped_details.last_seen | +| **x-grouped-details**:detection_source | detection.grouped_details.detection_source | +| **x-grouped-details**:detection_slug | detection.grouped_details.detection_slug | +| **x-grouped-details**:account_ref.user_id | detection.grouped_details.src_account.id | +| **x-grouped-details**:ja3_hashes | detection.grouped_details.ja3_hashes | +| **x-grouped-details**:ja3s_hashes | detection.grouped_details.ja3s_hashes | +| **x-grouped-details**:x_num_sessions | detection.grouped_details.num_sessions | +| **x-grouped-details**:start | detection.grouped_details.first_timestamp | +| **x-grouped-details**:end | detection.grouped_details.last_timestamp | +| **x-grouped-details**:count | detection.grouped_details.count | +| **x-grouped-details**:client_name | detection.grouped_details.client_name | +| **x-grouped-details**:client_token | detection.grouped_details.client_token | +| **x-grouped-details**:dst_byte_count | detection.grouped_details.bytes_received | +| **x-grouped-details**:src_byte_count | detection.grouped_details.bytes_sent | +| **x-grouped-details**:subnet | detection.grouped_details.subnet | +| **x-grouped-details**:rpc_function_uuid | detection.grouped_details.uuid | +| **x-grouped-details**:num_services_requested | detection.grouped_details.num_services_requested | +| **x-grouped-details**:num_services_high_privilege | detection.grouped_details.num_services_high_privilege | +| **x-grouped-details**:service_privilege | detection.grouped_details.service_privilege | +| **x-service-accessed-info**:name | detection.grouped_details.service_accessed.name | +| **x-service-accessed-info**:privilege_category | detection.grouped_details.service_accessed.privilege_category | +| **x-service-accessed-info**:privilege_level | detection.grouped_details.service_accessed.privilege_level | +| **x-ldap-event**:base_object | detection.grouped_details.events.base_object | +| **x-ldap-event**:request | detection.grouped_details.events.request | +| **x-ldap-event**:response_code | detection.grouped_details.events.response_code | +| **x-ldap-event**:num_response_objects | detection.grouped_details.events.num_response_objects | +| **x-ldap-event**:last_timestamp | detection.grouped_details.events.last_timestamp | +| **x-sql-request-info**:http_segment | detection.grouped_details.targets.events.http_segment | +| **x-sql-request-info**:user_agent | detection.grouped_details.targets.events.user_agent | +| **x-sql-request-info**:sql_fragment | detection.grouped_details.targets.events.sql_fragment | +| **x-sql-request-info**:response_code | detection.grouped_details.targets.events.response_code | +| **x-sql-request-info**:bytes_received | detection.grouped_details.targets.events.bytes_received | +| **x-sql-request-info**:last_seen | detection.grouped_details.targets.events.last_seen | +| **x-anomalous-rpc**:function_call | detection.grouped_details.anomalous_profiles.function_call | +| **x-anomalous-rpc**:rpc_function_uuid | detection.grouped_details.anomalous_profiles.function_uuid | +| **x-anomalous-rpc**:count | detection.grouped_details.anomalous_profiles.count | +| **x-anomalous-rpc**:start | detection.grouped_details.anomalous_profiles.first_timestamp | +| **x-anomalous-rpc**:end | detection.grouped_details.anomalous_profiles.last_timestamp | +| **x-services-requested**:service | detection.grouped_details.services_requested.service | +| **x-services-requested**:privilege | detection.grouped_details.services_requested.privilege | +| **x-new-host-info**:artifact | detection.grouped_details.artifact | +| **x-new-host-info**:via | detection.grouped_details.via | +| **x-new-host-info**:role | detection.grouped_details.role | +| **x-new-host-info**:end | detection.grouped_details.last_timestamp | +|
| | +### Supported STIX Objects and Properties for Query Results +| STIX Object | STIX Property | Data Source Field | +|--|--|--| +| ipv4-addr | value | detection.src_ip | +| ipv4-addr | value | detection.grouped_details.dst_ips | +| ipv4-addr | value | detection.grouped_details.dst_hosts.dst_ip | +| ipv4-addr | value | detection.grouped_details.normal_admin_hosts.ip | +| ipv4-addr | value | detection.grouped_details.dst_hosts.ip | +| ipv4-addr | value | detection.grouped_details.origin_ip | +| ipv4-addr | value | detection.grouped_details.sessions.dst_ip | +| ipv4-addr | value | detection.grouped_details.events.dst_ip | +| ipv4-addr | value | detection.grouped_details.events.dst_ips | +| ipv4-addr | value | detection.grouped_details.events.sessions.dst_ip | +| ipv4-addr | value | detection.grouped_details.connection_events.target_host.ip | +|
| | | +| ipv6-addr | value | detection.src_ip | +| ipv6-addr | value | detection.grouped_details.dst_ips | +| ipv6-addr | value | detection.grouped_details.dst_hosts.dst_ip | +| ipv6-addr | value | detection.grouped_details.normal_admin_hosts.ip | +| ipv6-addr | value | detection.grouped_details.dst_hosts.ip | +| ipv6-addr | value | detection.grouped_details.origin_ip | +| ipv6-addr | value | detection.grouped_details.sessions.dst_ip | +| ipv6-addr | value | detection.grouped_details.events.dst_ip | +| ipv6-addr | value | detection.grouped_details.events.dst_ips | +| ipv6-addr | value | detection.grouped_details.events.sessions.dst_ip | +| ipv6-addr | value | detection.grouped_details.connection_events.target_host.ip | +|
| | | +| domain-name | value | detection.grouped_details.target_domains | +| domain-name | value | detection.grouped_details.origin_domain | +| domain-name | value | detection.grouped_details.events.target_domains | +| domain-name | value | detection.grouped_details.connection_events.target_host.dst_dns | +|
| | | +| network-traffic | dst_port | detection.grouped_details.dst_ports | +| network-traffic | dst_port | detection.grouped_details.dst_hosts.dst_port | +| network-traffic | dst_port | detection.grouped_details.origin_port | +| network-traffic | dst_port | detection.grouped_details.sessions.dst_port | +| network-traffic | dst_port | detection.grouped_details.events.dst_port | +| network-traffic | dst_port | detection.grouped_details.events.sessions.dst_port | +| network-traffic | dst_port | detection.grouped_details.events.target_summary.dst_port | +| network-traffic | dst_port | detection.grouped_details.connection_events.dst_port | +| network-traffic | src_port | detection.grouped_details.src_port | +| network-traffic | src_ref.value | detection.src_ip | +| network-traffic | dst_ref.value | detection.grouped_details.dst_ips | +| network-traffic | dst_ref.value | detection.grouped_details.dst_hosts.dst_ip | +| network-traffic | dst_ref.value | detection.grouped_details.sessions.dst_ip | +| network-traffic | dst_ref.value | detection.grouped_details.origin_ip | +| network-traffic | dst_ref.value | detection.grouped_details.events.sessions.dst_ip | +| network-traffic | dst_ref.value | detection.grouped_details.connection_events.target_host.ip | +| network-traffic | protocols[*] | detection.grouped_details.protocol | +| network-traffic | protocols[*] | detection.grouped_details.app_protocol | +| network-traffic | protocols[*] | detection.grouped_details.dst_protocol | +| network-traffic | protocols[*] | detection.grouped_details.origin_protocol | +| network-traffic | protocols[*] | detection.grouped_details.sessions.protocol | +| network-traffic | protocols[*] | detection.grouped_details.sessions.app_protocol | +| network-traffic | protocols[*] | detection.grouped_details.events.protocol | +| network-traffic | protocols[*] | detection.grouped_details.events.sessions.app_protocol | +| network-traffic | protocols[*] | detection.grouped_details.events.sessions.protocol | +| network-traffic | protocols[*] | detection.grouped_details.events.target_summary.app_protocol | +| network-traffic | protocols[*] | detection.grouped_details.events.target_summary.protocol | +| network-traffic | protocols[*] | detection.grouped_details.connection_events.protocol | +| network-traffic | src_byte_count | detection.grouped_details.bytes_sent | +| network-traffic | src_byte_count | detection.grouped_details.sessions.bytes_sent | +| network-traffic | src_byte_count | detection.grouped_details.events.bytes_sent | +| network-traffic | src_byte_count | detection.grouped_details.connection_events.total_bytes_sent | +| network-traffic | dst_byte_count | detection.grouped_details.bytes_received | +| network-traffic | dst_byte_count | detection.grouped_details.sessions.bytes_received | +| network-traffic | dst_byte_count | detection.grouped_details.events.bytes_received | +| network-traffic | dst_byte_count | detection.grouped_details.events.sessions.bytes_received | +| network-traffic | dst_byte_count | detection.grouped_details.connection_events.total_bytes_rcvd | +| network-traffic | start | detection.first_timestamp | +| network-traffic | start | detection.grouped_details.first_timestamp | +| network-traffic | start | detection.grouped_details.sessions.first_timestamp | +| network-traffic | start | detection.grouped_details.events.first_timestamp | +| network-traffic | start | detection.grouped_details.events.sessions.first_timestamp | +| network-traffic | start | detection.grouped_details.events.target_summary.first_timestamp | +| network-traffic | start | detection.grouped_details.connection_events.first_timestamp | +| network-traffic | end | detection.last_timestamp | +| network-traffic | end | detection.grouped_details.last_timestamp | +| network-traffic | end | detection.grouped_details.dst_hosts.last_timestamp | +| network-traffic | end | detection.grouped_details.sessions.last_timestamp | +| network-traffic | end | detection.grouped_details.events.last_seen | +| network-traffic | end | detection.grouped_details.events.last_timestamp | +| network-traffic | end | detection.grouped_details.events.target_summary.last_timestamp | +| network-traffic | end | detection.grouped_details.connection_events.last_timestamp | +| network-traffic | x_count | detection.grouped_details.events.count | +| network-traffic | x_dst_country | detection.grouped_details.events.dst_country | +| network-traffic | x_num_accounts | detection.grouped_details.num_accounts | +| network-traffic | x_reason | detection.grouped_details.reason | +| network-traffic | x_num_attempts | detection.grouped_details.num_attempts | +| network-traffic | x_tunnel_type | detection.grouped_details.sessions.tunnel_type | +| network-traffic | x_num_sessions | detection.grouped_details.num_sessions | +| network-traffic | x_user_agent | detection.grouped_details.user_agent | +| network-traffic | x_dst_geo_latitude | detection.grouped_details.dst_geo_lat | +| network-traffic | x_dst_geo_latitude | detection.grouped_details.origin_geo_lat | +| network-traffic | x_dst_geo_latitude | detection.grouped_details.sessions.dst_geo_lat | +| network-traffic | x_dst_geo_longitude | detection.grouped_details.dst_geo_lon | +| network-traffic | x_dst_geo_longitude | detection.grouped_details.origin_geo_lon | +| network-traffic | x_dst_geo_longitude | detection.grouped_details.sessions.dst_geo_lon | +| network-traffic | x_dst_geo | detection.grouped_details.dst_geo | +| network-traffic | x_dst_geo | detection.grouped_details.origin_geo | +| network-traffic | x_dst_geo | detection.grouped_details.sessions.dst_geo | +| network-traffic | x_num_response_objects | detection.grouped_details.num_response_objects | +| network-traffic | x_client_name | detection.grouped_details.client_name | +| network-traffic | x_client_token | detection.grouped_details.client_token | +| network-traffic | x_is_normally_accessed_by_rdp | detection.grouped_details.events.is_normally_accessed_by_rdp | +| network-traffic | x_rpc_uuid | detection.grouped_details.uuid | +| network-traffic | x_nt_referrer | detection.grouped_details.events.referrer | +| network-traffic | x_num_events | detection.grouped_details.num_events | +| network-traffic | x_time_duration | detection.grouped_details.duration | +| network-traffic | x_time_duration | detection.grouped_details.events.duration | +| network-traffic | x_time_duration | detection.grouped_details.events.sessions.duration | +| network-traffic | x_time_duration | detection.grouped_details.connection_events.duration | +| network-traffic | x_status_code | detection.grouped_details.status_code | +| network-traffic | x_named_pipe | detection.grouped_details.named_pipe | +| network-traffic | x_uri | detection.grouped_details.uri | +| network-traffic | x_src_session_uid | detection.grouped_details.metadata.orig_sluid | +| network-traffic | x_executed_functions | detection.grouped_details.executed_functions | +| network-traffic | x_event_type | detection.grouped_details.events.event_type | +| network-traffic | x_error_code | detection.grouped_details.events.error_code | +| network-traffic | x_target_domain_refs[*].value | detection.grouped_details.events.target_domains | +| network-traffic | x_is_external | detection.grouped_details.connection_events.is_external | +| network-traffic | x_request_uri | detection.grouped_details.uri | +| network-traffic | x_period_identified | detection.grouped_details.period_identified | +| network-traffic | x_smb_share | detection.grouped_details.share | +| network-traffic | x_account_uid | detection.grouped_details.account_uid | +| network-traffic | extensions.'http-request-ext'.request_method | detection.grouped_details.events.http_method | +| network-traffic | extensions.'http-request-ext'.x_response_code | detection.grouped_details.events.response_code | +| network-traffic | extensions.'http-request-ext'.request_header.'User-Agent' | detection.grouped_details.user_agent | +|
| | | +| user-account | user_id | detection.grouped_details.dst_accounts.uid | +| user-account | user_id | detection.grouped_details.src_account.name | +| user-account | user_id | detection.grouped_details.normal_users | +| user-account | user_id | detection.summary.accounts | +| user-account | account_login | detection.grouped_details.src_account.name | +| user-account | x_privilege_category | detection.grouped_details.src_account.privilege_category | +| user-account | x_privilege_level | detection.grouped_details.src_account.privilege_level | +|
| | | +| x-ibm-finding | name | detection.detection_type | +| x-ibm-finding | alert_id | detection.id | +| x-ibm-finding | description | detection.description | +| x-ibm-finding | description | detection.summary.description | +| x-ibm-finding | x_num_sessions | detection.grouped_details.num_sessions | +| x-ibm-finding | severity | detection.threat | +| x-ibm-finding | confidence | detection.certainty | +| x-ibm-finding | start | detection.first_timestamp | +| x-ibm-finding | end | detection.last_timestamp | +| x-ibm-finding | time_observed | detection.created_timestamp | +| x-ibm-finding | event_count | detection.summary.num_sessions | +| x-ibm-finding | event_count | detection.summary.num_attempts | +| x-ibm-finding | x_state | detection.state | +| x-ibm-finding | x_num_successes | detection.summary.num_successes | +| x-ibm-finding | x_assigned_to | detection.assigned_to | +| x-ibm-finding | x_assigned_date | detection.assigned_date | +| x-ibm-finding | x_sensor_name | detection.sensor_name | +| x-ibm-finding | x_is_triaged | detection.is_triaged | +| x-ibm-finding | src_ip_ref | detection.src_ip | +| x-ibm-finding | x_dst_ports | detection.summary.dst_ports | +| x-ibm-finding | x_account_refs.user_id | detection.summary.accounts | +| x-ibm-finding | x_shares | detection.summary.shares | +| x-ibm-finding | x_probable_owner | detection.summary.probable_owner | +| x-ibm-finding | x_matches | detection.summary.matches | +|
| | | +| x-ibm-ttp-tagging | name | detection.detection_category | +| x-ibm-ttp-tagging | confidence | detection.certainty | +| x-ibm-ttp-tagging | kill_chain_phases.phase_name | detection.detection_category | +|
| | | +| x-oca-asset | hostname | detection.src_host.name | +| x-oca-asset | device_id | detection.src_host.id | +| x-oca-asset | x_is_key_asset | detection.src_host.is_key_asset | +| x-oca-asset | ip_refs[*].value | detection.src_ip | +| x-oca-asset | x_threat | detection.src_host.threat | +| x-oca-asset | x_certainty | detection.src_host.certainty | +| x-oca-asset | x_privilege_category | detection.grouped_details.src_host.privilege_category | +| x-oca-asset | x_privilege_level | detection.grouped_details.src_host.privilege_level | +|
| | | +| x-grouped-details | first_seen | detection.grouped_details.first_seen | +| x-grouped-details | last_seen | detection.grouped_details.last_seen | +| x-grouped-details | detection_source | detection.grouped_details.detection_source | +| x-grouped-details | detection_slug | detection.grouped_details.detection_slug | +| x-grouped-details | account_ref.user_id | detection.grouped_details.src_account.id | +| x-grouped-details | ja3_hashes | detection.grouped_details.ja3_hashes | +| x-grouped-details | ja3s_hashes | detection.grouped_details.ja3s_hashes | +| x-grouped-details | x_num_sessions | detection.grouped_details.num_sessions | +| x-grouped-details | start | detection.grouped_details.first_timestamp | +| x-grouped-details | end | detection.grouped_details.last_timestamp | +| x-grouped-details | count | detection.grouped_details.count | +| x-grouped-details | client_name | detection.grouped_details.client_name | +| x-grouped-details | client_token | detection.grouped_details.client_token | +| x-grouped-details | dst_byte_count | detection.grouped_details.bytes_received | +| x-grouped-details | src_byte_count | detection.grouped_details.bytes_sent | +| x-grouped-details | subnet | detection.grouped_details.subnet | +| x-grouped-details | rpc_function_uuid | detection.grouped_details.uuid | +| x-grouped-details | num_services_requested | detection.grouped_details.num_services_requested | +| x-grouped-details | num_services_high_privilege | detection.grouped_details.num_services_high_privilege | +| x-grouped-details | service_privilege | detection.grouped_details.service_privilege | +|
| | | +| x-service-accessed-info | name | detection.grouped_details.service_accessed.name | +| x-service-accessed-info | privilege_category | detection.grouped_details.service_accessed.privilege_category | +| x-service-accessed-info | privilege_level | detection.grouped_details.service_accessed.privilege_level | +|
| | | +| x-ldap-event | base_object | detection.grouped_details.events.base_object | +| x-ldap-event | request | detection.grouped_details.events.request | +| x-ldap-event | response_code | detection.grouped_details.events.response_code | +| x-ldap-event | num_response_objects | detection.grouped_details.events.num_response_objects | +| x-ldap-event | last_timestamp | detection.grouped_details.events.last_timestamp | +|
| | | +| x-sql-request-info | http_segment | detection.grouped_details.targets.events.http_segment | +| x-sql-request-info | user_agent | detection.grouped_details.targets.events.user_agent | +| x-sql-request-info | sql_fragment | detection.grouped_details.targets.events.sql_fragment | +| x-sql-request-info | response_code | detection.grouped_details.targets.events.response_code | +| x-sql-request-info | bytes_received | detection.grouped_details.targets.events.bytes_received | +| x-sql-request-info | last_seen | detection.grouped_details.targets.events.last_seen | +|
| | | +| x-anomalous-rpc | function_call | detection.grouped_details.anomalous_profiles.function_call | +| x-anomalous-rpc | rpc_function_uuid | detection.grouped_details.anomalous_profiles.function_uuid | +| x-anomalous-rpc | count | detection.grouped_details.anomalous_profiles.count | +| x-anomalous-rpc | start | detection.grouped_details.anomalous_profiles.first_timestamp | +| x-anomalous-rpc | end | detection.grouped_details.anomalous_profiles.last_timestamp | +|
| | | +| x-services-requested | service | detection.grouped_details.services_requested.service | +| x-services-requested | privilege | detection.grouped_details.services_requested.privilege | +|
| | | +| x-new-host-info | artifact | detection.grouped_details.artifact | +| x-new-host-info | via | detection.grouped_details.via | +| x-new-host-info | role | detection.grouped_details.role | +| x-new-host-info | end | detection.grouped_details.last_timestamp | +|
| | | From 83be2211830faeafccb3a771374e252ec18b96c8 Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh Date: Thu, 20 Jul 2023 09:24:48 +0530 Subject: [PATCH 2/8] Dummy check in - restarting code coverage --- stix_shifter_modules/vectra/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/vectra/README.md b/stix_shifter_modules/vectra/README.md index ddc37e43b..79410d9fd 100644 --- a/stix_shifter_modules/vectra/README.md +++ b/stix_shifter_modules/vectra/README.md @@ -436,4 +436,4 @@ vectra - [Vectra REST API Guide v2.4](https://support.vectra.ai/s/article/KB-VS-1626) - [Advanced Search Reference Guide](https://support.vectra.ai/s/article/KB-VS-1116) - [Understanding Vectra AI](https://support.vectra.ai/s/article/KB-VS-1285) -- [Detection and Campaign lifespan and retention periods](https://support.vectra.ai/s/article/KB-VS-1099) +- [Detection and Campaign lifespan and retention periods](https://support.vectra.ai/s/article/KB-VS-1099) From a170e0873ef965468b9fa4e9cf45e6f3cc103ebd Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh Date: Wed, 26 Jul 2023 13:14:21 +0530 Subject: [PATCH 3/8] Updated code to correct the page number calculation Updated code to correct the page number calculation --- stix_shifter_modules/vectra/stix_transmission/connector.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/stix_shifter_modules/vectra/stix_transmission/connector.py b/stix_shifter_modules/vectra/stix_transmission/connector.py index 7b960c05b..f4e8d6742 100644 --- a/stix_shifter_modules/vectra/stix_transmission/connector.py +++ b/stix_shifter_modules/vectra/stix_transmission/connector.py @@ -52,12 +52,9 @@ async def create_results_connection(self, query, offset, length): page = 1 # changing offset to page number - if offset > length: + if offset >= length: pagination_offset = offset % length - if pagination_offset < 5: - page = round(offset / length) + 1 - else: - page = round(offset / length) + page = (offset // length) + 1 # set total record count if self.api_client.result_limit < total_records: From 5dc7dba7aa9b41ca7361078e446ecc42846a8256 Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh Date: Wed, 26 Jul 2023 20:08:24 +0530 Subject: [PATCH 4/8] Mapping validator errors resolved Mapping validator errors resolved --- .../json/stix_2_1/to_stix_map.json | 42 ++++++++++++------- .../stix_translation/json/to_stix_map.json | 42 ++++++++++++------- 2 files changed, 52 insertions(+), 32 deletions(-) diff --git a/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json index a664dde41..d5d306bdf 100644 --- a/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json @@ -113,14 +113,24 @@ "transformer": "ConvertToReal" } ], - "first_timestamp": { - "key": "x-ibm-finding.start", - "object": "detection" - }, - "last_timestamp": { - "key": "x-ibm-finding.end", - "object": "detection" - }, + "first_timestamp": [ + { + "key": "x-ibm-finding.start", + "object": "detection" + }, + { + "key": "first_observed" + } + ], + "last_timestamp": [ + { + "key": "x-ibm-finding.end", + "object": "detection" + }, + { + "key": "last_observed" + } + ], "created_timestamp": { "key": "x-ibm-finding.time_observed", "object": "detection" @@ -560,23 +570,23 @@ "events": { "base_object": { "key": "x-ldap-event.base_object", - "object": "event" + "object": "ldap_event" }, "request": { "key": "x-ldap-event.request", - "object": "event" + "object": "ldap_event" }, "response_code": { "key": "x-ldap-event.response_code", - "object": "event" + "object": "ldap_event" }, "num_response_objects": { "key": "x-ldap-event.num_response_objects", - "object": "event" + "object": "ldap_event" }, "last_timestamp": { "key": "x-ldap-event.last_timestamp", - "object": "event" + "object": "ldap_event" } }, "targets": { @@ -616,10 +626,10 @@ "group_ref": true }, "groupEventReference": { - "key": "x-network-traffic.x_ldap_event_refs", + "key": "network-traffic.x_ldap_event_refs", "object": "nt_session", "references": [ - "event" + "ldap_event" ], "group_ref": true }, @@ -944,7 +954,7 @@ "transformer": "ToLowercaseArray" }, "groupSessionReference": { - "key": "x-network-traffic.x_session_refs", + "key": "network-traffic.x_session_refs", "object": "event", "references": [ "event_session" diff --git a/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json index 2dfb1d994..599ddaf51 100644 --- a/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json @@ -113,14 +113,24 @@ "transformer": "ConvertToReal" } ], - "first_timestamp": { - "key": "x-ibm-finding.start", - "object": "detection" - }, - "last_timestamp": { - "key": "x-ibm-finding.end", - "object": "detection" - }, + "first_timestamp": [ + { + "key": "x-ibm-finding.start", + "object": "detection" + }, + { + "key": "first_observed" + } + ], + "last_timestamp": [ + { + "key": "x-ibm-finding.end", + "object": "detection" + }, + { + "key": "last_observed" + } + ], "created_timestamp": { "key": "x-ibm-finding.time_observed", "object": "detection" @@ -560,23 +570,23 @@ "events": { "base_object": { "key": "x-ldap-event.base_object", - "object": "event" + "object": "ldap_event" }, "request": { "key": "x-ldap-event.request", - "object": "event" + "object": "ldap_event" }, "response_code": { "key": "x-ldap-event.response_code", - "object": "event" + "object": "ldap_event" }, "num_response_objects": { "key": "x-ldap-event.num_response_objects", - "object": "event" + "object": "ldap_event" }, "last_timestamp": { "key": "x-ldap-event.last_timestamp", - "object": "event" + "object": "ldap_event" } }, "targets": { @@ -616,10 +626,10 @@ "group_ref": true }, "groupEventReference": { - "key": "x-network-traffic.x_ldap_event_refs", + "key": "network-traffic.x_ldap_event_refs", "object": "nt_session", "references": [ - "event" + "ldap_event" ], "group_ref": true }, @@ -944,7 +954,7 @@ "transformer": "ToLowercaseArray" }, "groupSessionReference": { - "key": "x-network-traffic.x_session_refs", + "key": "network-traffic.x_session_refs", "object": "event", "references": [ "event_session" From c12b95af9f1bfcefae0d27aa91f064350230dace Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh Date: Thu, 27 Jul 2023 08:03:01 +0530 Subject: [PATCH 5/8] Dummy check in - restarting code coverage Dummy check in - restarting code coverage --- stix_shifter_modules/vectra/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/vectra/README.md b/stix_shifter_modules/vectra/README.md index 79410d9fd..ddc37e43b 100644 --- a/stix_shifter_modules/vectra/README.md +++ b/stix_shifter_modules/vectra/README.md @@ -436,4 +436,4 @@ vectra - [Vectra REST API Guide v2.4](https://support.vectra.ai/s/article/KB-VS-1626) - [Advanced Search Reference Guide](https://support.vectra.ai/s/article/KB-VS-1116) - [Understanding Vectra AI](https://support.vectra.ai/s/article/KB-VS-1285) -- [Detection and Campaign lifespan and retention periods](https://support.vectra.ai/s/article/KB-VS-1099) +- [Detection and Campaign lifespan and retention periods](https://support.vectra.ai/s/article/KB-VS-1099) From f1710fb4a1cd34251db542f36a392451f54916ab Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh Date: Fri, 28 Jul 2023 15:14:08 +0530 Subject: [PATCH 6/8] Removed default result limit Removed default result limit --- .../vectra/stix_translation/query_constructor.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/stix_shifter_modules/vectra/stix_translation/query_constructor.py b/stix_shifter_modules/vectra/stix_translation/query_constructor.py index 530898b6f..a5624c177 100644 --- a/stix_shifter_modules/vectra/stix_translation/query_constructor.py +++ b/stix_shifter_modules/vectra/stix_translation/query_constructor.py @@ -11,7 +11,6 @@ START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)" STOP_TIME = datetime.utcnow() -DEFAULT_LIMIT = 10000 CONFIG_MAP_PATH = "json/config_map.json" # API query limit is 1024 (MAX_QUERY_LENGTH+TIMESTAMP_LENGTH) @@ -412,9 +411,6 @@ def parse_expression(self, pattern: Pattern) -> list: :param pattern: expression object, ANTLR parsed expression object """ query = self._parse_expression(pattern) - if self.options['result_limit'] > DEFAULT_LIMIT: - self.options['result_limit'] = DEFAULT_LIMIT - vectra_queries = [] # Query length exceed the max query limit will split the query From 2ecbd7132ebb34bcbe5e9edf2204f6678a484b28 Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh Date: Fri, 4 Aug 2023 17:52:43 +0530 Subject: [PATCH 7/8] Pagination handled using metadata and src_ip referred in network object Pagination handled using metadata and src_ip referred in network object --- .../json/stix_2_1/to_stix_map.json | 90 +++++++++---- .../stix_translation/json/to_stix_map.json | 90 +++++++++---- .../vectra/stix_transmission/api_client.py | 6 +- .../vectra/stix_transmission/connector.py | 126 ++++++++++++------ .../test/stix_transmission/test_vectra.py | 28 +++- 5 files changed, 247 insertions(+), 93 deletions(-) diff --git a/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json index d5d306bdf..00e982618 100644 --- a/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json @@ -292,10 +292,17 @@ "key": "network-traffic.start", "object": "nt_session" }, - "last_timestamp": { - "key": "network-traffic.end", - "object": "nt_session" - }, + "last_timestamp": [ + { + "key": "network-traffic.end", + "object": "nt_session" + }, + { + "key": "network-traffic.src_ref", + "object": "nt_session", + "references": "source_ip" + } + ], "duration": { "key": "network-traffic.x_time_duration", "object": "nt_session" @@ -776,10 +783,17 @@ "references": "dst_ip" } ], - "last_timestamp": { - "key": "network-traffic.end", - "object": "nt_host" - }, + "last_timestamp": [ + { + "key": "network-traffic.end", + "object": "nt_host" + }, + { + "key": "network-traffic.src_ref", + "object": "nt_host", + "references": "source_ip" + } + ], "group_nt_Reference": { "key": "x-grouped-details.host_network_refs", "object": "group_detail", @@ -854,10 +868,17 @@ "key": "network-traffic.x_event_type", "object": "event" }, - "first_timestamp": { - "key": "network-traffic.start", - "object": "event" - }, + "first_timestamp": [ + { + "key": "network-traffic.start", + "object": "event" + }, + { + "key": "network-traffic.src_ref", + "object": "event", + "references": "source_ip" + } + ], "http_method": { "key": "network-traffic.extensions.http-request-ext.request_method", "object": "event" @@ -943,10 +964,17 @@ "key": "network-traffic.x_time_duration", "object": "event_session" }, - "first_timestamp": { - "key": "network-traffic.start", - "object": "event_session" - }, + "first_timestamp": [ + { + "key": "network-traffic.start", + "object": "event_session" + }, + { + "key": "network-traffic.src_ref", + "object": "event_session", + "references": "source_ip" + } + ], "protocol": { "key": "network-traffic.protocols", "object": "event_session", @@ -1060,10 +1088,17 @@ "object": "session", "transformer": "ToInteger" }, - "first_timestamp": { - "key": "network-traffic.start", - "object": "session" - }, + "first_timestamp": [ + { + "key": "network-traffic.start", + "object": "session" + }, + { + "key": "network-traffic.src_ref", + "object": "session", + "references": "source_ip" + } + ], "last_timestamp": { "key": "network-traffic.end", "object": "session" @@ -1099,10 +1134,17 @@ "key": "network-traffic.x_time_duration", "object": "con_event" }, - "first_timestamp": { - "key": "network-traffic.start", - "object": "con_event" - }, + "first_timestamp": [ + { + "key": "network-traffic.start", + "object": "con_event" + }, + { + "key": "network-traffic.src_ref", + "object": "con_event", + "references": "source_ip" + } + ], "last_timestamp": { "key": "network-traffic.end", "object": "con_event" diff --git a/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json index 599ddaf51..230a6696c 100644 --- a/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json @@ -292,10 +292,17 @@ "key": "network-traffic.start", "object": "nt_session" }, - "last_timestamp": { - "key": "network-traffic.end", - "object": "nt_session" - }, + "last_timestamp": [ + { + "key": "network-traffic.end", + "object": "nt_session" + }, + { + "key": "network-traffic.src_ref", + "object": "nt_session", + "references": "source_ip" + } + ], "duration": { "key": "network-traffic.x_time_duration", "object": "nt_session" @@ -776,10 +783,17 @@ "references": "dst_ip" } ], - "last_timestamp": { - "key": "network-traffic.end", - "object": "nt_host" - }, + "last_timestamp": [ + { + "key": "network-traffic.end", + "object": "nt_host" + }, + { + "key": "network-traffic.src_ref", + "object": "nt_host", + "references": "source_ip" + } + ], "group_nt_Reference": { "key": "x-grouped-details.host_network_refs", "object": "group_detail", @@ -854,10 +868,17 @@ "key": "network-traffic.x_event_type", "object": "event" }, - "first_timestamp": { - "key": "network-traffic.start", - "object": "event" - }, + "first_timestamp": [ + { + "key": "network-traffic.start", + "object": "event" + }, + { + "key": "network-traffic.src_ref", + "object": "event", + "references": "source_ip" + } + ], "http_method": { "key": "network-traffic.extensions.http-request-ext.request_method", "object": "event" @@ -943,10 +964,17 @@ "key": "network-traffic.x_time_duration", "object": "event_session" }, - "first_timestamp": { - "key": "network-traffic.start", - "object": "event_session" - }, + "first_timestamp": [ + { + "key": "network-traffic.start", + "object": "event_session" + }, + { + "key": "network-traffic.src_ref", + "object": "event_session", + "references": "source_ip" + } + ], "protocol": { "key": "network-traffic.protocols", "object": "event_session", @@ -1060,10 +1088,17 @@ "object": "session", "transformer": "ToInteger" }, - "first_timestamp": { - "key": "network-traffic.start", - "object": "session" - }, + "first_timestamp": [ + { + "key": "network-traffic.start", + "object": "session" + }, + { + "key": "network-traffic.src_ref", + "object": "session", + "references": "source_ip" + } + ], "last_timestamp": { "key": "network-traffic.end", "object": "session" @@ -1099,10 +1134,17 @@ "key": "network-traffic.x_time_duration", "object": "con_event" }, - "first_timestamp": { - "key": "network-traffic.start", - "object": "con_event" - }, + "first_timestamp": [ + { + "key": "network-traffic.start", + "object": "con_event" + }, + { + "key": "network-traffic.src_ref", + "object": "con_event", + "references": "source_ip" + } + ], "last_timestamp": { "key": "network-traffic.end", "object": "con_event" diff --git a/stix_shifter_modules/vectra/stix_transmission/api_client.py b/stix_shifter_modules/vectra/stix_transmission/api_client.py index 982ad6c6f..01ddc7fcd 100644 --- a/stix_shifter_modules/vectra/stix_transmission/api_client.py +++ b/stix_shifter_modules/vectra/stix_transmission/api_client.py @@ -14,6 +14,7 @@ def __init__(self, connection, configuration): 'Content-Type': "application/json", 'Cache-Control': "no-cache"} self.client = RestApiClientAsync(connection.get('host'), port=None, headers=self.headers) + self.host = connection.get('host') async def ping_data_source(self): """ @@ -29,5 +30,8 @@ async def get_search_results(self, query): :return: Response Object """ self.logger.debug("query: %s", query) - url = self.QUERY_ENDPOINT + '/?' + query + if self.QUERY_ENDPOINT not in query: + url = self.QUERY_ENDPOINT + '/?' + query + else: + url = query.replace('https://' + self.host, '') # removing host address for next page url return await self.client.call_api(url, 'GET', headers=self.headers, data=None) diff --git a/stix_shifter_modules/vectra/stix_transmission/connector.py b/stix_shifter_modules/vectra/stix_transmission/connector.py index f4e8d6742..b1a25a935 100644 --- a/stix_shifter_modules/vectra/stix_transmission/connector.py +++ b/stix_shifter_modules/vectra/stix_transmission/connector.py @@ -3,6 +3,11 @@ from stix_shifter_utils.utils import logger from .api_client import APIClient import json +import re + + +class InvalidMetadataException(Exception): + pass class Connector(BaseJsonSyncConnector): @@ -33,65 +38,99 @@ async def ping_connection(self): ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) return return_obj - async def create_results_connection(self, query, offset, length): + async def create_results_connection(self, query, offset, length, metadata=None): """ Function to create query connection :param query: str :param offset: int :param length: str + :param metadata: dict :return: list """ response_dict = {} return_obj = {} data = [] + local_result_count = 0 try: - offset = int(offset) - length = int(length) - pagination_offset = offset - total_records = length - page = 1 - - # changing offset to page number - if offset >= length: - pagination_offset = offset % length - page = (offset // length) + 1 - - # set total record count - if self.api_client.result_limit < total_records: - total_records = self.api_client.result_limit - - # set page size + if metadata: + if isinstance(metadata, dict) and metadata.get('result_count') and metadata.get('next_page_url'): + result_count, next_page_url = metadata['result_count'], metadata['next_page_url'] + result_count = int(result_count) + total_records = int(length) + if abs(self.api_client.result_limit - result_count) < total_records: + total_records = abs(self.api_client.result_limit - result_count) + + else: + # raise exception when metadata doesnt contain result count or next page token + raise InvalidMetadataException(metadata) + else: + result_count, next_page_url = 0, None + total_records = int(offset) + int(length) + if self.api_client.result_limit < total_records: + total_records = self.api_client.result_limit if total_records <= Connector.VECTRA_MAX_PAGE_SIZE: page_size = total_records else: page_size = Connector.VECTRA_MAX_PAGE_SIZE - query = f"page_size={page_size}&{query}&page={page}" - response_wrapper = await self.api_client.get_search_results(query) - response_code = response_wrapper.code - response = json.loads(response_wrapper.read().decode('utf-8')) + if (result_count == 0 and next_page_url is None) or (next_page_url and result_count < + self.api_client.result_limit): + if next_page_url: + response_wrapper = await self.api_client.get_search_results(next_page_url) + else: + query = f"page_size={page_size}&{query}&page=1" + response_wrapper = await self.api_client.get_search_results(query) + response_dict = json.loads(response_wrapper.read().decode('utf-8')) + response_code = response_wrapper.code + response = response_dict + if response_wrapper.code == 200: + return_obj['success'] = True + data += response_dict['results'] + result_count += len(response_dict['results']) + local_result_count += len(response_dict['results']) + next_url = response_dict.get('next') + # loop until if there is next page link and records fetched is less than total records + while next_url: + if not metadata and result_count < total_records: + remaining_records = total_records - result_count + elif metadata and local_result_count < total_records: + remaining_records = total_records - local_result_count + else: + break - if response_code == 200: - data += response['results'] - next_url = response.get('next') - while next_url: - if len(data) >= total_records: - break - next_url = next_url.split('/api/v2.4/search/detections/?') - query = next_url[1] - next_response_wrapper = await self.api_client.get_search_results(query) - response_code = next_response_wrapper.code - next_response = json.loads(next_response_wrapper.read().decode('utf-8')) - next_url = next_response.get('next') - if response_code == 200: - data += next_response['results'] - else: + if remaining_records > Connector.VECTRA_MAX_PAGE_SIZE: + page_size = Connector.VECTRA_MAX_PAGE_SIZE + else: + page_size = remaining_records + next_url = re.sub(r'page_size=([^>]*?)&', f'page_size={page_size}&', next_url) + next_response_wrapper = await self.api_client.get_search_results(next_url) + next_response = json.loads(next_response_wrapper.read().decode('utf-8')) + response_code = next_response_wrapper.code response = next_response - break - - if response_code == 200: - return_obj['success'] = True - return_obj['data'] = self.get_results_data(data[pagination_offset:total_records]) + if next_response_wrapper.code == 200: + data += next_response['results'] + next_url = response_dict.get('next') + result_count += len(next_response['results']) + local_result_count += len(next_response['results']) + else: + data = [] + break + if data: + final_result = self.get_results_data(data) + if metadata: + return_obj['data'] = final_result if final_result else [] + else: + return_obj['data'] = final_result[int(offset): total_records] if final_result else [] + if next_url and result_count < self.api_client.result_limit: + return_obj['metadata'] = {"result_count": result_count, + "next_page_url": next_url} + else: + if not return_obj.get('error') and return_obj.get('success') is not False: + return_obj['success'] = True + return_obj['data'] = [] + else: + return_obj['success'] = True + return_obj['data'] = [] # handling error response if response_code == 200: @@ -114,6 +153,11 @@ async def create_results_connection(self, query, offset, length): response_dict['message'] = str(response) ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + except InvalidMetadataException as ex: + response_dict['code'] = 422 + response_dict['message'] = f'Invalid metadata: {str(ex)}' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + except Exception as ex: if "timeout_error" in str(ex): response_dict['code'] = 408 diff --git a/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py b/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py index 66d6c75ed..8e79640f7 100644 --- a/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py +++ b/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py @@ -446,7 +446,7 @@ def test_result_invalid_query(self, mock_results_response): @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') def test_result_response(self, mock_results_response): - """Test invalid query for results""" + """Test valid query for results""" query = '(detection.detection:"Hidden HTTPS Tunnel" AND (detection.last_timestamp:[2022-11-16T0000 to ' \ '2023-05-23T0000]))' mock_response_1 = json.dumps(self.mocked_response) @@ -455,9 +455,31 @@ def test_result_response(self, mock_results_response): mock_results_response.side_effect = [get_mock_response(200, mock_response_1, 'byte'), get_mock_response(200, mock_response_2, 'byte')] transmission = stix_transmission.StixTransmission('vectra', self.connection(), self.configuration()) - offset = 20 - length = 10 + offset = 0 + length = 4 result_response = transmission.results(query, offset, length) assert result_response is not None assert result_response['success'] is True assert result_response['data'] is not None + assert result_response['metadata'] is not None + assert result_response['metadata']['next_page_url'] is not None + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_result_response_with_metadata(self, mock_results_response): + """Test using metadata to get the results""" + metadata = {'result_count': 4, 'next_page_url': 'https://test1/api/v2.4/search/detections/?page_size=4' + '&query_string=detection.detection:"Hidden HTTPS Tunnel"' + '&page=2'} + query = 'detection.detection:"Hidden HTTPS Tunnel"' + mock_response_1 = json.dumps(self.mocked_response) + self.mocked_response['next'] = None + mock_response_2 = json.dumps(self.mocked_response) + mock_results_response.side_effect = [get_mock_response(200, mock_response_1, 'byte'), + get_mock_response(200, mock_response_2, 'byte')] + transmission = stix_transmission.StixTransmission('vectra', self.connection(), self.configuration()) + offset = 4 + length = 4 + result_response = transmission.results(query, offset, length, metadata) + assert result_response is not None + assert result_response['success'] is True + assert result_response['data'] is not None From 76fffad4a46796a1164e85452f3e964afac839a3 Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh Date: Tue, 22 Aug 2023 18:45:38 +0530 Subject: [PATCH 8/8] Update stix_shifter.py, pass the metadata argument in transmit --- stix_shifter/scripts/stix_shifter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stix_shifter/scripts/stix_shifter.py b/stix_shifter/scripts/stix_shifter.py index 8720fe071..461a038c5 100644 --- a/stix_shifter/scripts/stix_shifter.py +++ b/stix_shifter/scripts/stix_shifter.py @@ -417,7 +417,10 @@ def transmit(args): search_id = args.search_id offset = args.offset length = args.length - result = transmission.results(search_id, offset, length, metadata=None) + if args.metadata: + result = transmission.results(search_id, offset, length, metadata=metadata) + else: + result = transmission.results(search_id, offset, length, metadata=None) elif operation_command == stix_transmission.RESULTS_STIX: search_id = args.search_id offset = args.offset