diff --git a/adapter-guide/connectors/azure_sentinel_supported_stix.md b/adapter-guide/connectors/azure_sentinel_supported_stix.md index 0daca7993..a5944643f 100644 --- a/adapter-guide/connectors/azure_sentinel_supported_stix.md +++ b/adapter-guide/connectors/azure_sentinel_supported_stix.md @@ -1,4 +1,4 @@ -##### Updated on 02/27/23 +##### Updated on 05/02/23 ## Microsoft Graph Security ### Supported STIX Operators *Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).* @@ -45,7 +45,7 @@ | **process**:pid | processes.processId, processes.parentProcessId, registryKeyStates.processId | | **process**:created | processes.createdDateTime | | **process**:parent_ref.pid | processes.parentProcessId | -| **process**:binary_ref.path | processes.path | +| **process**:binary_ref.parent_directory_ref.path | processes.path | | **domain-name**:value | hostStates.fqdn, hostStates.netBiosName, networkConnections.destinationDomain, userStates.domainName | | **user-account**:user_id | userStates.accountName, processes.accountName, userStates.aadUserId | | **user-account**:account_login | userStates.logonId | @@ -54,11 +54,11 @@ | **software**:name | vendorInformation.provider | | **software**:vendor | vendorInformation.vendor | | **software**:version | vendorInformation.providerVersion | -| **url**:name | networkConnections.destinationUrl | +| **url**:value | networkConnections.destinationUrl | | **windows-registry-key**:key | registryKeyStates.key | -| **windows-registry-key**:extensions.windows-registry-value-type.valueData | registryKeyStates.valueData | -| **windows-registry-key**:extensions.windows-registry-value-type.name | registryKeyStates.valueName | -| **windows-registry-key**:extensions.windows-registry-value-type.valueType | registryKeyStates.valueType | +| **windows-registry-key**:values[*].data | registryKeyStates.valueData | +| **windows-registry-key**:values[*].name | registryKeyStates.valueName | +| **windows-registry-key**:values[*].data_type | registryKeyStates.valueType | | **x-msazure-sentinel**:tenant_id | azureTenantId | | **x-msazure-sentinel**:subscription_id | azureSubscriptionId | | **x-msazure-sentinel-alert**:activityGroupName | activityGroupName | @@ -148,10 +148,6 @@ | domain-name | value | destinationDomain | | domain-name | value | domainName | |
| | | -| extensions | windows-registry-value-type.valueData | registryKeyStates | -| extensions | windows-registry-value-type.name | registryKeyStates | -| extensions | windows-registry-value-type.valuetype | registryKeyStates | -|
| | | | file | hashes.SHA-256 | sha256 | | file | hashes.SHA-1 | sha1 | | file | hashes.MD5 | md5 | @@ -201,6 +197,9 @@ | user-account | account_login | logonId | |
| | | | windows-registry-key | key | registryKeyStates | +| windows-registry-key | values.data | registryKeyStates | +| windows-registry-key | values.name | registryKeyStates | +| windows-registry-key | values.data_type | registryKeyStates | |
| | | | x-ibm-finding | dst_application_ref | destinationServiceName | | x-ibm-finding | createddatetime | createdDateTime | diff --git a/requirements-dev.txt b/requirements-dev.txt index c0c538692..18135f649 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,7 @@ astroid==2.12.12 autopep8==1.3.4 coverage==6.5.0 +debugpy-run flake8==3.5.0 freezegun==1.2.2 isort==4.3.4 diff --git a/stix_shifter_modules/alienvault_otx/.coveragerc b/stix_shifter_modules/alienvault_otx/.coveragerc new file mode 100644 index 000000000..3dbfbb408 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = tests/* \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/__init__.py b/stix_shifter_modules/alienvault_otx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/alienvault_otx/configuration/config.json b/stix_shifter_modules/alienvault_otx/configuration/config.json new file mode 100644 index 000000000..61ab4082f --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/configuration/config.json @@ -0,0 +1,117 @@ +{ + "connection": { + "type": { + "id": "OTXQuery_Connector", + "displayName": "AlienVault OTX", + "description": "Query AlienVault OTX for IPs, domains, URLs, or file hashes" + }, + "help": { + "default": "www.ibm.com", + "type": "link" + }, + "options": { + "type": "fields", + "concurrent": { + "default": 4, + "min": 1, + "max": 100, + "type": "number", + "previous": "connection.maxConcurrentSearches" + }, + "result_limit": { + "default": 10000, + "min": 1, + "max": 500000, + "type": "number", + "previous": "connection.resultSizeLimit", + "hidden": true + }, + "time_range": { + "default": 5, + "min": 1, + "max": 10000, + "type": "number", + "previous": "connection.timerange", + "nullable": true, + "hidden": true + }, + "timeout": { + "default": 30, + "min": 1, + "max": 60, + "type": "number", + "previous": "connection.timeoutLimit" + } + }, + "namespace":{ + "type": "text", + "default": "9d4bedaf-d351-4f50-930f-f8eb121e5bae", + "hidden": true + }, + "host": { + "type": "text", + "default": "", + "hidden": true + }, + "port": { + "default": 443, + "type": "number", + "min": 1, + "max": 65535, + "hidden": true + } + }, + "configuration": { + "auth": { + "type": "fields", + "key": { + "type": "password" + } + }, + "rateLimit": { + "type": "fields", + "rateLimit": { + "default": 10000, + "type": "number", + "hidden": true + }, + "rateUnit": { + "type": "text", + "default": "Hour", + "hidden": true + } + }, + "cacheDuration": { + "type": "fields", + "cacheDuration": { + "default": 10, + "type": "number", + "hidden": true + }, + "unit": { + "default": "Minute", + "type": "text", + "hidden": true + } + }, + "dataTypeList": { + "type": "fields", + "ip": { + "type": "checkbox", + "default": true + }, + "domain": { + "type": "checkbox", + "default": true + }, + "url": { + "type": "checkbox", + "default": true + }, + "hash": { + "type": "checkbox", + "default": true + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/configuration/lang_en.json b/stix_shifter_modules/alienvault_otx/configuration/lang_en.json new file mode 100644 index 000000000..a507f4536 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/configuration/lang_en.json @@ -0,0 +1,73 @@ +{ + "connection": { + "options": { + "concurrent": { + "label": "Concurrent Search Limit", + "description": "The number of simultaneous connections that can be made between the host and the data source. Valid input range is {{min}} to {{max}}." + }, + "search_timeout": { + "label": "Query Search Timeout Limit", + "description": "The limit on how long the query will run, in minutes, on the data source." + } + }, + "host": { + "label": "Management IP address or Hostname", + "placeholder": "192.168.1.10", + "description": "Specify the OCP Cluster hostname or the XForce API host URL" + }, + "port": { + "label": "Host Port", + "description": "Set the port number that is associated with the Host name or IP" + }, + "namespace": { + "label": "The UUID Namespace to generate unique ", + "description": "Supply a UUID to generate deterministic UUIDs for the resulting STIX bundle" + } + }, + "configuration": { + "auth": { + "key": { + "label": "key", + "description": "The APIKey for the Alienvault OTX" + } + }, + "rateLimit": { + "rateLimit": { + "label": "Rate Limit", + "description": "The number of queries allowed by Alienvault OTX" + }, + "rateUnit": { + "label": "Rate Unit", + "description": "The rate unit for rate limit in [seconds, minutes, days, months, years ...]" + } + }, + "cacheDuration": { + "cacheDuration": { + "label": "Cache Duration", + "description": "How long should we cache the results of the STIX Bundle execution?" + }, + "unit": { + "label": "Rate Unit", + "description": "The unit for cache in [seconds, minutes, days, months, years ...]" + } + }, + "dataTypeList": { + "ip": { + "label": "IP Address", + "description": "Whether IP Address lookup queries are supported by Alienvault OTX based on the User's API Provisioning" + }, + "domain": { + "label": "Domain", + "description": "Whether Domain queries are supported by Alienvault OTX based on the User's API Provisioning" + }, + "url": { + "label": "URL", + "description": "Whether Domain queries are supported by Alienvault OTX based on the User's API Provisioning" + }, + "hash": { + "label": "Hash", + "description": "Whether Hash queries are supported by Alienvault OTX based on the User's API Provisioning" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/entry_point.py b/stix_shifter_modules/alienvault_otx/entry_point.py new file mode 100644 index 000000000..f7cca2478 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/entry_point.py @@ -0,0 +1,42 @@ +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint +from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from .stix_transmission.ping_connector import PingConnector +from .stix_transmission.delete_connector import DeleteConnector +from .stix_transmission.results_connector import ResultsConnector +from .stix_transmission.api_client import APIClient +from .stix_translation.query_translator import QueryTranslator +from .stix_translation.results_translator import ResultsTranslator +import os + +class EntryPoint(BaseEntryPoint): + + # python main.py translate virus_total results '{}' "[ipv4-addr:value = '127.0.0.1']" + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + self.set_async(False) + if connection: + api_client = APIClient(connection, configuration) + base_sync_connector = BaseSyncConnector() + ping_connector = PingConnector(api_client) + query_connector = base_sync_connector + status_connector = base_sync_connector + results_connector = ResultsConnector(api_client) + delete_connector = DeleteConnector(api_client) + + self.set_results_connector(results_connector) + self.set_status_connector(status_connector) + self.set_delete_connector(delete_connector) + self.set_query_connector(query_connector) + self.set_ping_connector(ping_connector) + + # Use default translation setup with default dialect otherwise... + # self.setup_translation_simple(dialect_default='default') + + basepath = os.path.dirname(__file__) + filepath = os.path.abspath(os.path.join(basepath, "stix_translation")) + + dialect = 'default' + query_translator = QueryTranslator(options, dialect, filepath) + results_translator = ResultsTranslator(options, dialect, filepath) + self.add_dialect(dialect, query_translator=query_translator, results_translator=results_translator, default=True) \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/requirements.txt b/stix_shifter_modules/alienvault_otx/requirements.txt new file mode 100644 index 000000000..112830057 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/requirements.txt @@ -0,0 +1 @@ +uuid==1.30 diff --git a/stix_shifter_modules/alienvault_otx/stix_translation/__init__.py b/stix_shifter_modules/alienvault_otx/stix_translation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/alienvault_otx/stix_translation/json/from_stix_map.json b/stix_shifter_modules/alienvault_otx/stix_translation/json/from_stix_map.json new file mode 100644 index 000000000..5e41b2dfb --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_translation/json/from_stix_map.json @@ -0,0 +1,30 @@ +{ + "url": { + "fields": { + "value": ["Url"] + } + }, + "ipv4-addr": { + "fields": { + "value":["SourceIpV4", "DestinationIpV4"] + } + }, + "ipv6-addr": { + "fields":{ + "value":["SourceIpV6", "DestinationIpV6"] + } + }, + "domain-name":{ + "fields":{ + "value":["Url"] + } + }, + "file":{ + "fields":{ + "hashes.'SHA-256'": ["sha256hash"], + "hashes.MD5": ["md5hash"], + "hashes.'MD5'": ["md5hash"], + "hashes.'SHA-1'": ["sha1hash"] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/stix_translation/json/operators.json b/stix_shifter_modules/alienvault_otx/stix_translation/json/operators.json new file mode 100644 index 000000000..03ff0cbd4 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_translation/json/operators.json @@ -0,0 +1,16 @@ +{ + "ComparisonExpressionOperators.And": "AND", + "ComparisonExpressionOperators.Or": "OR", + "ComparisonComparators.GreaterThan": ">", + "ComparisonComparators.GreaterThanOrEqual": ">=", + "ComparisonComparators.LessThan": "<", + "ComparisonComparators.LessThanOrEqual": "<=", + "ComparisonComparators.Equal": "=", + "ComparisonComparators.NotEqual": "!=", + "ComparisonComparators.Like": "=", + "ComparisonComparators.In": "IN", + "ComparisonComparators.Matches": "CONTAINS", + "ComparisonComparators.IsSubSet": "insubnet", + "ObservationOperators.Or": "OR", + "ObservationOperators.And": "AND" +} \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/stix_translation/json/to_stix_map.json b/stix_shifter_modules/alienvault_otx/stix_translation/json/to_stix_map.json new file mode 100644 index 000000000..077404aaa --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_translation/json/to_stix_map.json @@ -0,0 +1,3 @@ +{ + +} \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/stix_translation/query_constructor.py b/stix_shifter_modules/alienvault_otx/stix_translation/query_constructor.py new file mode 100644 index 000000000..c5cb3b011 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_translation/query_constructor.py @@ -0,0 +1,226 @@ +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \ + ComparisonExpressionOperators, ComparisonComparators, Pattern, \ + CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators +from stix_shifter_utils.stix_translation.src.utils.transformers import TimestampToMilliseconds +from stix_shifter_utils.stix_translation.src.json_to_stix import observable +import logging +import re + +# Source and destination reference mapping for ip and mac addresses. +# Change the keys to match the data source fields. The value array indicates the possible data type that can come into from field. +REFERENCE_DATA_TYPES = {"SourceIpV4": ["ipv4", "ipv4_cidr"], + "SourceIpV6": ["ipv6"], + "DestinationIpV4": ["ipv4", "ipv4_cidr"], + "DestinationIpV6": ["ipv6"]} + +logger = logging.getLogger(__name__) +data = "" +dataType = "" + +class QueryStringPatternTranslator: + # Change comparator values to match with supported data source operators + def __init__(self, pattern: Pattern, data_model_mapper): + self.dmm = data_model_mapper + self.comparator_lookup = self.dmm.map_comparator() + self.pattern = pattern + self.translated = self.parse_expression(pattern) + + @staticmethod + def _format_set(values) -> str: + gen = values.element_iterator() + return "({})".format(' OR '.join([QueryStringPatternTranslator._escape_value(value) for value in gen])) + + @staticmethod + def _format_match(value) -> str: + raw = QueryStringPatternTranslator._escape_value(value) + if raw[0] == "^": + raw = raw[1:] + else: + raw = ".*" + raw + if raw[-1] == "$": + raw = raw[0:-1] + else: + raw = raw + ".*" + return "\'{}\'".format(raw) + + @staticmethod + def _format_equality(value) -> str: + return '\'{}\''.format(value) + + @staticmethod + def _format_like(value) -> str: + value = "'%{value}%'".format(value=value) + return QueryStringPatternTranslator._escape_value(value) + + @staticmethod + def _escape_value(value, comparator=None) -> str: + if isinstance(value, str): + return '{}'.format(value.replace('\\', '\\\\').replace('\"', '\\"').replace('(', '\\(').replace(')', '\\)')) + else: + return value + + @staticmethod + def _negate_comparison(comparison_string): + return "NOT ({})".format(comparison_string) + + @staticmethod + def _check_value_type(value): + value = str(value) + for key, pattern in observable.REGEX.items(): + if key != 'date' and bool(re.search(pattern, value)): + return key + return None + + @staticmethod + def _parse_reference(self, stix_field, value_type, mapped_field, value, comparator): + if value_type not in REFERENCE_DATA_TYPES["{}".format(mapped_field)]: + return None + else: + return "{mapped_field} {comparator} {value}".format( + mapped_field=mapped_field, comparator=comparator, value=value) + + @staticmethod + def _parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array): + comparison_string = "" + is_reference_value = self._is_reference_value(stix_field) + # Need to use expression.value to match against regex since the passed-in value has already been formated. + value_type = self._check_value_type(expression.value) if is_reference_value else None + mapped_fields_count = 1 if is_reference_value else len(mapped_fields_array) + + for mapped_field in mapped_fields_array: + if is_reference_value: + parsed_reference = self._parse_reference(self, stix_field, value_type, mapped_field, value, comparator) + if not parsed_reference: + continue + comparison_string += parsed_reference + else: + comparison_string += "{mapped_field} {comparator} {value}".format(mapped_field=mapped_field, comparator=comparator, value=value) + + if (mapped_fields_count > 1): + comparison_string += " OR " + mapped_fields_count -= 1 + return comparison_string + + @staticmethod + def _is_reference_value(stix_field): + return stix_field == 'src_ref.value' or stix_field == 'dst_ref.value' + + @staticmethod + def _lookup_comparison_operator(self, expression_operator): + if str(expression_operator) not in self.comparator_lookup: + raise NotImplementedError("Comparison operator {} unsupported for Dummy connector".format(expression_operator.name)) + return self.comparator_lookup[str(expression_operator)] + + def _parse_expression(self, expression, qualifier=None) -> str: + if isinstance(expression, ComparisonExpression): # Base Case + # Resolve STIX Object Path to a field in the target Data Model + stix_object, stix_field = expression.object_path.split(':') + mapped_fields_array = self.dmm.map_field(stix_object, stix_field) + comparator = self._lookup_comparison_operator(self, expression.comparator) + if stix_field == 'start' or stix_field == 'end': + transformer = TimestampToMilliseconds() + expression.value = transformer.transform(expression.value) + # Some values are formatted differently based on how they're being compared + if expression.comparator == ComparisonComparators.Matches: # needs forward slashes + value = self._format_match(expression.value) + # should be (x, y, z, ...) + elif expression.comparator == ComparisonComparators.In: + value = self._format_set(expression.value) + elif expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: + # Should be in single-quotes + value = self._format_equality(expression.value) + # '%' -> '*' wildcard, '_' -> '?' single wildcard + elif expression.comparator == ComparisonComparators.Like: + value = self._format_like(expression.value) + else: + value = self._escape_value(expression.value) + + get_data_source_query(stix_field=stix_field, stix_object=stix_object, value=value) + + comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array) + if(len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): + # More than one data source field maps to the STIX attribute, so group comparisons together. + grouped_comparison_string = "(" + comparison_string + ")" + comparison_string = grouped_comparison_string + + if expression.negated: + comparison_string = self._negate_comparison(comparison_string) + if qualifier is not None: + return "{} {}".format(comparison_string, qualifier) + else: + return "{}".format(comparison_string) + + elif isinstance(expression, CombinedComparisonExpression): + operator = self._lookup_comparison_operator(self, expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + if not expression_01 or not expression_02: + return '' + if isinstance(expression.expr1, CombinedComparisonExpression): + expression_01 = "({})".format(expression_01) + if isinstance(expression.expr2, CombinedComparisonExpression): + expression_02 = "({})".format(expression_02) + query_string = "{} {} {}".format(expression_01, operator, expression_02) + if qualifier is not None: + return "{} {}".format(query_string, qualifier) + else: + return "{}".format(query_string) + elif isinstance(expression, ObservationExpression): + return self._parse_expression(expression.comparison_expression, qualifier) + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + if isinstance(expression.observation_expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(self, expression.observation_expression.operator) + expression_01 = self._parse_expression(expression.observation_expression.expr1) + # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions + expression_02 = self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + return "{} {} {}".format(expression_01, operator, expression_02) + else: + return self._parse_expression(expression.observation_expression.comparison_expression, expression.qualifier) + elif isinstance(expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(self, expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + if expression_01 and expression_02: + return "({}) {} ({})".format(expression_01, operator, expression_02) + elif expression_01: + return "{}".format(expression_01) + elif expression_02: + return "{}".format(expression_02) + else: + return '' + elif isinstance(expression, Pattern): + return "{expr}".format(expr=self._parse_expression(expression.expression)) + else: + raise RuntimeError("Unknown Recursion Case for expression={}, type(expression)={}".format( + expression, type(expression))) + + def parse_expression(self, pattern: Pattern): + return self._parse_expression(pattern) + + +def translate_pattern(pattern: Pattern, data_model_mapping, options): + # Query result limit and time range can be passed into the QueryStringPatternTranslator if supported by the data source. + query = QueryStringPatternTranslator(pattern, data_model_mapping).translated + # Add space around START STOP qualifiers + query = re.sub("START", "START ", query) + query = re.sub("STOP", " STOP ", query) + + translated_query = {"data": data, "dataType": dataType} + return [str(translated_query)] + +def get_data_source_query(stix_field, stix_object, value): + global data, dataType + dataType = get_data_type(stix_object, stix_field) # can ipv4-addr, ipv6-addr, url, domain, hash + data = value.replace("'", "") + +def get_data_type(stix_object, stix_field): + if "ipv4" in stix_object or "ipv6" in stix_object: + return "ip" + elif "url" in stix_object: + return "url" + elif "domain-name" in stix_object: + return "domain" + elif "file" in stix_object and "hashes" in stix_field: + return "hash" + else: + return "Unsupported Data Type" \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/stix_translation/query_translator.py b/stix_shifter_modules/alienvault_otx/stix_translation/query_translator.py new file mode 100644 index 000000000..575372d07 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_translation/query_translator.py @@ -0,0 +1,26 @@ +import logging + +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator +from . import query_constructor + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file + :param antlr_parsing_object: Antlr parsing objects for the STIX pattern + :type antlr_parsing_object: object + :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into another format. This should default to something if one isn't passed in + :type mapping: str (filepath) + :return: transformed query string + :rtype: str + """ + + logger.info("Converting STIX2 Pattern to data source query") + + query_string = query_constructor.translate_pattern( + antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/alienvault_otx/stix_translation/results_translator.py b/stix_shifter_modules/alienvault_otx/stix_translation/results_translator.py new file mode 100644 index 000000000..f5abeeb62 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_translation/results_translator.py @@ -0,0 +1,515 @@ +from stix_shifter_utils.modules.base.stix_translation.base_results_translator import BaseResultTranslator +from . import sdo_translator +# from stix_shifter_utils.normalization.normalization_helper import create_attributes +import json +from ipaddress import ip_network, IPv4Network, IPv6Network +import uuid +from urllib.parse import urlparse +from stix_shifter_utils.utils import logger +from copy import deepcopy + +def create_attributes(attribute_fields, data): + threat_attribute_report = [] + if isinstance(attribute_fields, dict): + for attribute, value in attribute_fields.items(): + if attribute in data: + attribute_dict = {} + attribute_dict['attribute_name'] = value + attribute_dict['attribute_value'] = str(data[attribute]) + attribute_dict['attribute_type'] = evaluate_attribute_type(data[attribute]) + threat_attribute_report.append(attribute_dict) if attribute_dict['attribute_type'] is not None else '' + elif type(attribute_fields) is str: + attribute_dict = {} + attribute_dict['attribute_name'] = attribute_fields + attribute_dict['attribute_value'] = str(data) + attribute_dict['attribute_type'] = evaluate_attribute_type(data) + threat_attribute_report.append(attribute_dict) if attribute_dict['attribute_type'] is not None else '' + return threat_attribute_report + + +def evaluate_attribute_type(attribute): + # supported types = string, number, uri, ip, lat_lng + attribute_type = None + if isinstance(attribute, bool): + attribute_type = 'string' if attribute is True else None + elif isinstance(attribute, (int, float, complex)): + attribute_type = 'number' + if isinstance(attribute, (str)): + attribute_type = 'string' + if uri_validator(attribute): + attribute_type = 'uri' + try: + if isinstance(ip_network(attribute), (IPv4Network, IPv6Network)): + attribute_type = 'ip' + except ValueError: + pass + + return attribute_type + + +def uri_validator(x): + result = urlparse(x) + return all([result.scheme, result.netloc]) + + +def findkeys(node, key): + if hasattr(node, 'items'): + for k, v in node.items(): + if k == key and v: + yield v + if isinstance(v, dict): + for r in findkeys(v, key): + yield r + elif isinstance(v, list): + for i in v: + for r in findkeys(i, key): + yield r + elif isinstance(node, list): + for m in node: + for n in findkeys(m, key): + yield n + +class ResultsTranslator(BaseResultTranslator): + def get_indicator_types(self, data): + indicator_type = 'unknown' + verdict_list = list(findkeys(data, 'verdict')) + if verdict_list: + verdict = verdict_list[0] + if verdict == 'Whitelisted': + indicator_type = 'benign' + + if indicator_type == 'unknown': + score_list = list(findkeys(data, 'combined_score')) + if score_list: + score = score_list[0] + if score < 3: + indicator_type = 'benign' + elif score < 7: + indicator_type = 'anomalous-activity' + elif score >= 7: + indicator_type = 'malicious-activity' + else: + indicator_type = 'unknown' + return {'indicator_types': [indicator_type]} + + + def find_hash_type_by_length(self, value): + HASH_LENGTH = {'40': 'sha-1', '64': 'sha-256', '32': 'md5'} + hash_type = HASH_LENGTH.get(str(len(value)), '') + if hash_type in ['sha-1', 'sha-256']: + return "file:hashes.'{}'".format(hash_type.upper()) + else: + return "file:hashes.{}".format(hash_type.upper()) + + + def get_ip_address(self, ip): + return 'ipv4' if isinstance(ip_network(ip), IPv4Network) else 'ipv6' + + + #Get permalink from the report + def get_external_reference_from_json(self, data): + ext_data = data['external_reference'] + if ext_data.get('url') == '' or ext_data.get('url') == 'N/A': + return None + url = data['external_reference'] + external_reference = {"external_references":[url]} + return external_reference + + + # Get required pattern field from the report, the pattern is a combination of data and dataType fields in the Connector result JSON + def get_pattern_from_json(self, data): + pattern_type, pattern_value = data['dataType'], data['data'] + pattern = self.evaluate_pattern(pattern_type, pattern_value) + pattern = {"pattern": pattern} + return pattern + + + def evaluate_pattern(self, pattern_type, value): + pattern = [] + # "dataTypeList": ["file", "hash", "domain", "ip", "url", "unknown"], + if pattern_type == "url": + pattern = "["+ pattern_type + ":value='" + value + "']" + return pattern + + elif pattern_type == 'domain': + pattern = "["+ pattern_type + "-name:value='" + value + "']" + return pattern + + # Get ipv4 or ipv6 + elif pattern_type == 'ip': + pattern_type = self.get_ip_address(value) + pattern = "["+ pattern_type + "-addr:value='" + value + "']" + return pattern + + # Get Hash of type SHA-256, SHA-1, or MD5 + elif pattern_type == 'hash': + pattern_type = self.find_hash_type_by_length(value) + pattern = "["+ pattern_type + "='" + value + "']" + return pattern + + return pattern + + + def get_description(self, data): + pulse_list = [] + pulse_count = 0 + try: + pulse_count = data['report']['full']['pulse_info']['count'] + pulse_list = data['report']['full']['pulse_info']['pluses'] + except: + pass + + malicious_count = 0 + for entry in pulse_list: + taglist = entry.get('tags') if entry.get('tags') is not None else [] + for t in taglist: + if t.lower() == 'malicious': + malicious_count += 1 + + description = f"Malicious Pulses: {str(malicious_count)}/{str(pulse_count)}" + description_object = {'description': description} + return description_object + + + def get_threat_score(self, data, indicator): + if not indicator: + return None + + BENIGN_SCORE_MIN = 0 + BENIGN_SCORE_MAX = 9 + UNKNOWN_SCORE_MIN = 10 + SUSPICIOUS_SCORE_MIN = 30 + SUSPICIOUS_SCORE_MAX = 69 + MALICIOUS_SCORE_MIN = 70 + MALICIOUS_SCORE_MAX = 100 + + # Normalize the score out of 100 + score_list = list(findkeys(data, 'combined_score')) + if score_list: + combined_score = score_list[0] + threat_score = combined_score + else: + if(indicator['indicator_types'][0] == 'benign'): + return {"threat_score": BENIGN_SCORE_MIN} + elif(indicator['indicator_types'][0] == 'unknown'): + return {"threat_score": UNKNOWN_SCORE_MIN} + if(indicator['indicator_types'][0] == 'anomalous-activity'): + return {"threat_score": SUSPICIOUS_SCORE_MIN} + if(indicator['indicator_types'][0] == 'malicious-activity'): + return {"threat_score": MALICIOUS_SCORE_MIN} + + if threat_score > 10: + TOTAL = threat_score + else: + TOTAL = 10 + threat_feed_ranges = [(0,1), (2,6), (7, TOTAL)] + + tis_ranges = [(BENIGN_SCORE_MIN, BENIGN_SCORE_MAX), + (SUSPICIOUS_SCORE_MIN, SUSPICIOUS_SCORE_MAX), + (MALICIOUS_SCORE_MIN, MALICIOUS_SCORE_MAX)] + + for i in range(0, len(threat_feed_ranges)): + if threat_score <= threat_feed_ranges[i][1]: + range_min = threat_feed_ranges[i][0] + range_max = threat_feed_ranges[i][1] + tis_range_min = tis_ranges[i][0] + tis_range_max = tis_ranges[i][1] + + tis_score = tis_range_min + (tis_range_max-tis_range_min) * 1.0 * (threat_score-range_min) / (range_max - range_min) + return {"threat_score": round(tis_score, 1)} + + + def sighting_from_report_attributes(self,data): + try: + sighting={} + pulse_count_list = list(findkeys(data, 'pulse_info')) + if pulse_count_list: + count = pulse_count_list[0]['count'] + sighting['count'] = int(count) + else: + sighting['count'] = 0 + return sighting + except ValueError: + raise ValueError("Exception occurred to parse report data for sighting SDO") + + + def retrieve_malware_object(self, src_malware_families, dst_objects): + if isinstance(src_malware_families, list): + for item in src_malware_families: + malware_obj = {} + malware_obj['name'] = item + malware_obj['malware_type'] = 'unknown' + is_existed = False + for existed_obj in dst_objects: + if existed_obj['name'] and malware_obj['name'].lower() == existed_obj['name'].lower(): + is_existed = True + break + if is_existed == False: + dst_objects.append(malware_obj) + + + def malware_from_report_attributes(self,data): + try: + malware_objects=[] + json_data = data[0] + if 'report' in json_data: + report = json_data['report'] + if 'full' in report: + report_full = report['full'] + if not isinstance(report_full, list): + report_full = [report_full] + for full_report in report_full: + malwareObject={} + + detection_list = list(findkeys(full_report, 'detection')) + if detection_list: + detections = detection_list[0] + for d in detections: + mObj = {} + mObj['name'] = d + if 'Ransom' in d: + mObj['malware_types'] = 'ransomware' + else: + mObj['malware_types'] = 'unknown' + if (mObj not in malware_objects): + malware_objects.append(mObj) + + ids_detection_list = list(findkeys(full_report, 'ids_detections')) + if ids_detection_list: + ids_detections = ids_detection_list[0] + for idsdetection in ids_detections: + if 'category' in idsdetection and idsdetection['category'] is not None and idsdetection['category'].lower() == 'malware': + if 'malware_name' in idsdetection and idsdetection['malware_name'] != '': + malwareObject['name'] = idsdetection['malware_name'] + if 'subcategory' in idsdetection and idsdetection['subcategory'] != '': + malwareObject['malware_types'] = idsdetection['subcategory'] + if (malwareObject not in malware_objects): + malware_objects.append(malwareObject) + + alienvault_malware_families = [] + try: + alienvault_malware_families = full_report['pulse_info']['related']['alienvault']['malware_families'] + except: + pass + self.retrieve_malware_object(alienvault_malware_families, malware_objects) + + other_malware_families = [] + try: + other_malware_families = full_report['pulse_info']['related']['other']['malware_families'] + except: + pass + self.retrieve_malware_object(other_malware_families, malware_objects) + + return malware_objects + except ValueError: + raise ValueError("Exception occured to parse report data for malware SDO") + + + def create_indicator_object(self, *properties): + indicator_object = {} + for prop in properties: + if prop is not None: + for key, value in prop.items(): + indicator_object[key] = value + return indicator_object + + + def get_threat_report(self, data): + return {'x_ibm_original_threat_feed_data': data['report']} + + + def add_fields_to_data(self, data): + full_object = data['report'].get('full', {}) + backup_combined_score = None + + try: + backup_combined_score = full_object['analysis']['plugins']['cuckoo']['result']['info']['combined_score'] + full_object['analysis']['combined_score'] = backup_combined_score + except: + pass + + try: + # Remove unnecessary parts from full report to reduce total size + if full_object['analysis'].get('plugins', {}): + del full_object['analysis']['plugins'] + except: + pass + + data['report']['full'] = full_object + + return data + + + def get_threat_attributes(self, data): + threat_attribute_report = [] + ids_detection_report = [] + + if 'report' in data: + full_report = data['report'].get('full', {}) + ids_detection_report = full_report.get('ids_detections', []) + # We pass the fields we want as attributes, with then new names of the fields + + if data['dataType'] == 'hash': + analysis_results = full_report.get('analysis', {}).get('info', {}) + analysis_fields = { + 'md5': 'MD5', + 'sha1': 'SHA-1', + 'sha256': 'SHA-256', + 'filesize': 'File Size', + 'file_type': 'File Type', + 'file_class': 'File Class' + } + field_dict = { 'results': analysis_fields, } + + full_report = analysis_results + for attribute_keys, attribute_value in field_dict.items(): + if full_report.get(attribute_keys): + threat_attribute_report += create_attributes(attribute_value, full_report[attribute_keys]) + + elif data['dataType'] == 'ip': + full_fields = { + 'asn': 'ASN', + 'city': 'City', + 'country_name': 'Country Name', + 'latitude': 'Latitude', + 'longitude': 'Longitude' + } + + field_dict = { 'full': full_fields, } + + full_report = data['report'] + for attribute_keys, attribute_value in field_dict.items(): + if attribute_keys in full_report: + threat_attribute_report += create_attributes(attribute_value, full_report[attribute_keys]) + + for threat in threat_attribute_report: + if threat.get('attribute_name').lower() == 'latitude' or threat.get('attribute_name').lower() == 'longitude': + threat['attribute_type'] = 'lat_lng' + + + # We use the same logic to add IDS Detection attribute for hash/IP + if ids_detection_report: + id_string = '' + for ids in ids_detection_report: + if ids.get('name'): + id_string += ids.get('name') + ', ' + if id_string: + id_string = id_string[:-2] # Get rid of ', ' at the end + ids_dict = { + 'attribute_name': 'IDS Detections', + 'attribute_value': id_string, + 'attribute_type': 'string' + } + threat_attribute_report.append(ids_dict) + + self.get_location(threat_attribute_report) + + return {'threat_attributes' : threat_attribute_report} if threat_attribute_report else None + + + def get_location(self, threat_attributes): + location_dict = {} + remove_threat = [] + for threat in threat_attributes: + if threat.get('attribute_name').lower() == 'latitude' or threat.get('attribute_name').lower() == 'longitude': + if threat.get('attribute_name').lower() == 'latitude': location_dict['lat'] = threat['attribute_value'] + else: location_dict['lng'] = threat['attribute_value'] + remove_threat.append(threat) + if location_dict: + location_attribute = { + 'attribute_name': 'Location', + 'attribute_value': json.dumps(location_dict), + 'attribute_type': 'lat_lng' + } + threat_attributes.append(location_attribute) + for threat in remove_threat: + threat_attributes.remove(threat) + + + def translate_results(self, data_source, data): + """ + Translates JSON data into STIX results based on a mapping file + :param data: JSON formatted data to translate into STIX format + :type data: str + :param mapping: The mapping file path to use as instructions on how to translate the given JSON data to STIX. + Defaults the path to whatever is passed into the constructor for JSONToSTIX (This should be the to_stix_map.json in the module's json directory) + :type mapping: str (filepath) + :return: STIX formatted results + :rtype: str + """ + + self.logger = logger.set_logger(__name__) + + json_data = data[0] + + CONNECTOR_NAME = "OTXQuery_Connector" + data_source['id'] = CONNECTOR_NAME + try: + uuid.UUID(json_data["namespace"]) + except ValueError: + raise ValueError("Namespace is not valid UUID") + NAMESPACE = json_data["namespace"] + data_source['name'] = CONNECTOR_NAME + + # Add fields to json_report + json_data = self.add_fields_to_data(json_data) + + pattern = self.get_pattern_from_json(json_data) + external_reference = self.get_external_reference_from_json(json_data) + indicator_types = self.get_indicator_types(json_data) #get indicator types from the report + description = self.get_description(json_data) #get description from the report + threat_score = self.get_threat_score(json_data, indicator_types) + threat_attributes = self.get_threat_attributes(json_data) + indicator_name = {'name': json_data['data']} + indicator_object = self.create_indicator_object(pattern, external_reference, indicator_types, description, indicator_name) + data_to_enrich_pattern = pattern['pattern'] + + full_report = deepcopy(json_data) + full_report['report']['full'] = [full_report['report']['full']] + + # Create STIX Bundle and add SDOs + sdo_translator_object = sdo_translator.SdoTranslator(self.options) + stix_bundle = sdo_translator_object.create_stix_bundle() + + # Add Indentity SDO + identity_object = sdo_translator_object.create_identity_sdo(data_source, NAMESPACE) + stix_bundle['objects'] += identity_object + + # Add extension-definition SDO + toplevel_properties = ['x_ibm_original_threat_feed_data', 'threat_score'] + nested_properties = [] + if (threat_attributes): + toplevel_properties.append('threat_attributes') + + extension_object = sdo_translator_object.create_extension_sdo(identity_object[0], NAMESPACE, nested_properties, toplevel_properties=toplevel_properties) + stix_bundle['objects'] += extension_object + + # Add Indicator SDO + extension_id = extension_object[0]['id'] + identity_id = identity_object[0]['id'] + report = self.get_threat_report(full_report) + nested_indicator = [] + top_indicator = [threat_score, report] + if (threat_attributes): + top_indicator.append(threat_attributes) + + indicator_object = sdo_translator_object.create_indicator_sdo(indicator_object, identity_id, extension_id, nested_indicator, top_indicator) + stix_bundle['objects'] += indicator_object + + # Add Malware SDO + data = json.dumps(full_report) + data = "[" + data + "]" + full_report = json.loads(data) + + malware_object = self.malware_from_report_attributes(full_report) + if malware_object: + malware_stix_object = sdo_translator_object.create_malware_sdo(malware_object,indicator_object[0]['id'],data_to_enrich_pattern) + stix_bundle['objects'] += malware_stix_object + + #Sighting SDO + sighting_object = self.sighting_from_report_attributes(full_report) + if (len(sighting_object) > 0): + sighting_stix_object = sdo_translator_object.create_sighting_sdo(sighting_object,indicator_object[0]['id']) + stix_bundle['objects'] += sighting_stix_object + + return stix_bundle diff --git a/stix_shifter_modules/alienvault_otx/stix_translation/sdo_translator.py b/stix_shifter_modules/alienvault_otx/stix_translation/sdo_translator.py new file mode 100644 index 000000000..966751ed2 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_translation/sdo_translator.py @@ -0,0 +1,21 @@ +from stix_shifter_utils.normalization.BaseNormalization import BaseNormalization + +class SdoTranslator(BaseNormalization): + def create_sighting_sdo(self, sighting_object, indicator_id): + return super().create_sighting_sdo(sighting_object, indicator_id) + + def create_infrastructure_object_sdo(self, infrastructure_object, enriched_ioc, indicator_id): + return super().create_infrastructure_object_sdo(infrastructure_object, enriched_ioc, indicator_id) + + def create_malware_sdo(self, malware_object, indicator_id, enriched_ioc): + return super().create_malware_sdo(malware_object, indicator_id, enriched_ioc) + + def create_identity_sdo(self, data_source, namespace): + return super().create_identity_sdo(data_source, namespace) + + def create_extension_sdo(self, identity_object, namespace, nested_properties, toplevel_properties): + # Create an extension-definition object to be used in conjunction with STIX Indicator object + return super().create_extension_sdo(identity_object, namespace, nested_properties, toplevel_properties) + + def create_indicator_sdo(self, indicator_object, identity_id, extension_id, nested_properties, top_properties=None): + return super().create_indicator_sdo(indicator_object, identity_id, extension_id, nested_properties, top_properties) diff --git a/stix_shifter_modules/alienvault_otx/stix_transmission/__init__.py b/stix_shifter_modules/alienvault_otx/stix_transmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/alienvault_otx/stix_transmission/api_client.py b/stix_shifter_modules/alienvault_otx/stix_transmission/api_client.py new file mode 100644 index 000000000..4a890e78b --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_transmission/api_client.py @@ -0,0 +1,158 @@ +from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync +import json +import urllib +import requests +from urllib3.exceptions import InsecureRequestWarning +from urllib.parse import urlparse +from ipaddress import ip_address, IPv4Address + + +class APIClient(): + + def __init__(self, connection, configuration): + headers = dict() + auth = configuration.get('auth') + headers['X-OTX-API-KEY'] = auth.get('key') + headers['Accept'] = "application/json" + url_modifier_function = get_url_endpoint + self.url = "https://otx.alienvault.com" + self.namespace = connection.get('namespace') + self.verify = configuration.get("verify", True) + self.timeout = connection['options'].get('timeout') + if not self.verify: + requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + self.client = RestApiClientAsync(host=self.url, + port=None, + headers=headers, + url_modifier_function=url_modifier_function, + cert_verify=self.verify + ) + + async def ping_alienvault(self): + # Pings the data source + ping_endpoint = "/api/v1/user/me" + ping_return = await self.client.call_api(ping_endpoint, 'GET', timeout=self.timeout) + return ping_return.read(), ping_return.code + + async def get_search_results(self, query_expression, range_start=None, range_end=None): + # Queries the data source + # extract the data + self.data_type = query_expression['dataType'] + self.data = query_expression['data'] + if self.data_type == 'ip': + rep = await ip_query(self, self.data) + return rep, self.namespace + elif self.data_type == 'domain': + rep = await domain_query(self, self.data) + return rep, self.namespace + elif self.data_type == 'url': + rep = await url_query(self, self.data) + return rep, self.namespace + elif self.data_type == 'hash': + rep = await hash_query(self, self.data) + return rep, self.namespace + else: + return {"code": 401, "error": "IoC Type not supported"}, self.namespace + + async def delete_search(self, search_id): + # Optional since this may not be supported by the data source API + # Delete the search + return {"code": 200, "success": True} + +def get_url_endpoint(url, endpoint, headers=None): + return url + endpoint + +async def ip_query(self, data): + response_data = dict() + baseurl = ":443/api/v1/indicators/IPv4/%s/" % data + analysis_url = "/otxapi/indicators/ip/analysis/%s/" % data + + try: + query1 = await self.client.call_api(baseurl, 'GET', timeout=self.timeout) + response1 = json.loads(query1.read().decode('utf-8')) if query1.code == 200 else {} + query2 = await self.client.call_api(analysis_url, 'GET', timeout=self.timeout) + response2 = json.loads(query2.read().decode('utf-8')) if query2.code == 200 else {} + + response_data = {**response1, **response2} + + if response_data.get('detections', {}).get('ids_detections'): + response_data['ids_detections'] = response_data['detections']['ids_detections'] + del response_data['detections']['ids_detections'] + + return prepare_response(self, response_data) + except Exception as e: + response_data["code"] = 500 + response_data["error"] = "OS error: {0}".format(e) + return response_data + +async def domain_query(self, data): + response_data = dict() + baseurl = ":443/api/v1/indicators/domain/%s/" % data + analysis_url = "/otxapi/indicators/domain/analysis/%s/" % data + + try: + query1 = await self.client.call_api(baseurl, 'GET', timeout=self.timeout) + response1 = json.loads(query1.read().decode('utf-8')) if query1.code == 200 else {} + query2 = await self.client.call_api(analysis_url, 'GET', timeout=self.timeout) + response2 = json.loads(query2.read().decode('utf-8')) if query2.code == 200 else {} + + response_data = {**response1, **response2} + return prepare_response(self, response_data) + + except Exception as e: + response_data["code"] = 500 + response_data["error"] = "OS error: {0}".format(e) + + return response_data + +async def hash_query(self, data): + response_data = dict() + baseurl = ":443/api/v1/indicators/file/%s/" % data + analysis_url = "/otxapi/indicators/file/analysis/%s/" % data + try: + query1 = await self.client.call_api(baseurl, 'GET', timeout=self.timeout) + response1 = json.loads(query1.read().decode('utf-8')) if query1.code == 200 else {} + query2 = await self.client.call_api(analysis_url, 'GET', timeout=self.timeout) + response2 = json.loads(query2.read().decode('utf-8')) if query2.code == 200 else {} + + response_data = {**response1, **response2} + + rules = response_data.get('analysis', {}).get('plugins', {}).get('cuckoo', {}).get('result', {}).get('suricata', {}).get('rules', {}) + if rules: + response_data['ids_detections'] = rules + + return prepare_response(self, response_data) + except Exception as e: + response_data["code"] = 500 + response_data["error"] = "OS error: {0}".format(e) + + return response_data + +async def url_query(self, data): + response_data = dict() + # data = urllib.parse.quote_plus(data) + data = data.replace('/', '%252F') + analysis_url = "/otxapi/indicators/url/analysis/%s" % data + try: + query1 = await self.client.call_api(analysis_url, 'GET', timeout=self.timeout) + response1 = json.loads(query1.read().decode('utf-8')) if query1.code == 200 else {} + query2 = await self.client.call_api(analysis_url, 'GET', timeout=self.timeout) + response2 = json.loads(query2.read().decode('utf-8')) if query2.code == 200 else {} + response_data = {**response1, **response2} + return prepare_response(self, response_data) + + except Exception as e: + response_data["code"] = 500 + response_data["error"] = "OS error: {0}".format(e) + + return response_data + +def prepare_response(self, response): + response_data = dict() + response_data["code"] = 200 + response_data["data"] = { + "success" : True, + "full" : response + } + + return response_data diff --git a/stix_shifter_modules/alienvault_otx/stix_transmission/delete_connector.py b/stix_shifter_modules/alienvault_otx/stix_transmission/delete_connector.py new file mode 100644 index 000000000..111f8a786 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_transmission/delete_connector.py @@ -0,0 +1,24 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + +class DeleteConnector(BaseDeleteConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + async def delete_query_connection(self, search_id): + try: + response_dict = await self.api_client.delete_search(search_id) + response_code = response_dict["code"] + + # Construct a response object + return_obj = dict() + if response_code == 200: + return_obj['success'] = response_dict['success'] + else: + ErrorResponder.fill_error(return_obj, response_dict, ['message']) + return return_obj + except Exception as err: + self.logger.error('error when deleting search {}:'.format(err)) + raise diff --git a/stix_shifter_modules/alienvault_otx/stix_transmission/error_mapper.py b/stix_shifter_modules/alienvault_otx/stix_transmission/error_mapper.py new file mode 100644 index 000000000..db19bbfbd --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_transmission/error_mapper.py @@ -0,0 +1,50 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + # These are only examples. Change the keys to reflect the error codes that come back from the data source API. + # search does not exist + 1002: ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS, + # The search cannot be created. The requested search ID that was provided in the query expression is already in use. + # Please use a unique search ID (or allow one to be generated). + 1004: ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR.value, + # A request parameter is not valid + 1005: ErrorCode.TRANSMISSION_INVALID_PARAMETER, + # The server might be temporarily unavailable or offline. Please try again later. + 1010: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + # An error occurred during the attempt + 1020: ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR.value, + #error in query + 2000: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + + 400: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + + 403: ErrorCode. TRANSMISSION_FORBIDDEN, + + 401: ErrorCode.TRANSMISSION_AUTH_CREDENTIALS +} + + +class ErrorMapper(): + + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + logger = logger.set_logger(__name__) + + @staticmethod + def set_error_code(json_data, return_obj): + code = None + try: + code = int(json_data['code']) + except Exception: + pass + + error_code = ErrorMapper.DEFAULT_ERROR + + if code in error_mapping: + error_code = error_mapping[code] + + if error_code == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: " + str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_code) diff --git a/stix_shifter_modules/alienvault_otx/stix_transmission/ping_connector.py b/stix_shifter_modules/alienvault_otx/stix_transmission/ping_connector.py new file mode 100644 index 000000000..495c2c033 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_transmission/ping_connector.py @@ -0,0 +1,25 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + +class PingConnector(BasePingConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def ping_connection(self): + try: + response_dict, response_code = await self.api_client.ping_alienvault() + return_obj = dict() + if response_code == 200: + return_obj['success'] = True + return_obj['code'] = response_code + return_obj['connector'] = self.connector + else: + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + self.logger.error(return_obj) + return return_obj + except Exception as err: + self.logger.error('error when pinging datasource {}:'.format(err)) + raise diff --git a/stix_shifter_modules/alienvault_otx/stix_transmission/results_connector.py b/stix_shifter_modules/alienvault_otx/stix_transmission/results_connector.py new file mode 100644 index 000000000..ecbfe238e --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/stix_transmission/results_connector.py @@ -0,0 +1,49 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_json_results_connector import BaseJsonResultsConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import json + +class ResultsConnector(BaseJsonResultsConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def permalink(self, input): + url = "https://otx.alienvault.com/indicator" + try: + if input['dataType'] == "domain": + url += "/url/"+input['data'] + elif input['dataType'] == 'url': + data = input['data'].replace('/', '%252F') + url += "/url/"+data + elif input['dataType'] == "ip": + url += "/ip/"+input['data'] + else: + url += "/file/"+input['data'] + except: + url = "N/A" + + return {"source_name":"OTXQuery_Connector","url":url} + + async def create_results_connection(self, query_data, offset, length): + try: + query_data = query_data.replace('\'', "\"") + query_json = json.loads(query_data) + response, namespace = await self.api_client.get_search_results(query_json) + response_code = response['code'] + return_obj = dict() + if response_code == 200: + response['report'] = response['data'] + response['data'] = query_json['data'] + response['dataType'] = query_json['dataType'] + response['external_reference'] = self.permalink(response) + response['namespace'] = namespace + return_obj['success'] = True + return_obj['data'] = [response] + else: + ErrorResponder.fill_error(return_obj, response) + return_obj['error'] = response['error'] + return return_obj + except Exception as err: + self.logger.error('error when creating search: {}'.format(err)) + raise diff --git a/stix_shifter_modules/alienvault_otx/tests/stix_translation/json_translation.py b/stix_shifter_modules/alienvault_otx/tests/stix_translation/json_translation.py new file mode 100644 index 000000000..92b7f2deb --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/tests/stix_translation/json_translation.py @@ -0,0 +1,625 @@ +ip_value = "203.190.254.239" #benign +query_pattern = "[ipv4-addr:value='"+ip_value+"']" + +domain_value = "moncleroutlets.com" #unknown +query_pattern_domain = "[domain-name:value='"+domain_value+"']" + +hash_value = "16cda323189d8eba4248c0a2f5ad0d8f" #mal +query_pattern_hash = "[file:hashes.MD5='"+hash_value+"']" + +url = "linkprotect.cudasvc.com/url" #anomalous +query_pattern_url = "[url:value='"+url+"']" + +transmitQueryData_ip_benign = { +"data": [ + { + "code": 200, + "data": ip_value, + "report": { + "success": 'true', + "full": { + "whois": "http://whois.domaintools.com/203.190.254.239", + "reputation": 0, + "indicator": ip_value, + "type": "IPv4", + "type_title": "IPv4", + "base_indicator": {}, + "pulse_info": { + "count": 0, + "pulses": [], + "references": [], + "related": { + "alienvault": { + "adversary": [], + "malware_families": [], + "industries": [] + }, + "other": { + "adversary": [], + "malware_families": [], + "industries": [] + } + } + }, + "false_positive": [], + "validation": [], + "asn": "AS24323 aamra networks limited", + "city_data": 'true', + "city": 'null', + "region": 'null', + "continent_code": "AS", + "country_code3": "BGD", + "country_code2": "BD", + "subdivision": 'null', + "latitude": 23.7018, + "postal_code": 'null', + "longitude": 90.3742, + "accuracy_radius": 200, + "country_code": "BD", + "country_name": "Bangladesh", + "dma_code": 0, + "charset": 0, + "area_code": 0, + "flag_url": "/assets/images/flags/bd.png", + "flag_title": "Bangladesh", + "sections": [ + "general", + "geo", + "reputation", + "url_list", + "passive_dns", + "malware", + "nids_list", + "http_scans" + ], + "detections": { + "antivirus_detections": [], + "malicious_benign_ratio": "0 / 0", + "ids_detections": [] + }, + "facts": { + "verdict": 'Whitelisted', + "reverse_dns": "www.icddrb.org", + "otx_telemetry_7_days": 'false', + "otx_telemetry_30_days": 'false', + "otx_telemetry_all": 'false', + "unique_cves_7_days": [], + "unique_cves_30_days": [], + "unique_cves_all": [], + "indicator_popularity": 0, + "ssl_certificates": [ + { + "port": 443, + "subject": "CN=*.icddrb.org", + "issuer": "C=US, O=DigiCert Inc, CN=DigiCert TLS RSA SHA256 2020 CA1", + "ja3": "771,49172,65281-11", + "fingerprint": "20:89:ce:2e:e1:11:8c:d8:ec:0b:4c:83:40:ef:10:35:e8:f8:15:6e:18:89:59:7b:c6:a0:af:9b:96:50:80:af" + } + ], + "number_of_domains_resolving_7_days": 0, + "number_of_domains_resolving_30_days": 0, + "number_of_domains_resolving_all": 3, + "unique_tlds_from_domains_resolving": 1, + "number_of_dynamic_dns_in_pdns": 0, + "url_indicators_from_the_ip_in_av_pulses": 0, + "has_twitter_discussion": 'false', + "ip_classification": 'null', + "has_webserver": 'true', + "has_ssh": 'false', + "has_rdp": 'false', + "has_vnc": 'false', + "has_telnet": 'false', + "has_x11": 'false', + "has_db": [], + "open_ports": [ + 80, + 443 + ], + "is_open_proxy": 'false', + "is_known_scanner": 'false', + "is_tor": 'false', + "is_vpn_node": 'false', + "is_mining_pool": 'false', + "is_external_ip_lookout": 'false', + "is_sinkhole": 'false', + "is_mining_node": 'false', + "is_possible_mirai_infected": 'false', + "intelmq_feeds": [] + } + } + }, + "dataType": "ip", + "external_reference": { + "source_name": "OTXQuery_Connector", + "url": "https://otx.alienvault.com/indicator/ip/203.190.254.239" + }, + "namespace": "9d4bedaf-d351-4f50-930f-f8eb121e5bae" + } + ] +} + +transmitQueryData_ip_benign_num = { +"data": [ + { + "code": 200, + "data": ip_value, + "report": { + "success": 'true', + "full": { + "whois": "http://whois.domaintools.com/203.190.254.239", + "reputation": 0, + "indicator": ip_value, + "type": "IPv4", + "type_title": "IPv4", + "base_indicator": {}, + "pulse_info": { + "count": 0, + "pulses": [], + "references": [], + "related": { + "alienvault": { + "adversary": [], + "malware_families": [], + "industries": [] + }, + "other": { + "adversary": [], + "malware_families": [], + "industries": [] + } + } + }, + "false_positive": [], + "validation": [], + "asn": "AS24323 aamra networks limited", + "city_data": 'true', + "city": 'null', + "region": 'null', + "continent_code": "AS", + "country_code3": "BGD", + "country_code2": "BD", + "subdivision": 'null', + "latitude": 23.7018, + "postal_code": 'null', + "longitude": 90.3742, + "accuracy_radius": 200, + "country_code": "BD", + "country_name": "Bangladesh", + "dma_code": 0, + "charset": 0, + "area_code": 0, + "flag_url": "/assets/images/flags/bd.png", + "flag_title": "Bangladesh", + "sections": [ + "general", + "geo", + "reputation", + "url_list", + "passive_dns", + "malware", + "nids_list", + "http_scans" + ], + "detections": { + "antivirus_detections": [], + "malicious_benign_ratio": "0 / 0", + "ids_detections": [] + }, + "facts": { + "verdict": 'null', + "combined_score": 1, + "reverse_dns": "www.icddrb.org", + "otx_telemetry_7_days": 'false', + "otx_telemetry_30_days": 'false', + "otx_telemetry_all": 'false', + "unique_cves_7_days": [], + "unique_cves_30_days": [], + "unique_cves_all": [], + "indicator_popularity": 0, + "ssl_certificates": [ + { + "port": 443, + "subject": "CN=*.icddrb.org", + "issuer": "C=US, O=DigiCert Inc, CN=DigiCert TLS RSA SHA256 2020 CA1", + "ja3": "771,49172,65281-11", + "fingerprint": "20:89:ce:2e:e1:11:8c:d8:ec:0b:4c:83:40:ef:10:35:e8:f8:15:6e:18:89:59:7b:c6:a0:af:9b:96:50:80:af" + } + ], + "number_of_domains_resolving_7_days": 0, + "number_of_domains_resolving_30_days": 0, + "number_of_domains_resolving_all": 3, + "unique_tlds_from_domains_resolving": 1, + "number_of_dynamic_dns_in_pdns": 0, + "url_indicators_from_the_ip_in_av_pulses": 0, + "has_twitter_discussion": 'false', + "ip_classification": 'null', + "has_webserver": 'true', + "has_ssh": 'false', + "has_rdp": 'false', + "has_vnc": 'false', + "has_telnet": 'false', + "has_x11": 'false', + "has_db": [], + "open_ports": [ + 80, + 443 + ], + "is_open_proxy": 'false', + "is_known_scanner": 'false', + "is_tor": 'false', + "is_vpn_node": 'false', + "is_mining_pool": 'false', + "is_external_ip_lookout": 'false', + "is_sinkhole": 'false', + "is_mining_node": 'false', + "is_possible_mirai_infected": 'false', + "intelmq_feeds": [] + } + } + }, + "dataType": "ip", + "external_reference": { + "source_name": "OTXQuery_Connector", + "url": "https://otx.alienvault.com/indicator/ip/203.190.254.239" + }, + "namespace": "9d4bedaf-d351-4f50-930f-f8eb121e5bae" + } + ] +} + +transmitQueryData_hash_mal = { +"data": [ + { + "code": 200, + "data": hash_value, + "report": { + "success": 'true', + "full": { + "whois": "http://whois.domaintools.com/203.190.254.239", + "reputation": 0, + "type": "md5", + "type_title": "FileHash-MD5", + "indicator": hash_value, + "base_indicator": {}, + "pulse_info": { + "count": 0, + "pulses": [], + "references": [], + "related": { + "alienvault": { + "adversary": [], + "malware_families": [], + "industries": [] + }, + "other": { + "adversary": [], + "malware_families": [], + "industries": [] + } + } + }, + "false_positive": [], + "validation": [], + "asn": "AS24323 aamra networks limited", + "city_data": 'true', + "city": 'null', + "region": 'null', + "continent_code": "AS", + "country_code3": "BGD", + "country_code2": "BD", + "subdivision": 'null', + "latitude": 23.7018, + "postal_code": 'null', + "longitude": 90.3742, + "accuracy_radius": 200, + "country_code": "BD", + "country_name": "Bangladesh", + "dma_code": 0, + "charset": 0, + "area_code": 0, + "flag_url": "/assets/images/flags/bd.png", + "flag_title": "Bangladesh", + "sections": [ + "general", + "geo", + "reputation", + "url_list", + "passive_dns", + "malware", + "nids_list", + "http_scans" + ], + "detections": { + "antivirus_detections": [], + "malicious_benign_ratio": "0 / 0", + "ids_detections": [] + }, + "facts": { + "verdict": 'null', + "combined_score": 12, + "reverse_dns": "www.icddrb.org", + "otx_telemetry_7_days": 'false', + "otx_telemetry_30_days": 'false', + "otx_telemetry_all": 'false', + "unique_cves_7_days": [], + "unique_cves_30_days": [], + "unique_cves_all": [], + "indicator_popularity": 0, + "ssl_certificates": [ + { + "port": 443, + "subject": "CN=*.icddrb.org", + "issuer": "C=US, O=DigiCert Inc, CN=DigiCert TLS RSA SHA256 2020 CA1", + "ja3": "771,49172,65281-11", + "fingerprint": "20:89:ce:2e:e1:11:8c:d8:ec:0b:4c:83:40:ef:10:35:e8:f8:15:6e:18:89:59:7b:c6:a0:af:9b:96:50:80:af" + } + ], + "number_of_domains_resolving_7_days": 0, + "number_of_domains_resolving_30_days": 0, + "number_of_domains_resolving_all": 3, + "unique_tlds_from_domains_resolving": 1, + "number_of_dynamic_dns_in_pdns": 0, + "url_indicators_from_the_ip_in_av_pulses": 0, + "has_twitter_discussion": 'false', + "ip_classification": 'null', + "has_webserver": 'true', + "has_ssh": 'false', + "has_rdp": 'false', + "has_vnc": 'false', + "has_telnet": 'false', + "has_x11": 'false', + "has_db": [], + "open_ports": [ + 80, + 443 + ], + "is_open_proxy": 'false', + "is_known_scanner": 'false', + "is_tor": 'false', + "is_vpn_node": 'false', + "is_mining_pool": 'false', + "is_external_ip_lookout": 'false', + "is_sinkhole": 'false', + "is_mining_node": 'false', + "is_possible_mirai_infected": 'false', + "intelmq_feeds": [] + } + } + }, + "dataType": "hash", + "external_reference": { + "source_name": "OTXQuery_Connector", + "url": "https://otx.alienvault.com/indicator/file/16cda323189d8eba4248c0a2f5ad0d8f" + }, + "namespace": "9d4bedaf-d351-4f50-930f-f8eb121e5bae" + } + ] +} + +transmitQueryData_domain_none = { +"data": [ + { + "code": 200, + "data": domain_value, + "report": { + "success": 'true', + "full": { + "sections": [ + "general", + "geo", + "url_list", + "passive_dns", + "malware", + "whois", + "http_scans" + ], + "whois": "http://whois.domaintools.com/moncleroutlets.com", + "alexa": "http://www.alexa.com/siteinfo/moncleroutlets.com", + "indicator": domain_value, + "type": "domain", + "type_title": "Domain", + "validation": [], + "base_indicator": {}, + "pulse_info": { + "count": 0, + "pulses": [], + "references": [], + "related": { + "alienvault": { + "adversary": [], + "malware_families": [], + "industries": [] + }, + "other": { + "adversary": [], + "malware_families": [], + "industries": [] + } + } + }, + "false_positive": [], + "detections": { + "antivirus_detections": [], + "malicious_benign_ratio": "0 / 0" + }, + "facts": { + "verdict": 'null', + "ip_verdict": 'null', + "ssl_certificates": [], + "current_ip_addresses": [ + "64.190.63.111" + ], + "current_nameservers": [ + "ns1.sedoparking.com.", + "ns2.sedoparking.com." + ], + "current_asns": [ + "AS47846 sedo" + ], + "current_country_codes": [ + "DE" + ], + "domain_resolve_number_of_ips": 1, + "domain_resolve_number_of_asns": 1, + "dns_resolve_malicious_ip": 'false', + "domain_registrar": "TurnCommerce, Inc. DBA NameBright.com", + "domain_creation_date": "2012-01-19T19:32:11", + "domain_registered_last_100_days": 'false', + "otx_telemetry_7_days": 'true', + "otx_telemetry_30_days": 'true', + "otx_telemetry_all": 'false', + "has_webserver": 'false', + "has_twitter_discussion": 'false', + "number_of_open_source_feeds_referencing_this_domain": 0, + "number_of_subdomains": 1, + "domain_has_spf": 'true', + "domain_not_resolving": 'false', + "domain_resolving_to_a_private_range": 'false', + "domain_in_alexa_100k": 'false', + "domain_in_umbrella_100k": 'false', + "urldomain_in_majestic_100k": 'false', + "domain_in_akamai_list": 'false', + "hostname_is_dynamic_dns": 'false', + "domain_number_of_malicious_files_hosted": 0, + "domain_number_of_malicious_files_communicating": 0, + "domain_safebrowsing_detected": 'false', + "domain_blocked_by_umbrella": 'false', + "domain_blocked_by_quad9": 'false', + "domain_blocked_by_akamai": 'false', + "is_ddns_domain": 'false', + "domain_has_its_own_nameserver": 'false', + "is_open_proxy": 'false', + "is_known_scanner": 'false', + "is_tor": 'false', + "is_vpn_node": 'false', + "is_mining_pool": 'false', + "is_external_ip_lookout": 'false', + "is_sinkhole": 'false', + "is_mining_node": 'false', + "ip_is_open_proxy": 'false', + "ip_is_known_scanner": 'false', + "ip_is_tor": 'false', + "ip_is_vpn_node": 'false', + "ip_is_mining_pool": 'false', + "ip_is_external_ip_lookout": 'false', + "ip_is_sinkhole": 'false', + "ip_is_mining_node": 'false', + "urls_from_domain_or_hostname_in_av_pulses": 'false', + "domain_suspicious_tld": 'false', + "is_filesharing": 'false', + "is_common_ocsp": 'false', + "has_wordpress": 'false', + "has_drupal": 'false', + "is_opendir": 'false', + "is_punycode": 'false', + "is_domain_shortener": 'false', + "domain_hosting_phishing": 'false', + "suspended_or_parked_domain": 'true', + "sauron_suspicious_tag": 'false', + "domain_is_dga": 'true', + "domain_is_free_hosting": 'null', + "has_unicode_homoglyph": 'null' + } + } + }, + "dataType": "domain", + "external_reference": { + "source_name": "OTXQuery_Connector", + "url": "https://otx.alienvault.com/indicator/url/moncleroutlets.com" + }, + "namespace": "9d4bedaf-d351-4f50-930f-f8eb121e5bae" + } + ] +} + +transmitQueryData_url_anom = { + "data": [ + { + "code": 200, + "data": url, + "report": { + "success": 'true', + "full": { + "indicators": { + "ip": {}, + "domain": { + "indicator": "cudasvc.com", + "whitelisted_message": [ + { + "source": "akamai", + "message": "Akamai rank: #1106", + "name": "Akamai Popular Domain" + }, + { + "source": "majestic", + "message": "Whitelisted domain cudasvc.com", + "name": "Whitelisted domain" + }, + { + "source": "whitelist", + "message": "Whitelisted domain cudasvc.com", + "name": "Whitelisted domain" + } + ], + "suspicious": 'false', + "pulse_count": 2, + "passivedns_count": 500, + "nids_count": 0, + "url_count": 21317, + "file_count": 0, + "whitelisted": 'true' + }, + "facts": { + "verdict": 'null', + "combined_score": 5 + }, + "hostname": { + "indicator": "linkprotect.cudasvc.com", + "whitelisted_message": [ + { + "source": "akamai", + "message": "Akamai rank: #1106", + "name": "Akamai Popular Domain" + }, + { + "source": "whitelist", + "message": "Whitelisted domain cudasvc.com", + "name": "Whitelisted domain" + }, + { + "source": "majestic", + "message": "Whitelisted domain cudasvc.com", + "name": "Whitelisted domain" + }, + { + "source": "false_positive", + "message": "Known False Positive", + "name": "Known False Positive" + } + ], + "suspicious": 'false', + "pulse_count": 0, + "passivedns_count": 496, + "nids_count": 0, + "url_count": 20369, + "file_count": 0, + "whitelisted": 'true' + } + }, + "stats": { + "indicators_with_pulses": { + "alienvault": 0, + "other": 0 + } + } + } + }, + "dataType": "url", + "external_reference": { + "source_name": "OTXQuery_Connector", + "url": "https://otx.alienvault.com/indicator/url/linkprotect.cudasvc.com/url" + }, + "namespace": "9d4bedaf-d351-4f50-930f-f8eb121e5bae" + } + ] +} \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/tests/stix_translation/test_alienvault_otx_json_to_stix.py b/stix_shifter_modules/alienvault_otx/tests/stix_translation/test_alienvault_otx_json_to_stix.py new file mode 100644 index 000000000..ce157a194 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/tests/stix_translation/test_alienvault_otx_json_to_stix.py @@ -0,0 +1,168 @@ +import json +import unittest +from functools import wraps +from stix_shifter_modules.alienvault_otx.entry_point import EntryPoint +from stix_shifter_modules.alienvault_otx.tests.stix_translation.json_translation import * + +MODULE = "alienvault_otx" +DATA_SOURCE = {"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "OTXQuery_Connector", + "identity_class": "system"} + +options = {'stix_validator':True} +entry_point = EntryPoint(options=options) +translation_options = {} + +extension_types = ["toplevel-property-extension"] +extension_properties = ["x_ibm_original_threat_feed_data", "threat_score", "threat_attributes"] + +class TestAlientvaultOTXResultsToStix(unittest.TestCase): + """ + class to perform unit test case for translate results + """ + + def __init__(self,*args, **kwargs): + super(TestAlientvaultOTXResultsToStix, self).__init__(*args, **kwargs) + self.result_translator = entry_point.create_default_results_translator(dialect='default') + self.result_bundle = self.result_translator.translate_results(data_source=DATA_SOURCE, data=transmitQueryData_ip_benign['data']) + self.result_bundle_objects = self.result_bundle['objects'] + self.result_bundle = self.result_translator.translate_results(data_source=DATA_SOURCE, data=transmitQueryData_ip_benign_num['data']) + self.result_bundle_objects_ben = self.result_bundle['objects'] + self.result_bundle = self.result_translator.translate_results(data_source=DATA_SOURCE, data=transmitQueryData_hash_mal['data']) + self.result_bundle_objects_mal = self.result_bundle['objects'] + self.result_bundle = self.result_translator.translate_results(data_source=DATA_SOURCE, data=transmitQueryData_domain_none['data']) + self.result_bundle_objects_none = self.result_bundle['objects'] + self.result_bundle = self.result_translator.translate_results(data_source=DATA_SOURCE, data=transmitQueryData_url_anom['data']) + self.result_bundle_objects_anomalous = self.result_bundle['objects'] + self.extension_property_names = [] + + @staticmethod + def exists(obj, chain): + """ + Check if the nested keys exist in the dictionary or not + """ + _key = chain.pop(0) + if _key in obj: + return TestAlientvaultOTXResultsToStix.exists(obj[_key], chain) if chain else obj[_key] + + @staticmethod + def get_first(itr, constraint): + return next( + (obj for obj in itr if constraint(obj)), + None + ) + + @staticmethod + def get_first_of_type(itr, typ): + return TestAlientvaultOTXResultsToStix.get_first(itr, lambda o: type(o) == dict and o.get('type') == typ) + + def check_stix_bundle_type(func): + """ + decorator function to convert the data source query result into stix bundle + """ + @wraps(func) + def wrapper_func(self, *args, **kwargs): + assert self.result_bundle['type'] == 'bundle' + return func(self, *args, **kwargs) + return wrapper_func + + @check_stix_bundle_type + def test_stix_identity_prop(self): + """ + to test the identity stix object properties + """ + stix_identity = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects, DATA_SOURCE['type']) + assert 'type' in stix_identity and stix_identity['type'] == DATA_SOURCE['type'] + assert 'name' in stix_identity and stix_identity['name'] == DATA_SOURCE['name'] + assert 'identity_class' in stix_identity and stix_identity['identity_class'] == DATA_SOURCE['identity_class'] + + @check_stix_bundle_type + def test_stix_extension_prop(self): + """ + to test the extension stix object properties + """ + sdo_type = 'extension-definition' + stix_extension = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects, sdo_type) + assert 'type' in stix_extension and stix_extension['type'] == sdo_type + assert 'name' in stix_extension + assert 'version' in stix_extension + assert 'extension_types' in stix_extension and stix_extension['extension_types'] == extension_types + assert 'extension_properties' in stix_extension and stix_extension['extension_properties'] == extension_properties + + @check_stix_bundle_type + def test_stix_indicator_prop(self): + """ + to test the indicator stix object properties + """ + sdo_type = 'indicator' + stix_indicator = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects, sdo_type) + assert 'type' in stix_indicator and stix_indicator['type'] == sdo_type + assert 'pattern' in stix_indicator and stix_indicator['pattern'] == query_pattern + assert 'valid_from' in stix_indicator + assert 'indicator_types' in stix_indicator and len(stix_indicator['indicator_types']) == 1 \ + and stix_indicator['indicator_types'][0] == 'benign' + + @check_stix_bundle_type + def test_stix_indicator_prop(self): + """ + to test the indicator stix object properties + """ + sdo_type = 'indicator' + stix_indicator = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects_ben, sdo_type) + assert 'type' in stix_indicator and stix_indicator['type'] == sdo_type + assert 'pattern' in stix_indicator and stix_indicator['pattern'] == query_pattern + assert 'valid_from' in stix_indicator + assert 'indicator_types' in stix_indicator and len(stix_indicator['indicator_types']) == 1 \ + and stix_indicator['indicator_types'][0] == 'benign' + + @check_stix_bundle_type + def test_stix_indicator_prop_mal(self): + """ + to test the indicator stix object properties + """ + sdo_type = 'indicator' + stix_indicator = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects_mal, sdo_type) + assert 'type' in stix_indicator and stix_indicator['type'] == sdo_type + assert 'pattern' in stix_indicator and stix_indicator['pattern'] == query_pattern_hash + assert 'valid_from' in stix_indicator + assert 'indicator_types' in stix_indicator and len(stix_indicator['indicator_types']) == 1 \ + and stix_indicator['indicator_types'][0] == 'malicious-activity' + + @check_stix_bundle_type + def test_stix_indicator_prop_unknown(self): + """ + to test the indicator stix object properties + """ + sdo_type = 'indicator' + stix_indicator = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects_none, sdo_type) + assert 'type' in stix_indicator and stix_indicator['type'] == sdo_type + assert 'pattern' in stix_indicator and stix_indicator['pattern'] == query_pattern_domain + assert 'valid_from' in stix_indicator + assert 'indicator_types' in stix_indicator and len(stix_indicator['indicator_types']) == 1 \ + and stix_indicator['indicator_types'][0] == 'unknown' + + @check_stix_bundle_type + def test_stix_indicator_prop_anomalous(self): + """ + to test the indicator stix object properties + """ + sdo_type = 'indicator' + stix_indicator = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects_anomalous, sdo_type) + assert 'type' in stix_indicator and stix_indicator['type'] == sdo_type + assert 'pattern' in stix_indicator and stix_indicator['pattern'] == query_pattern_url + assert 'valid_from' in stix_indicator + assert 'indicator_types' in stix_indicator and len(stix_indicator['indicator_types']) == 1 \ + and stix_indicator['indicator_types'][0] == 'anomalous-activity' + + @check_stix_bundle_type + def test_stix_indicator_extensions_prop(self): + """ + to test the indicator stix object extensions properties + """ + stix_extension = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects, 'extension-definition') + stix_indicator = TestAlientvaultOTXResultsToStix.get_first_of_type(self.result_bundle_objects, 'indicator') + assert 'x_ibm_original_threat_feed_data' in stix_indicator + extension_property = extension_properties[0] + property_name = "x_ibm_original_threat_feed_data.full" + is_exist = TestAlientvaultOTXResultsToStix.exists(stix_indicator, property_name.split(".")) + assert is_exist is not None + assert stix_indicator[extension_property]["full"][0] diff --git a/stix_shifter_modules/alienvault_otx/tests/stix_translation/test_alienvault_otx_stix_to_query.py b/stix_shifter_modules/alienvault_otx/tests/stix_translation/test_alienvault_otx_stix_to_query.py new file mode 100644 index 000000000..99d86ad15 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/tests/stix_translation/test_alienvault_otx_stix_to_query.py @@ -0,0 +1,131 @@ +import unittest +from stix_shifter.stix_translation import stix_translation + +translation = stix_translation.StixTranslation() +MODULE = 'alienvault_otx' + +def _test_query_assertions(query, queries): + assert isinstance(query, dict) is True + assert 'queries' in query + assert query['queries'] == [queries] + + +class TestAlientvaultOTXStixToQuery(unittest.TestCase, object): + + @staticmethod + def get_query_translation_result(stix_pattern, options={}): + return translation.translate(MODULE, 'query', MODULE, stix_pattern, options) + + def test_ipv4_query(self): + stix_pattern = "[ipv4-addr:value='194.147.78.155']" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': '194.147.78.155', 'dataType': 'ip'}" + _test_query_assertions(query, queries) + + def test_ipv6_query(self): + stix_pattern = "[ipv6-addr:value = '3001:0:0:0:0:0:0:2']" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': '3001:0:0:0:0:0:0:2', 'dataType': 'ip'}" + _test_query_assertions(query, queries) + + def test_multi_ipv4_expression_query(self): + stix_pattern = "([ipv4-addr:value = '194.147.78.155'] OR [ipv4-addr:value = '198.51.100.10'])" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': '198.51.100.10', 'dataType': 'ip'}" + _test_query_assertions(query, queries) + + def test_url_query(self): + stix_pattern = "[url:value='https://test.com']" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': 'https://test.com', 'dataType': 'url'}" + _test_query_assertions(query, queries) + + def test_NOT_and_not_equals_operators(self): + search_string1 = "www.example.com" + search_string2 = "www.example.ca" + stix_pattern = "[url:value != '{}' OR url:value NOT = '{}']".format(search_string1, search_string2) + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': 'www.example.com', 'dataType': 'url'}" + _test_query_assertions(query, queries) + + def test_file_hash_query(self): + stix_pattern = "[file:hashes.'SHA-1'='D5DD920BE5BCFEB904E95DA4B6D0CCCA0727D692']" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': 'D5DD920BE5BCFEB904E95DA4B6D0CCCA0727D692', 'dataType': 'hash'}" + _test_query_assertions(query, queries) + + def test_file_hash_md5_query(self): + stix_pattern = "[file:hashes.'MD5'='16cda323189d8eba4248c0a2f5ad0d8f']" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': '16cda323189d8eba4248c0a2f5ad0d8f', 'dataType': 'hash'}" + _test_query_assertions(query, queries) + + def test_generic_filehash_query(self): + stix_pattern = "[file:hashes.'SHA-256' = 'd7fc5162511d42d22462ad5b4c716b73903a677806119f9ad0314763ccd719ca']" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': 'd7fc5162511d42d22462ad5b4c716b73903a677806119f9ad0314763ccd719ca', 'dataType': 'hash'}" + _test_query_assertions(query, queries) + + def test_domain_query(self): + stix_pattern = "[domain-name:value='test.com']" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': 'test.com', 'dataType': 'domain'}" + _test_query_assertions(query, queries) + + def test_multi_expression_query(self): + stix_pattern = "[domain-name:value='test.com' OR ipv4-addr:value='194.147.78.155']" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': 'test.com', 'dataType': 'domain'}" + _test_query_assertions(query, queries) + + def test_not_comp_exp(self): + """ + Test with NOT operator + :return: + """ + stix_pattern = "[ipv4-addr:value NOT = '172.31.60.104'] START t'2020-05-01T08:43:10.003Z' " \ + "STOP t'2020-10-30T10:43:10.003Z'" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': '172.31.60.104', 'dataType': 'ip'}" + _test_query_assertions(query, queries) + + def test_in_comp_exp(self): + """ + Test with IN operator + """ + stix_pattern = "[ipv4-addr:value IN ('172.31.60.104','94.147.78.155')]" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': '(172.31.60.104 OR 94.147.78.155)', 'dataType': 'ip'}" + _test_query_assertions(query, queries) + + def test_one_obser_is_super_set_operator_network(self): + """ + to test single observation with an un-supported operator + """ + stix_pattern = "[ipv4-addr:value ISSUPERSET '172.217.0.0/24'] " \ + "START t'2019-04-10T08:43:10.003Z' STOP t'2019-04-23T10:43:10.003Z'" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + assert query['success'] is False + assert query['code'] == 'mapping_error' + # assert query['error'] == "data mapping error : Unable to map the following STIX objects and properties to data source fields: []" + + def test_like_comp_exp(self): + """ + Test with LIKE operator + """ + stix_pattern = "[ipv4-addr:value LIKE '172.31.60.104'] START t'2020-10-01T08:43:10.003Z' " \ + "STOP t'2020-10-30T10:43:10.003Z'" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': '%172.31.60.104%', 'dataType': 'ip'}" + _test_query_assertions(query, queries) + + def test_matches_comp_exp(self): + """ + Test with MATCHES operator + :return: + """ + stix_pattern = "[ipv4-addr:value MATCHES '\\\\d+'] START t'2020-10-01T08:43:10.003Z' STOP " \ + "t'2020-10-30T10:43:10.003Z'" + query = TestAlientvaultOTXStixToQuery.get_query_translation_result(stix_pattern) + queries = "{'data': '.*\\\\\\\\d+.*', 'dataType': 'ip'}" + _test_query_assertions(query, queries) \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/tests/stix_transmission/json_transmission.py b/stix_shifter_modules/alienvault_otx/tests/stix_transmission/json_transmission.py new file mode 100644 index 000000000..77d6baeed --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/tests/stix_transmission/json_transmission.py @@ -0,0 +1,77 @@ +from stix_shifter_utils.stix_transmission.utils.RestApiClient import ResponseWrapper + +DATA = { + "code": 200, + "data": { + "success": 'True', + "full": { + "data": { + "attributes": { + "regional_internet_registry": "ARIN", + "network": "20.64.0.0/10", + "tags": [], + "country": "US", + "as_owner": "MICROSOFT-CORP-MSN-AS-BLOCK", + "last_analysis_stats": { + "harmless": 80, + "malicious": 4, + "suspicious": 0, + "undetected": 10, + "timeout": 0 + }, + "asn": 8075, + "whois_date": 1651598323, + "last_analysis_results": { + "CMC Threat Intelligence": { + "category": "malicious", + "result": "malware", + "method": "blacklist", + "engine_name": "CMC Threat Intelligence" + }, + "Baidu-International": { + "category": "harmless", + "result": "clean", + "method": "blacklist", + "engine_name": "Baidu-International" + } + }, + "reputation": 0, + "last_modification_date": 1653039072, + "total_votes": { + "harmless": 0, + "malicious": 0 + }, + "continent": "NA", + "whois": "NetRange: 20.33.0.0 - 20.128.255.255\nCIDR: 20.48.0.0/12, 20.34.0.0/15, 20.33.0.0/16, 20.64.0.0/10, 20.40.0.0/13, 20.128.0.0/16, 20.36.0.0/14\nNetName: MSFT\nNetHandle: NET-20-33-0-0-1\nParent: NET20 (NET-20-0-0-0-0)\nNetType: Direct Allocation\nOriginAS: \nOrganization: Microsoft Corporation (MSFT)\nRegDate: 2017-10-18\nUpdated: 2021-12-14\nRef: https://rdap.arin.net/registry/ip/20.33.0.0\nOrgName: Microsoft Corporation\nOrgId: MSFT\nAddress: One Microsoft Way\nCity: Redmond\nStateProv: WA\nPostalCode: 98052\nCountry: US\nRegDate: 1998-07-10\nUpdated: 2022-03-28\nComment: To report suspected security issues specific to traffic emanating from Microsoft online services, including the distribution of malicious content or other illicit or illegal material through a Microsoft online service, please submit reports to:\r\nComment: * https://cert.microsoft.com. \r\nComment: \r\nComment: For SPAM and other abuse issues, such as Microsoft Accounts, please contact:\r\nComment: * abuse@microsoft.com. \r\nComment: \r\nComment: To report security vulnerabilities in Microsoft products and services, please contact:\r\nComment: * secure@microsoft.com. \r\nComment: \r\nComment: For legal and law enforcement-related requests, please contact:\r\nComment: * msndcc@microsoft.com\r\nComment: \r\nComment: For routing, peering or DNS issues, please \r\nComment: contact:\r\nComment: * IOC@microsoft.com\nRef: https://rdap.arin.net/registry/entity/MSFT\nOrgTechHandle: MRPD-ARIN\nOrgTechName: Microsoft Routing, Peering, and DNS\nOrgTechPhone: +1-425-882-8080 \nOrgTechEmail: IOC@microsoft.com\nOrgTechRef: https://rdap.arin.net/registry/entity/MRPD-ARIN\nOrgAbuseHandle: MAC74-ARIN\nOrgAbuseName: Microsoft Abuse Contact\nOrgAbusePhone: +1-425-882-8080 \nOrgAbuseEmail: abuse@microsoft.com\nOrgAbuseRef: https://rdap.arin.net/registry/entity/MAC74-ARIN\nOrgTechHandle: IPHOS5-ARIN\nOrgTechName: IPHostmaster, IPHostmaster \nOrgTechPhone: +1-425-538-6637 \nOrgTechEmail: iphostmaster@microsoft.com\nOrgTechRef: https://rdap.arin.net/registry/entity/IPHOS5-ARIN\n" + }, + "type": "ip_address", + "id": "20.110.52.48", + "links": { + "self": "https://otx.alienvault.com/indicator/ip/20.110.52.48" + }, + "info": { + "detected_urls": { + "scan_date": "2022-05-20 05:31:12", + "positives": 4, + "total": 94 + }, + "permalink": "https://otx.alienvault.com/indicator/ip/20.110.52.48" + } + } + } + }, + "data": "20.110.52.48", + "dataType": "ip" +} + +class http_struct: + def __init__(self): + self.content = "" + self.headers = "" + self.status_code = 0 + +HTTP_RESPONSE = http_struct() +HTTP_RESPONSE.content = bytearray(b'{"detections": {"antivirus_detections": [], "malicious_benign_ratio": "0 / 0", "ids_detections": []}, "facts": {"verdict": "Whitelisted", "reverse_dns": null, "otx_telemetry_7_days": false, "otx_telemetry_30_days": false, "otx_telemetry_all": false, "unique_cves_7_days": [], "unique_cves_30_days": [], "unique_cves_all": [], "indicator_popularity": 0, "ssl_certificates": [], "number_of_domains_resolving_7_days": 0, "number_of_domains_resolving_30_days": 0, "number_of_domains_resolving_all": 0, "unique_tlds_from_domains_resolving": 0, "number_of_dynamic_dns_in_pdns": 0, "url_indicators_from_the_ip_in_av_pulses": 0, "has_twitter_discussion": false, "ip_classification": "Cloud provider", "has_webserver": false, "has_ssh": false, "has_rdp": false, "has_vnc": false, "has_telnet": false, "has_x11": false, "has_db": [], "open_ports": [], "is_open_proxy": false, "is_known_scanner": false, "is_tor": false, "is_vpn_node": false, "is_mining_pool": false, "is_external_ip_lookout": false, "is_sinkhole": false, "is_mining_node": false, "is_possible_mirai_infected": false, "intelmq_feeds": []}}') +HTTP_RESPONSE.status_code = 200 + +DATA_TRANS = ResponseWrapper(HTTP_RESPONSE) \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/tests/stix_transmission/test_alienvault_otx_transmission.py b/stix_shifter_modules/alienvault_otx/tests/stix_transmission/test_alienvault_otx_transmission.py new file mode 100644 index 000000000..eaced4d21 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/tests/stix_transmission/test_alienvault_otx_transmission.py @@ -0,0 +1,73 @@ +# tests.py +import json +import unittest +from unittest import mock +from functools import wraps +from stix_shifter_modules.alienvault_otx.stix_transmission.api_client import APIClient +from json_transmission import * + +namespace = '8af42ea1-e30d-41a2-a3ee-1aec759cf789' + +connection = { + "host": "www.data.com", + "port": 443, + "namespace": namespace, + "options": { + "timeout": 600 + } +} +config = { + "auth": { + "key": "testingKey" + } +} + +SAMPLE_DATA_IP = {"data": "20.110.52.48", "dataType": "ip"} +SAMPLE_DATA_HASH = {"data": "16cda323189d8eba4248c0a2f5ad0d8f", "dataType": "hash"} +SAMPLE_DATA_URL = {"data": "linkprotect.cudasvc.com/url", "dataType": "url"} +SAMPLE_DATA_DOMAIN = {"data": "moncleroutlets.com", "dataType": "domain"} + +class TestAlientvaultOTXTransmission(unittest.TestCase): + def __init__(self,*args, **kwargs): + super(TestAlientvaultOTXTransmission, self).__init__(*args, **kwargs) + self.api_client = APIClient(connection, config) + + @mock.patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.RestApiClientAsync.call_api') + async def test_alienvault_otx_results_ip(self, mock_client_get_json): + mock_client_get_json.return_value = DATA_TRANS + + response = await self.api_client.get_search_results(SAMPLE_DATA_IP) + + assert response[0]['data']['success'] == True + assert response[0]['code'] == 200 + assert response[0]['data']['full']['facts']['verdict'] == "Whitelisted" + + @mock.patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.RestApiClientAsync.call_api') + async def test_alienvault_otx_results_hash(self, mock_client_get_json): + mock_client_get_json.return_value = DATA_TRANS + + response = await self.api_client.get_search_results(SAMPLE_DATA_HASH) + + assert response[0]['data']['success'] == True + assert response[0]['code'] == 200 + assert response[0]['data']['full']['facts']['verdict'] == "Whitelisted" + + @mock.patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.RestApiClientAsync.call_api') + async def test_alienvault_otx_results_URL(self, mock_client_get_json): + mock_client_get_json.return_value = DATA_TRANS + + response = await self.api_client.get_search_results(SAMPLE_DATA_URL) + + assert response[0]['data']['success'] == True + assert response[0]['code'] == 200 + assert response[0]['data']['full']['facts']['verdict'] == "Whitelisted" + + @mock.patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.RestApiClientAsync.call_api') + async def test_alienvault_otx_results_domain(self, mock_client_get_json): + mock_client_get_json.return_value = DATA_TRANS + + response = await self.api_client.get_search_results(SAMPLE_DATA_DOMAIN) + + assert response[0]['data']['success'] == True + assert response[0]['code'] == 200 + assert response[0]['data']['full']['facts']['verdict'] == "Whitelisted" \ No newline at end of file diff --git a/stix_shifter_modules/alienvault_otx/tests/stix_transmission/test_alienvault_otx_transmit.py b/stix_shifter_modules/alienvault_otx/tests/stix_transmission/test_alienvault_otx_transmit.py new file mode 100644 index 000000000..61c24e777 --- /dev/null +++ b/stix_shifter_modules/alienvault_otx/tests/stix_transmission/test_alienvault_otx_transmit.py @@ -0,0 +1,293 @@ + +from stix_shifter.stix_transmission import stix_transmission +from unittest.mock import patch +import unittest +from collections import namedtuple +from types import SimpleNamespace +from json_transmission import * +from json import dumps + +MODULE_NAME = "alienvault_otx" +SAMPLE_DATA_IP = '{"data": "203.190.254.239", "dataType": "ip"}' +SAMPLE_DATA_HASH = '{"data": "16cda323189d8eba4248c0a2f5ad0d8f", "dataType": "hash"}' +SAMPLE_DATA_URL = '{"data": "linkprotect.cudasvc.com/url", "dataType": "url"}' +SAMPLE_DATA_DOMAIN = '{"data": "moncleroutlets.com", "dataType": "domain"}' + +namespace = '9d4bedaf-d351-4f50-930f-f8eb121e5bae' + +connection = { + "namespace":namespace +} +config = { + "auth": { + "key": "testingKey" + } +} +Response = namedtuple('Response', ['data', 'response_code']) + +class MockHttpResponse: + def __init__(self, string): + self.string = string + +class ResponseWrapper: + def __init__(self, obj, code): + self.object = str.encode(dumps(obj)) + self.code = code + + def read(self): + return self.object + +@patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.__init__', autospec=True) +class TestAlientvaultOTXConnection(unittest.TestCase, object): + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.ping_alienvault') + def test_alienvault_otx_ping(self, mock_ping_response, mock_api_client): + mock_api_client.return_value = None + mock_ping_response.return_value = {"success":True, "code": 200} + + transmission = stix_transmission.StixTransmission( + MODULE_NAME, connection, config) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.ping_alienvault') + def test_alienvault_otx_ping_error(self, mock_ping_response, mock_api_client): + response = MockHttpResponse('/exception') + mock_api_client.return_value = None + mock_ping_response.return_value = {"success":True, "code": 404} + + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.ping_alienvault') + def test_alienvault_otx_ping_exception(self, mock_ping_response, mock_api_client): + response = MockHttpResponse('/exception') + mock_api_client.return_value = None + mock_ping_response.return_value = {"success":True, "code": 404} + mock_ping_response.side_effect = Exception('an error occured retriving ping information') + + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.ping_alienvault') + def test_alienvault_otx_ping_exception2(self, mock_ping_response, mock_api_client): + response = MockHttpResponse('/exception') + mock_api_client.return_value = None + mock_ping_response.return_value = {"success":False} + mock_ping_response.side_effect = Exception('an error occured retriving ping information') + + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_alienvault_otx_results_ip(self, mock_result_connection, mock_api_client): + + mock_api_client.return_value = None + mock_result_connection.return_value = DATA.copy(), namespace + + transmission = stix_transmission.StixTransmission( + MODULE_NAME, connection, config) + query_response = transmission.query(SAMPLE_DATA_IP) + + assert query_response is not None + assert 'search_id' in query_response + assert query_response['search_id'] == SAMPLE_DATA_IP + + search_results_response = transmission.results( + query_response['search_id'], 0, 9) + report = search_results_response['data'][0] + + assert 'data' in report + assert 'dataType' in report + assert 'success' in search_results_response + assert search_results_response['success'] is True + assert type(search_results_response['data']) is list + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_alienvault_otx_results_hash(self, mock_result_connection, mock_api_client): + + mock_api_client.return_value = None + mock_result_connection.return_value = DATA.copy(), namespace + + transmission = stix_transmission.StixTransmission( + MODULE_NAME, connection, config) + query_response = transmission.query(SAMPLE_DATA_HASH) + + assert query_response is not None + assert 'search_id' in query_response + assert query_response['search_id'] == SAMPLE_DATA_HASH + + search_results_response = transmission.results( + query_response['search_id'], 0, 9) + report = search_results_response['data'][0] + + assert 'data' in report + assert 'dataType' in report + assert 'success' in search_results_response + assert search_results_response['success'] is True + assert type(search_results_response['data']) is list + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_alienvault_otx_results_url(self, mock_result_connection, mock_api_client): + + mock_api_client.return_value = None + mock_result_connection.return_value = DATA.copy(), namespace + + transmission = stix_transmission.StixTransmission( + MODULE_NAME, connection, config) + query_response = transmission.query(SAMPLE_DATA_URL) + + assert query_response is not None + assert 'search_id' in query_response + assert query_response['search_id'] == SAMPLE_DATA_URL + + search_results_response = transmission.results( + query_response['search_id'], 0, 9) + report = search_results_response['data'][0] + + assert 'data' in report + assert 'dataType' in report + assert 'success' in search_results_response + assert search_results_response['success'] is True + assert type(search_results_response['data']) is list + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_alienvault_otx_results_domain(self, mock_result_connection, mock_api_client): + + mock_api_client.return_value = None + mock_result_connection.return_value = DATA.copy(), namespace + + transmission = stix_transmission.StixTransmission( + MODULE_NAME, connection, config) + query_response = transmission.query(SAMPLE_DATA_DOMAIN) + + assert query_response is not None + assert 'search_id' in query_response + assert query_response['search_id'] == SAMPLE_DATA_DOMAIN + + search_results_response = transmission.results( + query_response['search_id'], 0, 9) + report = search_results_response['data'][0] + + assert 'data' in report + assert 'dataType' in report + assert 'success' in search_results_response + assert search_results_response['success'] is True + assert type(search_results_response['data']) is list + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_alienvault_otx_results_error(self, mock_result_connection, mock_api_client): + mock_api_client.return_value = None + mock_data = DATA = { + "error": "Invalid", + "success": False, + "code": 400 + } + mock_result_connection.return_value = mock_data, namespace + mock_result_connection.side_effect = Exception('an error occured retriving ping information') + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + query_response = transmission.query(SAMPLE_DATA_IP) + + assert query_response is not None + assert 'search_id' in query_response + assert query_response['search_id'] == SAMPLE_DATA_IP + + search_results_response = transmission.results(query_response['search_id'], 0, 9) + assert 'success' in search_results_response + assert search_results_response['success'] is False + assert 'code' in search_results_response, search_results_response['code'] == 'invalid_query' + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_alienvault_otx_results_error_code(self, mock_result_connection, mock_api_client): + mock_api_client.return_value = None + mock_data = DATA = { + "error": "Invalid", + "success": False, + "code": 400 + } + mock_result_connection.return_value = mock_data, namespace + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + query_response = transmission.query(SAMPLE_DATA_IP) + + assert query_response is not None + assert 'search_id' in query_response + assert query_response['search_id'] == SAMPLE_DATA_IP + + search_results_response = transmission.results(query_response['search_id'], 0, 9) + assert 'success' in search_results_response + assert search_results_response['success'] is False + assert 'code' in search_results_response, search_results_response['code'] == 'invalid_query' + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.delete_search', autospec=True) + def test_delete_query(self, mock_delete_response, mock_api_client): + error_msg = 'an error occured while checking the if the query is deleted' + mock_api_client.return_value = None + mock_delete_response.return_value = {"code": 200, "success": True} + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + query_response = transmission.delete(SAMPLE_DATA_IP) + assert 'success' in query_response, query_response['success'] is True + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.delete_search', autospec=True) + def test_delete_query_error(self, mock_delete_response, mock_api_client): + error_msg = 'an error occured while checking the if the query is deleted' + mock_api_client.return_value = None + mock_delete_response.return_value = {"code": 404, "success": False} + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + query_response = transmission.delete(SAMPLE_DATA_IP) + assert 'success' in query_response, query_response['success'] is False + assert 'error' in query_response, query_response['error'] == error_msg + + @patch('stix_shifter_modules.alienvault_otx.stix_transmission.api_client.APIClient.delete_search', autospec=True) + def test_delete_query_exception(self, mock_delete_response, mock_api_client): + error_msg = 'an error occured while checking the if the query is deleted' + mock_api_client.return_value = None + mock_delete_response.return_value = False + mock_delete_response.side_effect = Exception(error_msg) + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + query_response = transmission.delete("") + assert 'success' in query_response, query_response['success'] is False + assert 'error' in query_response, query_response['error'] == error_msg + + +class TestMockingDemo(unittest.IsolatedAsyncioTestCase): + async def test_otx_ping(self): + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + ping_response = await transmission.ping_async() + assert ping_response is not None + assert ping_response['success'] is False + assert 'code' in ping_response and ping_response['code'] == 'authentication_fail' + assert 'connector' in ping_response and ping_response['connector'] == 'alienvault_otx' + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api', autospec=True) + async def test_otx_get_search_results_ip(self, mock_call_api): + + mock_call_api.side_effect = [ + ResponseWrapper(DATA, 200), + ResponseWrapper(DATA, 401), + ResponseWrapper(DATA, 401) + ] + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + results_response = await transmission.results_async(SAMPLE_DATA_IP, 1, 1) + assert results_response is not None + assert results_response['success'] is True + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api', autospec=True) + async def test_otx_get_search_results_url(self, mock_call_api): + + mock_call_api.side_effect = [ + ResponseWrapper(DATA, 200), + ResponseWrapper(DATA, 401), + ResponseWrapper(DATA, 401) + ] + transmission = stix_transmission.StixTransmission(MODULE_NAME, connection, config) + results_response = await transmission.results_async(SAMPLE_DATA_URL, 1, 1) + assert results_response is not None + assert results_response['success'] is True + diff --git a/stix_shifter_modules/azure_sentinel/azure_sentinel_supported_stix.md b/stix_shifter_modules/azure_sentinel/azure_sentinel_supported_stix.md index 8fde796a2..a5944643f 100644 --- a/stix_shifter_modules/azure_sentinel/azure_sentinel_supported_stix.md +++ b/stix_shifter_modules/azure_sentinel/azure_sentinel_supported_stix.md @@ -1,4 +1,4 @@ -##### Updated on 04/28/23 +##### Updated on 05/02/23 ## Microsoft Graph Security ### Supported STIX Operators *Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).* @@ -45,7 +45,7 @@ | **process**:pid | processes.processId, processes.parentProcessId, registryKeyStates.processId | | **process**:created | processes.createdDateTime | | **process**:parent_ref.pid | processes.parentProcessId | -| **process**:binary_ref.path | processes.path | +| **process**:binary_ref.parent_directory_ref.path | processes.path | | **domain-name**:value | hostStates.fqdn, hostStates.netBiosName, networkConnections.destinationDomain, userStates.domainName | | **user-account**:user_id | userStates.accountName, processes.accountName, userStates.aadUserId | | **user-account**:account_login | userStates.logonId | @@ -56,9 +56,9 @@ | **software**:version | vendorInformation.providerVersion | | **url**:value | networkConnections.destinationUrl | | **windows-registry-key**:key | registryKeyStates.key | -| **windows-registry-key**:extensions.windows-registry-value-type.valueData | registryKeyStates.valueData | -| **windows-registry-key**:extensions.windows-registry-value-type.name | registryKeyStates.valueName | -| **windows-registry-key**:extensions.windows-registry-value-type.valueType | registryKeyStates.valueType | +| **windows-registry-key**:values[*].data | registryKeyStates.valueData | +| **windows-registry-key**:values[*].name | registryKeyStates.valueName | +| **windows-registry-key**:values[*].data_type | registryKeyStates.valueType | | **x-msazure-sentinel**:tenant_id | azureTenantId | | **x-msazure-sentinel**:subscription_id | azureSubscriptionId | | **x-msazure-sentinel-alert**:activityGroupName | activityGroupName | @@ -148,10 +148,6 @@ | domain-name | value | destinationDomain | | domain-name | value | domainName | |
| | | -| extensions | windows-registry-value-type.valueData | registryKeyStates | -| extensions | windows-registry-value-type.name | registryKeyStates | -| extensions | windows-registry-value-type.valuetype | registryKeyStates | -|
| | | | file | hashes.SHA-256 | sha256 | | file | hashes.SHA-1 | sha1 | | file | hashes.MD5 | md5 | @@ -201,6 +197,9 @@ | user-account | account_login | logonId | |
| | | | windows-registry-key | key | registryKeyStates | +| windows-registry-key | values.data | registryKeyStates | +| windows-registry-key | values.name | registryKeyStates | +| windows-registry-key | values.data_type | registryKeyStates | |
| | | | x-ibm-finding | dst_application_ref | destinationServiceName | | x-ibm-finding | createddatetime | createdDateTime | diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/alert_from_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/alert_from_stix_map.json index 6c41fd31f..640f7a937 100644 --- a/stix_shifter_modules/azure_sentinel/stix_translation/json/alert_from_stix_map.json +++ b/stix_shifter_modules/azure_sentinel/stix_translation/json/alert_from_stix_map.json @@ -58,7 +58,7 @@ "pid": ["processes.processId", "processes.parentProcessId", "registryKeyStates.processId"], "created": ["processes.createdDateTime"], "parent_ref.pid": ["processes.parentProcessId"], - "binary_ref.path": ["processes.path"], + "binary_ref.parent_directory_ref.path": ["processes.path"], "x_integrityLevel": ["processes.integrityLevel"], "x_isElevated": ["processes.isElevated"] } @@ -100,9 +100,9 @@ "windows-registry-key": { "fields": { "key": ["registryKeyStates.key"], - "extensions.windows-registry-value-type.valueData": [ "registryKeyStates.valueData" ], - "extensions.windows-registry-value-type.name": [ "registryKeyStates.valueName" ], - "extensions.windows-registry-value-type.valueType": [ "registryKeyStates.valueType" ] + "values[*].data": ["registryKeyStates.valueData"], + "values[*].name": ["registryKeyStates.valueName"], + "values[*].data_type": ["registryKeyStates.valueType"] } }, "x-msazure-sentinel": { diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/alertV2_from_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/alertV2_from_stix_map.json new file mode 100644 index 000000000..2975425e4 --- /dev/null +++ b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/alertV2_from_stix_map.json @@ -0,0 +1,16 @@ +{ + "software": { + "fields": { + "name": ["serviceSource"] } + }, + "x-ibm-finding": { + "fields": { + "severity": ["severity"], + "x_assignedTo": ["assignedTo"], + "x_classification": ["classification"], + "x_determination": ["determination"], + "x_lastUpdateDateTime": ["lastUpdateDateTime"], + "x_status": ["status"] + } + } +} diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/alert_from_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/alert_from_stix_map.json new file mode 100644 index 000000000..9a581f0af --- /dev/null +++ b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/alert_from_stix_map.json @@ -0,0 +1,177 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "networkConnections.sourceAddress", + "networkConnections.destinationAddress", + "networkConnections.natSourceAddress", + "networkConnections.natDestinationAddress" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": ["networkConnections.sourceAddress", "networkConnections.destinationAddress"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["networkConnections.sourcePort", "networkConnections.natSourcePort", "networkConnections.natDestinationPort"], + "dst_port": ["networkConnections.destinationPort", "networkConnections.natDestinationPort"], + "protocols[*]": ["networkConnections.protocol"], + "src_ref.value": ["networkConnections.sourceAddress"], + "dst_ref.value": ["networkConnections.destinationAddress"], + "x_applicationName": ["networkConnections.applicationName"], + "x_direction": ["networkConnections.direction"], + "x_domainRegisteredDateTime": ["networkConnections.domainRegisteredDateTime"], + "x_localDnsName": ["networkConnections.localDnsName"], + "x_riskScore": ["networkConnections.riskScore"], + "x_status": ["networkConnections.status"], + "x_urlParameters": ["networkConnections.urlParameters"] + } + }, + "directory": { + "fields": { + "path": ["fileStates.path", "process.path"] + } + }, + "file": { + "fields": { + "parent_directory_ref.path": ["fileStates.path"], + "name": ["fileStates.name"], + "hashes.'SHA-256'": ["fileStates.fileHash.hashValue"], + "hashes.'SHA-1'": ["fileStates.fileHash.hashValue"], + "hashes.MD5": ["fileStates.fileHash.hashValue"], + "hashes.authenticodeHash256": ["fileStates.fileHash.hashValue"], + "hashes.lsHash": ["fileStates.fileHash.hashValue"], + "hashes.ctph": ["fileStates.fileHash.hashValue"], + "hashes.peSha1": ["fileStates.fileHash.hashValue"], + "hashes.peSha256": ["fileStates.fileHash.hashValue"], + "hashes.unknown": ["fileStates.fileHash.hashValue"] + } + }, + "process" : { + "fields": { + "x_name": ["processes.name", "processes.parentProcessName"], + "parent_ref.name": ["processes.parentProcessName"], + "command_line": ["processes.commandLine"], + "pid": ["processes.processId", "processes.parentProcessId", "registryKeyStates.processId"], + "created_time": ["processes.createdDateTime"], + "parent_ref.pid": ["processes.parentProcessId"], + "image_ref.parent_directory_ref.path": ["processes.path"], + "x_integrityLevel": ["processes.integrityLevel"], + "x_isElevated": ["processes.isElevated"] + } + }, + "domain-name": { + "fields": { + "value": ["hostStates.fqdn", "hostStates.netBiosName", "networkConnections.destinationDomain", "userStates.domainName"] + } + }, + "user-account": { + "fields": { + "user_id": ["userStates.accountName", "processes.accountName", "userStates.aadUserId"], + "account_login": ["userStates.logonId"], + "account_type": ["userStates.userAccountType"], + "account_last_login": ["userStates.logonDateTime"], + "x_aadUserId": ["userStates.aadUserId"], + "x_emailRole": ["userStates.emailRole"], + "x_isVpn": ["userStates.isVpn"], + "x_logonLocation": ["userStates.logonLocation"], + "x_logonType": ["userStates.logonType"], + "x_onPremisesSecurityIdentifier": ["userStates.onPremisesSecurityIdentifier"], + "x_riskScore": ["userStates.riskScore"], + "x_userAccountType": ["userStates.userAccountType"], + "x_userPrincipalName": ["userStates.userPrincipalName"] + } + }, + "software": { + "fields": { + "name": ["vendorInformation.provider", "networkConnections.applicationName"], + "vendor": ["vendorInformation.vendor"], + "version": ["vendorInformation.providerVersion"] + } + }, + "url": { + "fields": { + "value": ["networkConnections.destinationUrl"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["registryKeyStates.key"], + "values[*].data": ["registryKeyStates.valueData"], + "values[*].name": ["registryKeyStates.valueName"], + "values[*].data_type": ["registryKeyStates.valueType"] + } + }, + "x-msazure-sentinel": { + "fields": { + "tenant_id": ["azureTenantId"], + "subscription_id": ["azureSubscriptionId"] + } + }, + "x-ibm-finding": { + "fields": { + "name": ["title"], + "alert_id": ["id"], + "description": ["description"], + "severity": ["severity"], + "start": ["createdDateTime"], + "end": ["closedDateTime"], + "finding_type": ["category"], + "src_ip_ref.value": ["networkConnections.natSourceAddress"], + "dst_ip_ref.value": ["networkConnections.natDestinationAddress"], + "src_os_ref.name": ["hostStates.os"], + "dst_application_ref.name": ["cloudAppStates.destinationServiceName"], + "src_geolocation": ["networkConnections.sourceLocation"], + "dst_geolocation": ["networkConnections.destinationLocation"], + "src_application_ref": ["networkConnections.applicationName"], + "src_application_user_ref.user_id":["userStates.aadUserId"], + "src_application_user_ref.type":["userStates.logonType"], + "time_observed": ["lastModifiedDateTime"], + "x_activityGroupName": ["activityGroupName"], + "x_assignedTo": ["assignedTo"], + "x_comments": ["comments"], + "confidence": ["confidence"], + "x_detectionIds": ["detectionIds"], + "x_feedback": ["feedback"], + "x_incidentIds": ["incidentIds"], + "x_recommendedActions": ["recommendedActions"], + "x_sourceMaterials": ["sourceMaterials"], + "x_status": ["status"], + "x_tags": ["tags"], + "x_cloudAppStates.destinationServiceName": ["cloudAppStates.destinationServiceName"], + "x_cloudAppStates.destinationServiceIp": ["cloudAppStates.destinationServiceIp"], + "x_cloudAppStates.riskScore": ["cloudAppStates.riskScore"], + "x_hostStates.isAzureAadJoined": ["hostStates.isAzureAadJoined"], + "x_hostStates.isAzureAadRegistered": ["hostStates.isAzureAadRegistered"], + "x_hostStates.isHybridAzureDomainJoined": ["hostStates.isHybridAzureDomainJoined"], + "x_hostStates.os": ["hostStates.os"], + "x_hostStates.publicIpAddress": ["hostStates.publicIpAddress"], + "x_hostStates.privateIpAddress": ["hostStates.privateIpAddress"], + "x_hostStates.riskScore": ["hostStates.riskScore"], + "x_malwareStates.category": ["malwareStates.category"], + "x_malwareStates.family": ["malwareStates.family"], + "x_malwareStates.name": ["malwareStates.family"], + "x_malwareStates.severity": ["malwareStates.family"], + "x_malwareStates.wasRunning": ["malwareStates.family"], + "x_securityResources.resource": ["securityResources.resource"], + "x_securityResources.resourceType": ["securityResources.resourceType"], + "x_triggers.name": ["triggers.name"], + "x_triggers.type": ["triggers.type"], + "x_triggers.value": ["triggers.value"], + "x_vulnerabilityStates.cve": ["vulnerabilityStates.cve"], + "x_vulnerabilityStates.severity": ["vulnerabilityStates.severity"], + "x_vulnerabilityStates.wasRunning": ["vulnerabilityStates.wasRunning"] + } + }, + "x-oca-event": { + "fields": { + "action": ["title"], + "category": ["category"], + "created": ["createdDateTime"], + "provider": ["vendorInformation.subProvider"] + } + } +} diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/from_stix_map.json deleted file mode 100644 index 3e992672c..000000000 --- a/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/from_stix_map.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "ipv4-addr": { - "fields": { - "value": ["networkConnections.sourceAddress", "networkConnections.destinationAddress", "hostStates.publicIpAddress", "hostStates.privateIpAddress", "userStates.logonIp"] - } - }, - "ipv6-addr": { - "fields": { - "value": ["networkConnections.sourceAddress", "networkConnections.destinationAddress"] - } - }, - "network-traffic": { - "fields": { - "src_port": ["networkConnections.sourcePort"], - "dst_port": ["networkConnections.destinationPort"], - "protocols[*]": ["networkConnections.protocol"], - "src_ref.value": ["networkConnections.sourceAddress"], - "dst_ref.value": ["networkConnections.destinationAddress"] - } - }, - "directory": { - "fields": { - "path": ["fileStates.path", "process.path"] - } - }, - "file": { - "fields": { - "parent_directory_ref.path": ["fileStates.path"], - "name": ["fileStates.name"], - "hashes.'SHA-256'": ["fileStates.fileHash.hashValue"], - "hashes.'SHA-1'": ["fileStates.fileHash.hashValue"], - "hashes.MD5": ["fileStates.fileHash.hashValue"], - "hashes.authenticodeHash256": ["fileStates.fileHash.hashValue"], - "hashes.lsHash": ["fileStates.fileHash.hashValue"], - "hashes.ctph": ["fileStates.fileHash.hashValue"], - "hashes.peSha1": ["fileStates.fileHash.hashValue"], - "hashes.peSha256": ["fileStates.fileHash.hashValue"], - "hashes.unknown": ["fileStates.fileHash.hashValue"] - } - }, - "process" : { - "fields": { - "name": ["processes.name", "processes.parentProcessName"], - "parent_ref.name": ["processes.parentProcessName"], - "command_line": ["processes.commandLine"], - "pid": ["processes.processId", "processes.parentProcessId", "registryKeyStates.processId"], - "created_time": ["processes.createdDateTime"], - "parent_ref.pid": ["processes.parentProcessId"], - "image_ref.path": ["processes.path"] - } - }, - "domain-name": { - "fields": { - "value": ["hostStates.fqdn", "hostStates.netBiosName", "networkConnections.destinationDomain", "userStates.domainName"] - } - }, - "user-account": { - "fields": { - "user_id": ["userStates.accountName", "processes.accountName"], - "account_login": ["userStates.logonId"], - "account_type": ["userStates.userAccountType"], - "account_last_login": ["userStates.logonDateTime"] - } - }, - "software": { - "fields": { - "name": ["vendorInformation.provider", "networkConnections.applicationName"], - "vendor": ["vendorInformation.vendor"], - "version": ["vendorInformation.providerVersion"] - } - }, - "url": { - "fields": { - "value": ["networkConnections.destinationUrl"] - } - }, - "windows-registry-key": { - "fields": { - "key": ["registryKeyStates.key"], - "extensions.windows-registry-value-type.valueData": [ "registryKeyStates.valueData" ], - "extensions.windows-registry-value-type.name": [ "registryKeyStates.valueName" ], - "extensions.windows-registry-value-type.valueType": [ "registryKeyStates.valueType" ] - } - }, - "x-msazure-sentinel": { - "fields": { - "tenant_id": ["azureTenantId"], - "subscription_id": ["azureSubscriptionId"] - } - }, - "x-msazure-sentinel-alert": { - "fields": { - "activityGroupName": ["activityGroupName"], - "assignedTo": ["assignedTo"], - "category": ["category"], - "closedDateTime": ["closedDateTime"], - "cloudAppStates.destinationServiceName": ["cloudAppStates.destinationServiceName"], - "cloudAppStates.destinationServiceIp": ["cloudAppStates.destinationServiceIp"], - "cloudAppStates.riskScore": ["cloudAppStates.riskScore"], - "comments": ["comments"], - "confidence": ["confidence"], - "createdDateTime": ["createdDateTime"], - "description": ["description"], - "detectionIds": ["detectionIds"], - "eventDateTime": ["eventDateTime"], - "feedback": ["feedback"], - "hostStates.isAzureAadJoined": ["hostStates.isAzureAadJoined"], - "hostStates.isAzureAadRegistered": ["hostStates.isAzureAadRegistered"], - "hostStates.isHybridAzureDomainJoined": ["hostStates.isHybridAzureDomainJoined"], - "hostStates.os": ["hostStates.os"], - "hostStates.privateIpAddress": ["hostStates.privateIpAddress"], - "hostStates.riskScore": ["hostStates.riskScore"], - "alert_id": ["id"], - "incidentIds": ["incidentIds"], - "lastModifiedDateTime": ["lastModifiedDateTime"], - "malwareStates.category": ["malwareStates.category"], - "malwareStates.family": ["malwareStates.family"], - "malwareStates.name": ["malwareStates.family"], - "malwareStates.severity": ["malwareStates.family"], - "malwareStates.wasRunning": ["malwareStates.family"], - "networkConnections.destinationLocation": ["networkConnections.destinationLocation"], - "networkConnections.direction": ["networkConnections.direction"], - "networkConnections.domainRegisteredDateTime": ["networkConnections.domainRegisteredDateTime"], - "networkConnections.localDnsName": ["networkConnections.localDnsName"], - "networkConnections.natDestinationAddress": ["networkConnections.natDestinationAddress"], - "networkConnections.natDestinationPort": ["networkConnections.natDestinationPort"], - "networkConnections.natSourceAddress": ["networkConnections.natSourceAddress"], - "networkConnections.natSourcePort": ["networkConnections.natSourcePort"], - "networkConnections.riskScore": ["networkConnections.riskScore"], - "networkConnections.sourceLocation": ["networkConnections.sourceLocation"], - "networkConnections.status": ["networkConnections.status"], - "networkConnections.urlParameters": ["networkConnections.urlParameters"], - "processes.integrityLevel": ["processes.integrityLevel"], - "processes.isElevated": ["processes.isElevated"], - "recommendedActions": ["recommendedActions"], - "securityResources.resource": ["securityResources.resource"], - "securityResources.resourceType": ["securityResources.resourceType"], - "severity": ["severity"], - "sourceMaterials": ["sourceMaterials"], - "status": ["status"], - "tags": ["tags"], - "title": ["title"], - "triggers.name": ["triggers.name"], - "triggers.type": ["triggers.type"], - "triggers.value": ["triggers.value"], - "userStates.aadUserId": ["userStates.aadUserId"], - "userStates.emailRole": ["userStates.emailRole"], - "userStates.isVpn": ["userStates.isVpn"], - "userStates.logonLocation": ["userStates.logonLocation"], - "userStates.logonType": ["userStates.logonType"], - "userStates.onPremisesSecurityIdentifier": ["userStates.onPremisesSecurityIdentifier"], - "userStates.riskScore": ["userStates.riskScore"], - "userStates.userAccountType": ["userStates.userAccountType"], - "userStates.userPrincipalName": ["userStates.userPrincipalName"], - "vendorInformation.subProvider": ["vendorInformation.subProvider"], - "vulnerabilityStates.cve": ["vulnerabilityStates.cve"], - "vulnerabilityStates.severity": ["vulnerabilityStates.severity"], - "vulnerabilityStates.wasRunning": ["vulnerabilityStates.wasRunning"] - } - } -} diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/to_stix_map.json index 9be30c417..4195defe0 100644 --- a/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/to_stix_map.json @@ -1,595 +1,1356 @@ { - "eventDateTime": [ + "eventDateTime": [ + { + "key": "first_observed", + "cybox": false + } + ], + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "azureTenantId": { + "key": "x-microsoft-graph.tenant_id", + "object": "graph" + }, + "azureSubscriptionId": { + "key": "x-microsoft-graph.subscription_id", + "object": "graph" + }, + "activityGroupName": { + "key": "x-ibm-finding.x_activityGroupName", + "object": "finding" + }, + "assignedTo": { + "key": "x-ibm-finding.x_assignedTo", + "object": "finding" + }, + "category": { + "key": "x-oca-event.category", + "object": "event" + } + , + "closedDateTime":{ + "key": "last_observed", + "cybox": false + }, + "cloudAppStates": { + "destinationServiceName": [ { - "key": "first_observed", - "cybox": false + "key":"software.name", + "object":"software" }, { - "key": "last_observed", - "cybox": false + "key":"x-ibm-finding.dst_application_ref", + "object":"finding", + "references":"software" } ], - "event_count": { - "key": "number_observed", - "cybox": false, - "transformer": "ToInteger" + "destinationServiceIp": { + "key": "x-ibm-finding.x_cloudAppStates.destinationServiceIp", + "object": "finding" }, - "azureTenantId": { - "key": "x-msazure-sentinel.tenant_id", - "object": "sentinel" + "riskScore": { + "key": "x-ibm-finding.x_cloudAppStates.riskScore", + "object": "finding" + } + }, + "comments": { + "key": "x-ibm-finding.x_comments", + "object": "finding" + }, + "confidence": { + "key": "x-ibm-finding.confidence", + "object": "finding" + }, + "createdDateTime": [ + { + "key": "created", + "cybox": false }, - "azureSubscriptionId": { - "key": "x-msazure-sentinel.subscription_id", - "object": "sentinel" + { + "key": "x-oca-event.created", + "object": "event" + } + ], + "description": { + "key": "x-ibm-finding.description", + "object": "finding" + }, + "detectionIds": { + "key": "x-ibm-finding.x_detectionids", + "object": "finding" + }, + "feedback": { + "key": "x-ibm-finding.x_feedback", + "object": "finding", + "transformer": "ToString" + }, + "fileStates": { + "fileHash": { + "sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "md5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "authenticodeHash256": { + "key": "file.hashes.authenticodeHash256", + "object": "file" + }, + "lsHash": { + "key": "file.hashes.lsHash", + "object": "file" + }, + "ctph": { + "key": "file.hashes.ctph", + "object": "file" + }, + "peSha1": { + "key": "file.hashes.peSha1", + "object": "file" + }, + "peSha256": { + "key": "file.hashes.peSha256", + "object": "file" + }, + "unknown": { + "key": "file.hashes.UNKNOWN", + "object": "file" + } }, - "activityGroupName": { - "key": "x-msazure-sentinel-alert.activityGroupName", - "object": "alert" + "name": { + "key": "file.name", + "object": "file" }, - "assignedTo": { - "key": "x-msazure-sentinel-alert.assignedTo", - "object": "alert" + "path": [ + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "riskScore": { + "key": "x-ibm-finding.x_fileStates.riskScore", + "object": "file" + } + }, + "hostStates": { + "fqdn": { + "key": "domain-name.value", + "object": "host" }, - "category": { - "key": "x-msazure-sentinel-alert.category", - "object": "alert" + "isAzureAadJoined": { + "key": "x-ibm-finding.x_hostStates.isAzureAadJoined", + "object": "finding" }, - "closedDateTime": { - "key": "x-msazure-sentinel-alert.closedDateTime", - "object": "alert" + "isAzureAadRegistered": { + "key": "x-ibm-finding.x_hostStates.isAzureAadRegistered", + "object": "finding" }, - "cloudAppStates": { - "destinationServiceName": { - "key": "x-msazure-sentinel-alert.cloudAppStates.destinationServiceName", - "object": "alert" - }, - "destinationServiceIp": { - "key": "x-msazure-sentinel-alert.cloudAppStates.destinationServiceIp", - "object": "alert" + "isHybridAzureDomainJoined": { + "key": "x-ibm-finding.x_hostStates.isHybridAzureDomainJoined", + "object": "finding" + }, + "os": [ + { + "key": "x-ibm-finding.src_os_ref", + "object": "finding", + "references": "application" }, - "riskScore": { - "key": "x-msazure-sentinel-alert.cloudAppStates.riskScore", - "object": "alert" + { + "key": "software.name", + "object": "application" } + ], + "privateIpAddress": { + "key": "ipv4-addr.value" }, - "comments": { - "key": "x-msazure-sentinel-alert.comments", - "object": "alert", - "transformer": "ToString" + "publicIpAddress": { + "key": "ipv4-addr.value" + }, + "riskScore": { + "key": "x-ibm-finding.x_hostStates.riskScore", + "object": "finding" + } + }, + "id": { + "key": "x-ibm-finding.alert_id", + "object": "finding" + }, + "incidentIds": { + "key": "x-ibm-finding.x_incidentIds", + "object": "finding" + }, + "lastModifiedDateTime": [ + { + "key": "modified", + "cybox": false + }, + { + "key": "x-ibm-finding.time_observed", + "object": "finding" + } + ], + "malwareStates": { + "category": { + "key": "x-ibm-finding.x_malwareStates.category", + "object": "finding" + }, + "family": { + "key": "x-ibm-finding.x_malwareStates.family", + "object": "finding" + }, + "name": { + "key": "x-ibm-finding.x_malwareStates.name", + "object": "finding" }, - "confidence": { - "key": "x-msazure-sentinel-alert.confidence", - "object": "alert" + "severity": { + "key": "x-ibm-finding.x_malwareStates.severity", + "object": "finding" }, - "createdDateTime": [ + "wasRunning": { + "key": "x-ibm-finding.x_malwareStates.wasRunning", + "object": "finding" + } + }, + "networkConnections": { + "applicationName": [ { - "key": "created", - "cybox": false + "key": "software.name", + "object": "application" }, { - "key": "x-msazure-sentinel-alert.createddatetime", - "object": "alert" + "key": "x-ibm-finding.src_application_ref", + "object": "finding", + "references": "application" } ], - "description": { - "key": "x-msazure-sentinel-alert.description", - "object": "alert" + "destinationAddress": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "destinationLocation": { + "key": "x-ibm-finding.dst_geolocation", + "object": "finding" }, - "detectionIds": { - "key": "x-msazure-sentinel-alert.detectionids", - "object": "alert", - "transformer": "ToString" + "destinationDomain": { + "key": "domain-name.value", + "object": "nt_domain_name", + "transformer": "ToDomainName" }, - "feedback": { - "key": "x-msazure-sentinel-alert.feedback", - "object": "alert", - "transformer": "ToString" + "destinationPort": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" }, - "fileStates": { - "fileHash": { - "sha256": { - "key": "file.hashes.SHA-256", - "object": "file" - }, - "sha1": { - "key": "file.hashes.SHA-1", - "object": "file" - }, - "md5": { - "key": "file.hashes.MD5", - "object": "file" - }, - "authenticodeHash256": { - "key": "file.hashes.authenticodeHash256", - "object": "file" - }, - "lsHash": { - "key": "file.hashes.lsHash", - "object": "file" - }, - "ctph": { - "key": "file.hashes.ctph", - "object": "file" - }, - "peSha1": { - "key": "file.hashes.peSha1", - "object": "file" - }, - "peSha256": { - "key": "file.hashes.peSha256", - "object": "file" - }, - "unknown": { - "key": "file.hashes.UNKNOWN", - "object": "file" - } - }, - "name": { - "key": "file.name", - "object": "file" + "destinationUrl": { + "key": "url.value", + "object": "url" + }, + "direction": { + "key": "network-traffic.x_direction", + "object": "nt" + }, + "domainRegisteredDateTime": { + "key": "network-traffic.x_domainRegisteredDateTime", + "object": "nt" + }, + "localDnsName": { + "key": "network-traffic.x_localDnsName", + "object": "nt" + }, + "natDestinationAddress": [ + { + "key": "ipv4-addr.value", + "object": "nat_dst_ip" }, - "path": [ - { - "key": "directory.path", - "object": "directory", - "transformer": "ToDirectoryPath" - }, - { - "key": "file.parent_directory_ref", - "object": "file", - "references": "directory" - } - ], - "riskScore": { - "key": "x-msazure-sentinel-alert.fileStates.riskScore", - "object": "file" + { + "key": "network-traffic.x_nat_destination_address", + "object": "nt", + "references": "nat_dst_ip" } + ], + "natDestinationPort": { + "key": "network-traffic.x_nat_destination_port", + "object": "nt", + "transformer": "ToInteger" }, - "hostStates": { - "fqdn": { - "key": "domain-name.value", - "object": "host" - }, - "isAzureAadJoined": { - "key": "x-msazure-sentinel-alert.hostStates.isAzureAadJoined", - "object": "alert" - }, - "isAzureAadRegistered": { - "key": "x-msazure-sentinel-alert.hostStates.isAzureAadRegistered", - "object": "alert" - }, - "isHybridAzureDomainJoined": { - "key": "x-msazure-sentinel-alert.hostStates.isHybridAzureDomainJoined", - "object": "alert" - }, - "os": { - "key": "x-msazure-sentinel-alert.hostStates.os", - "object": "alert" - }, - "privateIpAddress": { - "key": "ipv4-addr.value" + "natSourceAddress": [ + { + "key": "ipv4-addr.value", + "object": "nat_src_ip" }, - "publicIpAddress": { - "key": "ipv4-addr.value" + { + "key": "network-traffic.x_nat_src_ref", + "object": "nt", + "references": "nat_src_ip" + } + ], + "natSourcePort": { + "key": "network-traffic.x_nat_source_port", + "object": "nt", + "transformer": "ToInteger" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "group": true, + "transformer": "ToLowercaseArray" + }, + "riskScore": { + "key": "network-traffic.x_riskScore", + "object": "nt" + }, + "sourceAddress": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" }, - "riskScore": { - "key": "x-msazure-sentinel-alert.hostStates.riskScore", - "object": "alert" + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" } + ], + "sourceLocation": { + "key": "x-ibm-finding.src_geolocation", + "object": "finding" }, - "id": { - "key": "x-msazure-sentinel-alert.providerid", - "object": "alert" + "sourcePort": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" }, - "incidentIds": { - "key": "x-msazure-sentinel-alert.incidentIds", - "object": "alert", - "transformer": "ToString" + "status": { + "key": "network-traffic.x_status", + "object": "nt" }, - "lastModifiedDateTime": [ + "urlParameters": { + "key": "network-traffic.x_url_parameters", + "object": "nt" + } + }, + "processes": { + "accountName": [ { - "key": "modified", - "cybox": false + "key": "user-account.user_id", + "object": "user" }, { - "key": "x-msazure-sentinel-alert.lastmodifieddatetime", - "object": "alert" + "key": "process.creator_user_ref", + "object": "process", + "references": "user" } ], - "malwareStates": { - "category": { - "key": "x-msazure-sentinel-alert.malwareStates.category", - "object": "alert" - }, - "family": { - "key": "x-msazure-sentinel-alert.malwareStates.family", - "object": "alert" - }, - "name": { - "key": "x-msazure-sentinel-alert.malwareStates.name", - "object": "alert" - }, - "severity": { - "key": "x-msazure-sentinel-alert.malwareStates.severity", - "object": "alert" - }, - "wasRunning": { - "key": "x-msazure-sentinel-alert.malwareStates.wasRunning", - "object": "alert" + "commandLine": { + "key": "process.command_line", + "object": "process" + }, + "createdDateTime": { + "key": "process.created_time", + "object": "process" + }, + "fileHash": { + "sha256": { + "key": "file.hashes.SHA-256", + "object": "processType" + }, + "sha1": { + "key": "file.hashes.SHA-1", + "object": "processType" + }, + "md5": { + "key": "file.hashes.MD5", + "object": "processType" + }, + "authenticodeHash256": { + "key": "file.hashes.authenticodeHash256", + "object": "processType" + }, + "lsHash": { + "key": "file.hashes.lsHash", + "object": "processType" + }, + "ctph": { + "key": "file.hashes.ctph", + "object": "processType" + }, + "peSha1": { + "key": "file.hashes.peSha1", + "object": "processType" + }, + "peSha256": { + "key": "file.hashes.peSha256", + "object": "processType" + }, + "unknown": { + "key": "file.hashes.UNKNOWN", + "object": "processType" } }, - "networkConnections": { - "applicationName": { - "key": "software.name" + "integrityLevel": { + "key": "processes.x_integrityLevel", + "object": "process" + }, + "isElevated": { + "key": "processes.x_isElevated", + "object": "process" + }, + "name": { + "key": "process.x_name", + "object": "process" + }, + "parentProcessCreatedDateTime": { + "key": "process.created_time", + "object": "process_parent" + }, + "parentProcessId": [ + { + "key": "process.pid", + "object": "parent_process" }, - "destinationAddress": [ - { - "key": "ipv4-addr.value", - "object": "dst_ip" - }, - { - "key": "network-traffic.dst_ref", - "object": "nt", - "references": "dst_ip" - } + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "parentProcessName": { + "key": "process.x_name", + "object": "process_parent" + }, + "path": { + "key": "directory.path", + "object": "dir" + }, + "processId": { + "key": "process.pid", + "object": "process" + } + }, + "recommendedActions": { + "key": "x-ibm-finding.x_recommendedactions", + "object": "finding" + }, + "registryKeyStates": { + "hive": { + "key": "x-ibm-finding.x_registryKeyStates.hive", + "object": "finding", + "transformer": "ToString" + }, + "key": { + "key": "windows-registry-key.key", + "object": "registry" + }, + "oldKey": { + "key": "x-ibm-finding.x_registryKeyStates.oldKey", + "object": "finding" + }, + "oldValueData": { + "key": "x-ibm-finding.x_registryKeyStates.oldValueData", + "object": "finding" + }, + "oldValueName": { + "key": "x-ibm-finding.x_registryKeyStates.oldValueName", + "object": "finding" + }, + "operation": { + "key": "x-ibm-finding.x_registryKeyStates.operation", + "object": "finding", + "transformer": "ToString" + }, + "processId": { + "key": "process.pid" + }, + "valueData": { + "key": "windows-registry-key.values.data", + "object": "registry", + "group": "windows-registry-value-type" + }, + "valueName": { + "key": "windows-registry-key.values.name", + "object": "registry", + "group": "windows-registry-value-type" + }, + "valueType": { + "key": "windows-registry-key.values.data_type", + "object": "registry", + "transformer": "ToString", + "group": "windows-registry-value-type" + } + }, + "securityResources": { + "resource": { + "key": "x-ibm-finding.x_securityresources.resource", + "object": "finding" + }, + "resourceType": { + "key": "x-ibm-finding.x_securityresources.resourcetype", + "object": "finding", + "transformer": "ToString" + } + }, + "severity": { + "key": "x-ibm-finding.severity", + "object": "finding" + }, + "sourceMaterials": { + "key": "x-ibm-finding.x_sourcematerials", + "object": "finding" + }, + "status": { + "key": "x-ibm-finding.x_status", + "object": "finding", + "transformer": "ToString" + }, + "tags": { + "key": "x-ibm-finding.x_tags", + "object": "finding" + }, + "title": [ + { + "key": "x-ibm-finding.name", + "object": "finding" + }, + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "triggers": { + "name": { + "key": "x-ibm-finding.x_triggers.name", + "object": "finding" + }, + "type": { + "key": "x-ibm-finding.x_triggers.type", + "object": "finding" + }, + "value": { + "key": "x-ibm-finding.x_triggers.value", + "object": "finding" + } + }, + "userStates": { + "aadUserId": { + "key": "user-account.x_aad_user_id", + "object": "user" + }, + "accountName": { + "key": "user-account.user_id", + "object": "user" + }, + "domainName": { + "key": "domain-name.value", + "object": "domain-name" + }, + "emailRole": { + "key": "user-account.x_email_role", + "object": "user" + }, + "isVpn": { + "key": "user-account.x_isvpn", + "object": "user" + }, + "logonDateTime": { + "key": "user-account.account_last_login", + "object": "user" + }, + "logonId": { + "key": "user-account.account_login", + "object": "user" + }, + "logonIp": { + "key": "ipv4-addr.value" + }, + "logonLocation": { + "key": "user-account.x_logon_location", + "object": "user" + }, + "logonType": { + "key": "user-account.x_logon_type", + "object": "user" + }, + "onPremisesSecurityIdentifier": { + "key": "user-account.x_on_premises_security_identifier", + "object": "user" + }, + "riskScore": { + "key": "user-account.x_riskScore", + "object": "user" + }, + "userAccountType": { + "key": "user-account.x_user_account_type", + "object": "user" + }, + "userPrincipalName": { + "key": "user-account.x_user_principal_name", + "object": "user" + } + }, + "vendorInformation": { + "provider": { + "key": "software.name", + "object": "application" + }, + "vendor": { + "key": "software.vendor", + "object": "application" + }, + "providerVersion": { + "key": "software.version", + "object": "application" + }, + "subProvider": { + "key": "x-oca-event.provider", + "object": "event" + } + }, + "vulnerabilityStates": { + "cve": { + "key": "x-ibm-finding.x_vulnerabilityStates.cve", + "object": "finding" + }, + "severity": { + "key": "x-ibm-finding.x_vulnerabilityStates.severity", + "object": "finding" + }, + "wasRunning": { + "key": "x-ibm-finding.x_vulnerabilityStates.wasRunning", + "object": "finding" + } + }, + "@odata.type": { + "key": "x-ibm-finding.finding_type", + "object": "finding" + }, + "providerAlertId": { + "key": "x-ibm-finding.alert_id", + "object": "finding" + }, + "incidentId": { + "key": "x-ibm-finding.x_incidentId", + "object": "finding" + }, + "classification": { + "key": "x-ibm-finding.x_classification", + "object": "finding" + }, + "determination": { + "key": "x-ibm-finding.x_determination", + "object": "finding" + }, + "serviceSource": { + "key": "software.name", + "object": "service_software" + }, + "detectionSource": { + "key": "software.name", + "object": "detection_software" + }, + "detectorId": { + "key": "x-ibm-finding.x_detectorId", + "object": "finding" + }, + "tenantId": { + "key": "x-ibm-finding.x_tenantId", + "object": "finding" + }, + "alertWebUrl": { + "key": "url.value", + "object": "alert_url" + }, + "incidentWebUrl": { + "key": "url.value", + "object": "incident_url" + }, + "actorDisplayName": { + "key": "user-account.user_id", + "object": "actor" + }, + "threatDisplayName": { + "key": "x-ibm-finding.x_threatDisplayName", + "object": "finding" + }, + "threatFamilyName": { + "key": "x-ibm-finding.x_threatFamilyName", + "object": "finding" + }, + "mitreTechniques": { + "key": "x-ibm-finding.x_mitreTechniques", + "object": "finding" + }, + "lastUpdateDateTime": { + "key": "x-ibm-finding.x_lastUpdateDateTime", + "object": "finding" + }, + "resolvedDateTime": { + "key": "x-ibm-finding.end", + "object": "finding" + }, + "firstActivityDateTime": { + "key": "x-ibm-finding.start", + "object": "finding" + }, + "lastActivityDateTime": { + "key": "x-ibm-finding.x_lastActivityDateTime", + "object": "finding" + }, + "deviceEvidence": { + "@odata.type": [ + { + "key": "x-alert-evidence.evidence_type", + "object": "device-evidence" + }, + { + "key": "x-alert-evidence.process_ref", + "object": "device-evidence", + "references": "x-oca-asset" + } ], - "destinationLocation": { - "key": "x-msazure-sentinel-alert.networkConnections.destinationLocation", - "object": "alert" + "createdDateTime": { + "key": "x-alert-evidence.created", + "object": "device-evidence" + }, + "verdict": { + "key": "x-alert-evidence.verdict", + "object": "device-evidence" + }, + "remediationStatus": { + "key": "x-alert-evidence.remediationStatus", + "object": "device-evidence" + }, + "remediationStatusDetails": { + "key": "x-alert-evidence.remediationStatusDetails", + "object": "device-evidence" + }, + "roles": { + "key": "x-alert-evidence.roles", + "object": "device-evidence" + }, + "tags": { + "key": "x-alert-evidence.tags", + "object": "device-evidence" + }, + "firstSeenDateTime": { + "key": "x-oca-asset.x_firstSeenDateTime", + "object": "x-oca-asset" + }, + "mdeDeviceId": { + "key": "x-oca-asset.device_id", + "object": "x-oca-asset" + }, + "azureAdDeviceId": { + "key": "x-oca-asset.x_azureAdDeviceId", + "object": "x-oca-asset" + }, + "deviceDnsName": { + "key": "domain-name.value", + "object": "domain-name" + }, + "osPlatform": [ + { + "key": "software.name", + "object": "asset_software" + }, + { + "key": "x-oca-asset.os_ref", + "object": "x-oca-asset", + "references": "asset_software" + } + ], + "osBuild": { + "key": "x-oca-asset.x_tags", + "object": "x-oca-asset" }, - "destinationDomain": { - "key": "domain-name.value", - "transformer": "ToDomainName" + "version": { + "key": "software.version", + "object": "asset_software" }, - "destinationPort": { - "key": "network-traffic.dst_port", - "object": "nt", - "transformer": "ToInteger" + "healthStatus": { + "key": "x-oca-asset.x_tags", + "object": "x-oca-asset" + }, + "riskScore": { + "key": "x-oca-asset.x_tags", + "object": "x-oca-asset" }, - "destinationUrl": { - "key": "url.value", - "object": "url" + "rbacGroupId": { + "key": "x-oca-asset.x_tags", + "object": "x-oca-asset" }, - "direction": { - "key": "x-msazure-sentinel-alert.networkConnections.direction", - "object": "alert" + "rbacGroupName": { + "key": "x-oca-asset.x_tags", + "object": "x-oca-asset" }, - "domainRegisteredDateTime": { - "key": "x-msazure-sentinel-alert.networkConnections.domainRegisteredDateTime", - "object": "alert" + "onboardingStatus": { + "key": "x-oca-asset.x_tags", + "object": "x-oca-asset" }, - "localDnsName": { - "key": "x-msazure-sentinel-alert.networkConnections.localDnsName", - "object": "alert" + "defenderAvStatus": { + "key": "x-oca-asset.x_tags", + "object": "x-oca-asset" }, - "natDestinationAddress": { - "key": "x-msazure-sentinel-alert.networkConnections.natDestinationAddress", - "object": "alert" + "loggedOnUsers": { + "key": "x-oca-asset.x_tags", + "object": "x-oca-asset" }, - "natDestinationPort": { - "key": "x-msazure-sentinel-alert.networkConnections.natDestinationPort", - "object": "alert" + "vmMetadata": { + "key": "x-oca-asset.x_vmMetadata", + "object": "x-oca-asset" + } + }, + "fileEvidence": { + "@odata.type": [ + { + "key": "x-alert-evidence.evidence_type", + "object": "file-evidence" + }, + { + "key": "x-alert-evidence.process_ref", + "object": "file-evidence", + "references": "process" + } + ], + "createdDateTime": { + "key": "x-alert-evidence.created", + "object": "file-evidence" }, - "natSourceAddress": { - "key": "x-msazure-sentinel-alert.networkConnections.natSourceAddress", - "object": "alert" + "verdict": { + "key": "x-alert-evidence.verdict", + "object": "file-evidence" }, - "natSourcePort": { - "key": "x-msazure-sentinel-alert.networkConnections.natSourcePort", - "object": "alert" + "remediationStatus": { + "key": "x-alert-evidence.remediationStatus", + "object": "file-evidence" }, - "protocol": { - "key": "network-traffic.protocols", - "object": "nt", - "group": "True", - "transformer": "ToLowercaseArray" + "remediationStatusDetails": { + "key": "x-alert-evidence.remediationStatusDetails", + "object": "file-evidence" }, - "riskScore": { - "key": "x-msazure-sentinel-alert.networkConnections.riskScore", - "object": "alert" + "roles": { + "key": "x-alert-evidence.roles", + "object": "file-evidence" }, - "sourceAddress": [ - { - "key": "ipv4-addr.value", - "object": "src_ip" - }, - { - "key": "network-traffic.src_ref", - "object": "nt", - "references": "src_ip" - } - ], - "sourceLocation": { - "key": "x-msazure-sentinel-alert.networkConnections.sourceLocation", - "object": "alert" + "tags": { + "key": "x-alert-evidence.tags", + "object": "file-evidence" }, - "sourcePort": { - "key": "network-traffic.src_port", - "object": "nt", - "transformer": "ToInteger" + "detectionStatus": { + "key": "file.x_detectionStatus", + "object": "file" }, - "status": { - "key": "x-msazure-sentinel-alert.networkConnections.status", - "object": "alert" + "mdeDeviceId": { + "key": "file.x_mdeDeviceId", + "object": "file" }, - "urlParameters": { - "key": "x-msazure-sentinel-alert.networkConnections.urlParameters", - "object": "alert" + "fileDetails": { + "sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "fileName": { + "key": "file.name", + "object": "file" + }, + "filePath": [ + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "fileSize": { + "key": "file.size", + "object": "file" + }, + "filePublisher": { + "key": "file.x_filePublisher", + "object": "file" + }, + "signer": { + "key": "file.x_signer", + "object": "file" + }, + "issuer": { + "key": "file.x_issuer", + "object": "file" + } } - }, - "processes": { - "accountName": [ - { - "key": "user-account.user_id", - "object": "user" - }, - { - "key": "process.creator_user_ref", - "object": "process", - "references": "user" - } + }, + "processEvidence": { + "@odata.type": [ + { + "key": "x-alert-evidence.evidence_type", + "object": "process-evidence" + }, + { + "key": "x-alert-evidence.process_ref", + "object": "process-evidence", + "references": "process" + } ], - "commandLine": { - "key": "process.command_line", - "object": "process" - }, "createdDateTime": { - "key": "process.created_time", - "object": "process" - }, - "fileHash": { - "sha256": { - "key": "file.hashes.SHA-256", - "object": "processType" - }, - "sha1": { - "key": "file.hashes.SHA-1", - "object": "processType" - }, - "md5": { - "key": "file.hashes.MD5", - "object": "processType" - }, - "authenticodeHash256": { - "key": "file.hashes.authenticodeHash256", - "object": "processType" - }, - "lsHash": { - "key": "file.hashes.lsHash", - "object": "processType" - }, - "ctph": { - "key": "file.hashes.ctph", - "object": "processType" - }, - "peSha1": { - "key": "file.hashes.peSha1", - "object": "processType" - }, - "peSha256": { - "key": "file.hashes.peSha256", - "object": "processType" - }, - "unknown": { - "key": "file.hashes.UNKNOWN", - "object": "processType" - } - }, - "integrityLevel": { - "key": "x-msazure-sentinel-alert.processes.integrityLevel", - "object": "alert" - }, - "isElevated": { - "key": "x-msazure-sentinel-alert.processes.isElevated", - "object": "alert" - }, - "parentProcessCreatedDateTime": { - "key": "process.created_time", - "object": "process_parent" + "key": "x-alert-evidence.created", + "object": "process-evidence" }, - "parentProcessId": [ - { + "verdict": { + "key": "x-alert-evidence.verdict", + "object": "process-evidence" + }, + "remediationStatus": { + "key": "x-alert-evidence.remediationStatus", + "object": "process-evidence" + }, + "remediationStatusDetails": { + "key": "x-alert-evidence.remediationStatusDetails", + "object": "process-evidence" + }, + "roles": { + "key": "x-alert-evidence.roles", + "object": "process-evidence" + }, + "tags": { + "key": "x-alert-evidence.tags", + "object": "process-evidence" + }, + "processId": { "key": "process.pid", - "object": "parent_process" - }, - { - "key": "process.parent_ref", - "object": "process", - "references": "parent_process" - } + "object": "process" + }, + "parentProcessId": [ + { + "key": "process.pid", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } ], - "path": { - "key": "directory.path", - "object": "dir" + "processCommandLine": { + "key": "process.command_line", + "object": "process" }, - "processId": { - "key": "process.pid", - "object": "process" - } - }, - "recommendedActions": { - "key": "x-msazure-sentinel-alert.recommendedactions", - "object": "alert", - "transformer": "ToString" - }, - "registryKeyStates": { - "hive": { - "key": "x-msazure-sentinel-alert.registryKeyStates.hive", - "object": "alert", - "transformer": "ToString" + "processCreationDateTime": { + "key": "process.created_time", + "object": "process" }, - "key": { - "key": "windows-registry-key.key", - "object": "registry" + "parentProcessCreationDateTime": { + "key": "process.created_time", + "object": "parent_process" }, - "oldKey": { - "key": "x-msazure-sentinel-alert.registryKeyStates.oldKey", - "object": "alert" + "detectionStatus": { + "key": "process.x_detectionStatus", + "object": "process" + }, + "mdeDeviceId": { + "key": "process.x_mdeDeviceId", + "object": "process" + }, + "imageFile": { + "sha1": { + "key": "file.hashes.SHA-1", + "object": "process_file" + }, + "sha256": { + "key": "file.hashes.SHA-256", + "object": "process_file" + }, + "fileName": [ + { + "key": "file.name", + "object": "process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "filePath": [ + { + "key": "directory.path", + "object": "process_directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_directory" + } + ], + "fileSize": { + "key": "file.size", + "object": "process_file" + }, + "filePublisher": { + "key": "file.x_filePublisher", + "object": "process_file" + }, + "signer": { + "key": "file.x_signer", + "object": "process_file" + }, + "issuer": { + "key": "file.x_issuer", + "object": "process_file" + } + }, + "parentProcessImageFile": { + "sha1": { + "key": "file.hashes.SHA-1", + "object": "parent_process_file" + }, + "sha256": { + "key": "file.hashes.SHA-256", + "object": "parent_process_file" + }, + "fileName": [ + { + "key": "file.name", + "object": "parent_process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "parent_process_file" + } + ], + "filePath": [ + { + "key": "directory.path", + "object": "parent_process_directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "parent_process_file", + "references": "parent_process_directory" + } + ], + "fileSize": { + "key": "file.size", + "object": "parent_process_file" + }, + "filePublisher": { + "key": "file.x_filePublisher", + "object": "parent_process_file" + }, + "signer": { + "key": "file.x_signer", + "object": "parent_process_file" + }, + "issuer": { + "key": "file.x_issuer", + "object": "parent_process_file" + } + }, + "userAccount": { + "accountName": [ + { + "key": "user-account.user_id", + "object": "user-account" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user-account" + } + ], + "domainName": { + "key": "user-account.x_azure_domain_name", + "object": "user-account" + }, + "userSid": { + "key": "user-account.x_userSid", + "object": "user-account" + }, + "azureAdUserId": { + "key": "user-account.x_azureAdUserId", + "object": "user-account" + }, + "userPrincipalName": { + "key": "user-account.x_userPrincipalName", + "object": "user-account" + } + } + }, + "registryKeyEvidence": { + "@odata.type": [ + { + "key": "x-alert-evidence.evidence_type", + "object": "registry-evidence" + }, + { + "key": "x-alert-evidence.registry_ref", + "object": "registry-evidence", + "references": "registry" + } + ], + "createdDateTime": { + "key": "x-alert-evidence.created", + "object": "registry-evidence" }, - "oldValueData": { - "key": "x-msazure-sentinel-alert.registryKeyStates.oldValueData", - "object": "alert" + "verdict": { + "key": "x-alert-evidence.verdict", + "object": "registry-evidence" }, - "oldValueName": { - "key": "x-msazure-sentinel-alert.registryKeyStates.oldValueName", - "object": "alert" + "remediationStatus": { + "key": "x-alert-evidence.remediationStatus", + "object": "registry-evidence" }, - "operation": { - "key": "x-msazure-sentinel-alert.registryKeyStates.operation", - "object": "alert", - "transformer": "ToString" + "remediationStatusDetails": { + "key": "x-alert-evidence.remediationStatusDetails", + "object": "registry-evidence" }, - "processId": { - "key": "process.pid" + "roles": { + "key": "x-alert-evidence.roles", + "object": "registry-evidence" }, - "valueData": { - "key": "extensions.windows-registry-value-type.valueData", - "object": "registry" + "tags": { + "key": "x-alert-evidence.tags", + "object": "registry-evidence" }, - "valueName": { - "key": "extensions.windows-registry-value-type.name", - "object": "registry" + "registryKey": { + "key": "windows-registry-key.key", + "object": "registry" }, - "valueType": { - "key": "extensions.windows-registry-value-type.valuetype", - "object": "registry", - "transformer": "ToString" + "registryHive": { + "key": "windows-registry-key.x_registryHive", + "object": "registry" } - }, - "securityResources": { - "resource": { - "key": "x-msazure-sentinel-alert.securityresources.resource", - "object": "alert" + }, + "registryValueEvidence": { + "@odata.type": [ + { + "key": "x-alert-evidence.evidence_type", + "object": "registryValue-evidence" + }, + { + "key": "x-alert-evidence.registry_ref", + "object": "registryValue-evidence", + "references": "registryValue" + } + ], + "createdDateTime": { + "key": "x-alert-evidence.created", + "object": "registryValue-evidence" }, - "resourceType": { - "key": "x-msazure-sentinel-alert.securityresources.resourcetype", - "object": "alert", - "transformer": "ToString" - } - }, - "severity": { - "key": "x-msazure-sentinel-alert.severity", - "object": "alert" - }, - "sourceMaterials": { - "key": "x-msazure-sentinel-alert.sourcematerials", - "object": "alert", - "transformer": "ToString" - }, - "status": { - "key": "x-msazure-sentinel-alert.status", - "object": "alert", - "transformer": "ToString" - }, - "tags": { - "key": "x-msazure-sentinel-alert.tags", - "object": "alert", - "transformer": "ToString" - }, - "title": { - "key": "x-msazure-sentinel-alert.title", - "object": "alert" - }, - "triggers": { - "name": { - "key": "x-msazure-sentinel-alert.triggers.name", - "object": "alert" + "verdict": { + "key": "x-alert-evidence.verdict", + "object": "registryValue-evidence" }, - "type": { - "key": "x-msazure-sentinel-alert.triggers.type", - "object": "alert" + "remediationStatus": { + "key": "x-alert-evidence.remediationStatus", + "object": "registryValue-evidence" }, - "value": { - "key": "x-msazure-sentinel-alert.triggers.value", - "object": "alert" - } - }, - "userStates": { - "aadUserId": { - "key": "x-msazure-sentinel-alert.userStates.aaduserid", - "object": "alert" + "remediationStatusDetails": { + "key": "x-alert-evidence.remediationStatusDetails", + "object": "registryValue-evidence" }, - "accountName": { - "key": "user-account.user_id", - "object": "user" + "roles": { + "key": "x-alert-evidence.roles", + "object": "registryValue-evidence" }, - "domainName": { - "key": "domain-name.value" + "tags": { + "key": "x-alert-evidence.tags", + "object": "registryValue-evidence" }, - "emailRole": { - "key": "x-msazure-sentinel-alert.userStates.emailrole", - "object": "alert" + "registryKey": { + "key": "windows-registry-key.key", + "object": "registryValue" }, - "isVpn": { - "key": "x-msazure-sentinel-alert.userStates.isvpn", - "object": "alert" + "registryHive": { + "key": "windows-registry-key.x_registry_hive", + "object": "registryValue" }, - "logonDateTime": { - "key": "user-account.account_last_login", - "object": "user" + "registryValue": { + "key": "windows-registry-key.values.data", + "object": "registryValue" }, - "logonId": { - "key": "user-account.account_login", - "object": "user" + "registryValueName": { + "key": "windows-registry-key.values.name", + "object": "registryValue" }, - "logonIp": { - "key": "ipv4-addr.value" + "registryValueType": { + "key": "windows-registry-key.values.data_type", + "object": "registryValue" + } + }, + "ipEvidence": { + "@odata.type": [ + { + "key": "x-alert-evidence.evidence_type", + "object": "ip-evidence" + }, + { + "key": "x-alert-evidence.ip_ref", + "object": "ip-evidence", + "references": "ipv4-addr" + } + ], + "createdDateTime": { + "key": "x-alert-evidence.created", + "object": "ip-evidence" }, - "logonLocation": { - "key": "x-msazure-sentinel-alert.userStates.logonLocation", - "object": "alert" + "verdict": { + "key": "x-alert-evidence.verdict", + "object": "ip-evidence" }, - "logonType": { - "key": "x-msazure-sentinel-alert.userStates.logonType", - "object": "alert" + "remediationStatus": { + "key": "x-alert-evidence.remediationStatus", + "object": "ip-evidence" }, - "onPremisesSecurityIdentifier": { - "key": "x-msazure-sentinel-alert.userStates.onpremisessecurityidentifier", - "object": "alert" + "remediationStatusDetails": { + "key": "x-alert-evidence.remediationStatusDetails", + "object": "ip-evidence" }, - "riskScore": { - "key": "x-msazure-sentinel-alert.userStates.riskScore", - "object": "alert" + "ipAddress": { + "key": "ipv4-addr.value", + "object": "ipv4-addr" }, - "userAccountType": { - "key": "x-msazure-sentinel-alert.userStates.useraccounttype", - "object": "alert" + "countryLetterCode": { + "key": "ipv4-addr.x_country_letter_code", + "object": "ipv4-addr" }, - "userPrincipalName": { - "key": "x-msazure-sentinel-alert.userStates.userPrincipalName", - "object": "alert" + "roles": { + "key": "x-alert-evidence.roles", + "object": "ip-evidence" + }, + "tags": { + "key": "x-alert-evidence.tags", + "object": "ip-evidence" } - }, - "vendorInformation": { - "provider": { - "key": "software.name", - "object": "application" + }, + "userEvidence": { + "@odata.type": [ + { + "key": "x-alert-evidence.evidence_type", + "object": "user-evidence" + }, + { + "key": "x-alert-evidence.user_ref", + "object": "user-evidence", + "references": "user-account" + } + ], + "createdDateTime": { + "key": "x-alert-evidence.created", + "object": "user-evidence" + }, + "verdict": { + "key": "x-alert-evidence.verdict", + "object": "user-evidence" + }, + "remediationStatus": { + "key": "x-alert-evidence.remediationStatus", + "object": "user-evidence" + }, + "remediationStatusDetails": { + "key": "x-alert-evidence.remediationStatusDetails", + "object": "user-evidence" + }, + "roles": { + "key": "x-alert-evidence.roles", + "object": "user-evidence" + }, + "tags": { + "key": "x-alert-evidence.tags", + "object": "user-evidence" + }, + "userAccount": { + "accountName": { + "key": "user-account.user_id", + "object": "user-account" + }, + "domainName": { + "key": "user-account.x_azure_domain_name", + "object": "user-account" + }, + "userSid": { + "key": "user-account.x_user_sid", + "object": "user-account" + }, + "azureAdUserId": { + "key": "user-account.x_azure_ad_userid", + "object": "user-account" + }, + "userPrincipalName": { + "key": "user-account.x_userPrincipalName", + "object": "user-account" + } + } + }, + "urlEvidence": { + "@odata.type": [ + { + "key": "x-alert-evidence.evidence_type", + "object": "url-evidence" + }, + { + "key": "x-alert-evidence.user_ref", + "object": "url-evidence", + "references": "url" + } + ], + "createdDateTime": { + "key": "x-alert-evidence.created", + "object": "url-evidence" }, - "vendor": { - "key": "software.vendor", - "object": "application" + "verdict": { + "key": "x-alert-evidence.verdict", + "object": "url-evidence" }, - "providerVersion": { - "key": "software.version", - "object": "application" + "remediationStatus": { + "key": "x-alert-evidence.remediationStatus", + "object": "url-evidence" }, - "subProvider": { - "key": "x-msazure-sentinel-alert.vendorinformation.subprovider", - "object": "alert" - } - }, - "vulnerabilityStates": { - "cve": { - "key": "x-msazure-sentinel-alert.vulnerabilityStates.cve", - "object": "alert" + "remediationStatusDetails": { + "key": "x-alert-evidence.remediationStatusDetails", + "object": "url-evidence" + }, + "roles": { + "key": "x-alert-evidence.roles", + "object": "url-evidence" }, - "severity": { - "key": "x-msazure-sentinel-alert.vulnerabilityStates.severity", - "object": "alert" + "tags": { + "key": "x-alert-evidence.tags", + "object": "url-evidence" }, - "wasRunning": { - "key": "x-msazure-sentinel-alert.vulnerabilityStates.wasRunning", - "object": "alert" + "url": { + "key": "url.value", + "object": "url" } - } + } } \ No newline at end of file diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/to_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/to_stix_map.json index a6e9ec0bd..38171cead 100644 --- a/stix_shifter_modules/azure_sentinel/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/azure_sentinel/stix_translation/json/to_stix_map.json @@ -314,7 +314,7 @@ "protocol": { "key": "network-traffic.protocols", "object": "nt", - "group": "True", + "group": true, "transformer": "ToLowercaseArray" }, "riskScore": { @@ -483,17 +483,20 @@ "key": "process.pid" }, "valueData": { - "key": "extensions.windows-registry-value-type.valueData", - "object": "registry" + "key": "windows-registry-key.values.data", + "object": "registry", + "group": "windows-registry-value-type" }, "valueName": { - "key": "extensions.windows-registry-value-type.name", - "object": "registry" + "key": "windows-registry-key.values.name", + "object": "registry", + "group": "windows-registry-value-type" }, "valueType": { - "key": "extensions.windows-registry-value-type.valuetype", + "key": "windows-registry-key.values.data_type", "object": "registry", - "transformer": "ToString" + "transformer": "ToString", + "group": "windows-registry-value-type" } }, "securityResources": { diff --git a/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py b/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py index e790144d3..2877c7973 100644 --- a/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py +++ b/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py @@ -240,6 +240,27 @@ def test_unmapped_attribute_alone(): objects = observed_data['objects'] assert objects == {} + @staticmethod + def test_registry_key_state(): + """ + test windows-registry-key + """ + data = { "registryKeyStates": [ { "valueData": "Test Value Data", "valueName": "Test Value Name", "valueType": "Test Value Type" } ] } + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, [data], get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + observed_data = result_bundle_objects[1] + assert 'objects' in observed_data + objects = observed_data['objects'] + windows_reg_key_object = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'windows-registry-key') + assert windows_reg_key_object is not None, 'windows-registry-key object type not found' + windows_reg_key_values = windows_reg_key_object.get("values") + key_values_obj = windows_reg_key_values[0] + assert key_values_obj.get('data') == "Test Value Data" + assert key_values_obj.get('data_type') == "Test Value Type" + assert key_values_obj.get('name') == "Test Value Name" + + def test_alert_v2_tranlsation(self): entry_point = EntryPoint() result_file = open('stix_shifter_modules/azure_sentinel/tests/jsons/alertV2_respons.json', 'r').read() @@ -263,4 +284,4 @@ def test_alert_v2_tranlsation(self): assert observed_data['created'] is not None assert observed_data['first_observed'] is not None assert observed_data['last_observed'] is not None - assert observed_data['number_observed'] is not None \ No newline at end of file + assert observed_data['number_observed'] is not None diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json index 66e85495d..a02d8226f 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json @@ -619,7 +619,7 @@ { "key": "network-traffic.protocols", "object": "nt", - "group": "True", + "group": true, "transformer": "ToLowercaseArray" }, { @@ -639,7 +639,7 @@ { "key": "network-traffic.protocols", "object": "nt", - "group": "True", + "group": true, "transformer": "ToLowercaseArray" }, { @@ -659,7 +659,7 @@ { "key": "network-traffic.protocols", "object": "nt", - "group": "True", + "group": true, "transformer": "ToLowercaseArray" }, { diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 9893c349d..be9820393 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -615,7 +615,7 @@ { "key": "network-traffic.protocols", "object": "nt", - "group": "True", + "group": true, "transformer": "ToLowercaseArray" }, { @@ -635,7 +635,7 @@ { "key": "network-traffic.protocols", "object": "nt", - "group": "True", + "group": true, "transformer": "ToLowercaseArray" }, { @@ -655,7 +655,7 @@ { "key": "network-traffic.protocols", "object": "nt", - "group": "True", + "group": true, "transformer": "ToLowercaseArray" }, { diff --git a/stix_shifter_modules/error_test/stix_translation/query_translator.py b/stix_shifter_modules/error_test/stix_translation/query_translator.py index 46f6912ef..f94d21d8c 100644 --- a/stix_shifter_modules/error_test/stix_translation/query_translator.py +++ b/stix_shifter_modules/error_test/stix_translation/query_translator.py @@ -1,8 +1,9 @@ from stix_shifter_utils.modules.base.stix_translation.empty_query_translator import EmptyQueryTranslator import re from time import sleep +from datetime import datetime, timedelta -START_STOP_PATTERN = r"\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}.\d{1,3}Z'\s?" +START_STOP_PATTERN = r"\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d{1,3})?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d{1,3})?Z'\s?" ERROR_TYPE_PARSE_EXCEPTION = 'parse_exception' ERROR_TYPE_TRANSFORM_EXCEPTION = 'transform_exception' @@ -22,8 +23,15 @@ def transform_query(self, data): if error_type.startswith(ERROR_TYPE_TRANSFORM_DELAY_SEC): delay = int(error_type[len(ERROR_TYPE_TRANSFORM_DELAY_SEC):]) sleep(delay) + time_range = self.options['time_range'] # Passed from global config + # Data is a STIX pattern. - # stix2-matcher will break on START STOP qualifiers so remove before returning pattern. - # Remove this when ever stix2-matcher supports proper qualifier timestamps - data = re.sub(START_STOP_PATTERN, " ", data) + if not re.search(START_STOP_PATTERN, data): + # add START STOP qualifier for last x minutes if none present + now = datetime.now() + timerange_delta = timedelta(minutes=time_range) + some_minutes_ago = now - timerange_delta + start_time = some_minutes_ago.strftime("START t'%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z'" + stop_time = now.strftime("STOP t'%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z'" + data = data + " " + start_time + " " + stop_time return {'queries': [data]} diff --git a/stix_shifter_modules/error_test/stix_transmission/connector.py b/stix_shifter_modules/error_test/stix_transmission/connector.py index 30676dc90..1104545f2 100644 --- a/stix_shifter_modules/error_test/stix_transmission/connector.py +++ b/stix_shifter_modules/error_test/stix_transmission/connector.py @@ -1,14 +1,14 @@ +from aiohttp import BasicAuth import json import time -from aiohttp import BasicAuth +import re +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector +from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync from stix2matcher.matcher import Pattern from stix2matcher.matcher import MatchListener from stix2validator import validate_instance, ValidationOptions - -from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector -from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status -from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status ERROR_TYPE_TIMEOUT = 'timeout' ERROR_TYPE_BAD_CONNECTION = 'bad_connection' @@ -35,9 +35,9 @@ def __init__(self, connection, configuration): # We re-implement this method so we can fetch all the "bindings", as their method only # returns the first for some reason - def match(self, pattern, observed_data_sdos, verbose=False): - compiled_pattern = Pattern(pattern) - matcher = MatchListener(observed_data_sdos, verbose) + def match(self, pattern, observed_data_sdos, verbose=False, stix_version='2.0'): + compiled_pattern = Pattern(pattern, stix_version=stix_version) + matcher = MatchListener(observed_data_sdos, verbose, stix_version=stix_version) compiled_pattern.walk(matcher) found_bindings = matcher.matched() @@ -89,6 +89,10 @@ async def create_results_connection(self, search_id, offset, length): is_stix_21 = self.connection['options'].get("stix_2.1") stix_version = '2.1' if is_stix_21 else '2.0' + if not is_stix_21 and self.test_START_STOP_format(search_id): + # Remove leading 't' before timestamps from search_id. search_id is the stix pattern + search_id = re.sub("(?<=START\s)t|(?<=STOP\s)t", "", search_id) + response = None if self.connection['options'].get('error_type') == ERROR_TYPE_TIMEOUT: # httpstat.us/200?sleep=60000 for slow connection that is valid @@ -125,7 +129,7 @@ async def create_results_connection(self, search_id, offset, length): # Pattern match try: - results = self.match(search_id, observations, False) + results = self.match(search_id, observations, False, stix_version) if len(results) != 0: return_obj['success'] = True @@ -143,3 +147,9 @@ async def delete_query_connection(self, search_id): return_obj = dict() return_obj['success'] = True return return_obj + + def test_START_STOP_format(self, query_string) -> bool: + # Matches START t'1234-56-78T00:00:00.123Z' STOP t'1234-56-78T00:00:00.123Z' + pattern = "START\s(t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z')\sSTOP" + match = re.search(pattern, query_string) + return bool(match) diff --git a/stix_shifter_modules/gcp_chronicle/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/gcp_chronicle/stix_translation/json/stix_2_1/to_stix_map.json index 71561a416..92bb13e9c 100644 --- a/stix_shifter_modules/gcp_chronicle/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/gcp_chronicle/stix_translation/json/stix_2_1/to_stix_map.json @@ -1173,7 +1173,7 @@ "questions": { "key": "network-traffic.extensions.dns-ext.questions", "object": "nt", - "group": "true" + "group": true } }, "dhcp": { diff --git a/stix_shifter_modules/gcp_chronicle/stix_translation/json/to_stix_map.json b/stix_shifter_modules/gcp_chronicle/stix_translation/json/to_stix_map.json index 526dd7de7..ee92e4a62 100644 --- a/stix_shifter_modules/gcp_chronicle/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/gcp_chronicle/stix_translation/json/to_stix_map.json @@ -1214,7 +1214,7 @@ "questions": { "key": "network-traffic.extensions.dns-ext.questions", "object": "nt", - "group": "true" + "group": true } }, "dhcp": { diff --git a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py index 8bb61d735..d3f2e4e5e 100644 --- a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py +++ b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py @@ -208,9 +208,13 @@ def _add_property(self, type_name, property_key, parent_key_ind, value, objects, """ Add observable object property and its STIX valid value to the cached `objects` dictionary """ + named_group = isinstance(group, str) and group.lower() != "true" parent_key_ind_str = str(parent_key_ind) if not parent_key_ind_str in objects: if cybox: + # Grouped properties go in a list + if named_group: + value = [value] objects[parent_key_ind_str] = { 'type': type_name, property_key: value @@ -225,6 +229,9 @@ def _add_property(self, type_name, property_key, parent_key_ind, value, objects, else: if not property_key in objects[parent_key_ind_str]: objects[parent_key_ind_str][property_key] = value + # Add grouped value in existing list element + elif isinstance(value, dict) and named_group and isinstance(objects[parent_key_ind_str][property_key], list): + objects[parent_key_ind_str][property_key][0] = dict_merge(objects[parent_key_ind_str][property_key][0], value) elif isinstance(value, dict): objects[parent_key_ind_str][property_key] = dict_merge(objects[parent_key_ind_str][property_key], value) elif isinstance(objects[parent_key_ind_str][property_key], list) and group: @@ -312,8 +319,10 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje else: type_name = config_keys[0] property_key = config_keys[1] + # set the object to combine properties from same SCO parent_key = prop['object'] if 'object' in prop else type_name + # set the group to combine properties in a list group = prop['group'] if 'group' in prop else False substitute_key = prop['ds_key'] if 'ds_key' in prop else None