From e4e5998ad2a87bb38ab5138653e8b8d474e04965 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 5 Oct 2021 17:45:00 +0530 Subject: [PATCH 01/30] Added connector code for Proofpoint i) Added translator code ii)Added transmission code iii)Added test folder iv)Added mapping files --- stix_shifter_modules/proofpoint/__init__.py | 0 .../proofpoint/configuration/config.json | 39 ++ .../proofpoint/configuration/lang_en.json | 37 ++ .../proofpoint/entry_point.py | 43 ++ .../proofpoint/stix_translation/__init__.py | 0 .../json/dialect1_from_stix_map.json | 28 ++ .../stix_translation/json/to_stix_map.json | 371 ++++++++++++++++++ .../stix_translation/query_constructor.py | 240 +++++++++++ .../stix_translation/query_translator.py | 26 ++ .../stix_translation/results_translator.py | 62 +++ .../stix_translation/transformers.py | 15 + .../proofpoint/stix_transmission/__init__.py | 0 .../stix_transmission/api_client.py | 45 +++ .../stix_transmission/delete_connector.py | 24 ++ .../stix_transmission/error_mapper.py | 44 +++ .../stix_transmission/ping_connector.py | 23 ++ .../stix_transmission/results_connector.py | 55 +++ .../proofpoint/tests/__init__.py | 0 .../tests/stix_translation/__init__.py | 0 .../test_proofpoint_json_to_stix.py | 120 ++++++ .../test_proofpoint_stix_to_query.py | 86 ++++ .../proofpoint/tests/test_async_dummy.py | 57 +++ 22 files changed, 1315 insertions(+) create mode 100755 stix_shifter_modules/proofpoint/__init__.py create mode 100755 stix_shifter_modules/proofpoint/configuration/config.json create mode 100755 stix_shifter_modules/proofpoint/configuration/lang_en.json create mode 100755 stix_shifter_modules/proofpoint/entry_point.py create mode 100755 stix_shifter_modules/proofpoint/stix_translation/__init__.py create mode 100755 stix_shifter_modules/proofpoint/stix_translation/json/dialect1_from_stix_map.json create mode 100755 stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json create mode 100755 stix_shifter_modules/proofpoint/stix_translation/query_constructor.py create mode 100755 stix_shifter_modules/proofpoint/stix_translation/query_translator.py create mode 100755 stix_shifter_modules/proofpoint/stix_translation/results_translator.py create mode 100755 stix_shifter_modules/proofpoint/stix_translation/transformers.py create mode 100755 stix_shifter_modules/proofpoint/stix_transmission/__init__.py create mode 100755 stix_shifter_modules/proofpoint/stix_transmission/api_client.py create mode 100755 stix_shifter_modules/proofpoint/stix_transmission/delete_connector.py create mode 100755 stix_shifter_modules/proofpoint/stix_transmission/error_mapper.py create mode 100755 stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py create mode 100755 stix_shifter_modules/proofpoint/stix_transmission/results_connector.py create mode 100755 stix_shifter_modules/proofpoint/tests/__init__.py create mode 100755 stix_shifter_modules/proofpoint/tests/stix_translation/__init__.py create mode 100755 stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py create mode 100755 stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py create mode 100755 stix_shifter_modules/proofpoint/tests/test_async_dummy.py diff --git a/stix_shifter_modules/proofpoint/__init__.py b/stix_shifter_modules/proofpoint/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/stix_shifter_modules/proofpoint/configuration/config.json b/stix_shifter_modules/proofpoint/configuration/config.json new file mode 100755 index 000000000..03150aef0 --- /dev/null +++ b/stix_shifter_modules/proofpoint/configuration/config.json @@ -0,0 +1,39 @@ +{ + "connection": { + "type": { + "displayName": "Proofpoint" + }, + "host": { + "type": "text" + }, + "port": { + "type": "number", + "default": 443, + "min": 1, + "max": 65535 + }, + "help": { + "type": "link", + "default": "https://www.ibm.com/support/knowledgecenter/SSTDPP_1.6.0/platform/docs/scp-core/data-sources.html" + }, + "sni": { + "type": "text", + "optional": true + }, + "selfSignedCert": { + "type": "password", + "optional": true + } + }, + "configuration": { + "type" : "fields", + "auth": { + "username": { + "type": "password" + }, + "password": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/configuration/lang_en.json b/stix_shifter_modules/proofpoint/configuration/lang_en.json new file mode 100755 index 000000000..fa014b803 --- /dev/null +++ b/stix_shifter_modules/proofpoint/configuration/lang_en.json @@ -0,0 +1,37 @@ +{ + "connection": { + "host": { + "label": "Management IP address or Hostname", + "placeholder": "192.168.1.10", + "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" + }, + "port": { + "label": "Host Port", + "description": "Set the port number that is associated with the Host name or IP" + }, + "help": { + "label": "Need additional help?", + "description": "More details on the data source setting can be found in the specified link" + }, + "selfSignedCert": { + "label": " Connection Certificate", + "description": "Use self-signed SSL certificate and CA content(root and intermediate) of data source" + }, + "sni": { + "label": "Server Name Indicator", + "description": "The Server Name Indicator (SNI) enables a separate hostname to be provided for SSL authentication" + } + }, + "configuration": { + "auth": { + "username": { + "label": "Username", + "description": "Username with access to the search API" + }, + "password": { + "label": "Password", + "description": "Password of the user with access to the search API" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/entry_point.py b/stix_shifter_modules/proofpoint/entry_point.py new file mode 100755 index 000000000..326e245e9 --- /dev/null +++ b/stix_shifter_modules/proofpoint/entry_point.py @@ -0,0 +1,43 @@ +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint +from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from .stix_transmission.api_client import APIClient +from .stix_transmission.ping_connector import PingConnector +from .stix_transmission.results_connector import ResultsConnector +from .stix_transmission.delete_connector import DeleteConnector +from .stix_translation.query_translator import QueryTranslator +from .stix_translation.results_translator import ResultsTranslator +import os + + +class EntryPoint(BaseEntryPoint): + + # python main.py translate async_dummy query '{}' "[ipv4-addr:value = '127.0.0.1']" + # python main.py translate async_dummy:dialect1 query '{}' "[ipv4-addr:value = '127.0.0.1']" + # python main.py translate async_dummy:dialect2 query '{}' "[ipv4-addr:value = '127.0.0.1']" + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + self.set_async(False) + if connection: + api_client = APIClient(connection, configuration) + base_sync_connector = BaseSyncConnector() + ping_connector = PingConnector(api_client) + query_connector = base_sync_connector + status_connector = base_sync_connector + results_connector = ResultsConnector(api_client) + delete_connector = DeleteConnector(api_client) + + self.set_ping_connector(ping_connector) + self.set_query_connector(query_connector) + self.set_status_connector(status_connector) + self.set_results_connector(results_connector) + self.set_delete_connector(delete_connector) + + basepath = os.path.dirname(__file__) + filepath = os.path.abspath( + os.path.join(basepath, "stix_translation")) + + dialect = 'dialect1' + query_translator = QueryTranslator(options, dialect, filepath) + results_translator = ResultsTranslator(options, dialect) + self.add_dialect(dialect, query_translator=query_translator, results_translator=results_translator, default=True) \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_translation/__init__.py b/stix_shifter_modules/proofpoint/stix_translation/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/stix_shifter_modules/proofpoint/stix_translation/json/dialect1_from_stix_map.json b/stix_shifter_modules/proofpoint/stix_translation/json/dialect1_from_stix_map.json new file mode 100755 index 000000000..5be842db2 --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_translation/json/dialect1_from_stix_map.json @@ -0,0 +1,28 @@ +{ + + "format": { + "fields": { + "value": ["format"] + } + }, + "threatStatus": { + "fields": { + "value": ["threatStatus"] + } + }, + "sinceSeconds": { + "fields": { + "value": ["sinceSeconds"] + } + }, + "sinceTime": { + "fields": { + "value": ["sinceTime"] + } + }, + "threatType": { + "fields": { + "value": ["threatType"] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json b/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json new file mode 100755 index 000000000..e2eebfd5d --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json @@ -0,0 +1,371 @@ +{ + + "GUID": [ + { + "key": "x-proofpoint-msgevents.GUID", + "object": "msgevents" + } + ], + "Header": { + + "ccAddresses": [ + { + "key": "email-addr.value", + "object": "email-addr" + }, + { + "key": "email-message.cc_refs", + "object": "email-message", + "references": "email-addr" + } + ], + "fromAddress": [ + { + "key": "email-addr.value", + "object": "from_ref" + }, + { + "key": "email-message.from_ref", + "object": "email-message", + "references": "from_ref" + } + ], + "headerFrom": [ + { + "key": "email-addr.value", + "object": "email-addr" + }, + { + "key": "email-message.from_ref", + "object": "email-message", + "references": "email-addr" + } + ], + "headerReplyTo": [ + { + "key": "email-addr.value", + "object": "email-addr" + }, + { + "key": "email-message.from_ref", + "object": "email-message", + "references": "email-addr" + } + ], + "replyToAddress": [ + { + "key": "email-addr.value", + "object": "email-addr" + }, + { + "key": "email-message.from_ref", + "object": "email-message", + "references": "email-addr" + } + ], + "toAddresses": [ + { + "key": "email-addr.value", + "object": "to_refs" + }, + { + "key": "email-message.to_refs", + "object": "email-message", + "references": "to_refs" + } + ], + "xmailer": [ + { + "key": "x-proofpoint-msgevents.xmailer", + "object": "msgevents" + } + ] + + }, + "cluster": [ + { + "key": "x-proofpoint-msgevents.cluster", + "object": "msgevents" + } + ], + "completelyRewritten": [ + { + "key": "x-proofpoint-msgevents.Rewritten-Status", + "object": "msgevents" + } + ], + "id": [ + { + "key": "x-proofpoint-msgevents.ID", + "object": "msgevents" + } + ], + "impostorScore": [ + { + "key": "x-proofpoint-msgevents.impostorScore", + "transformer": "ToInteger", + "object": "msgevents" + } + ], + "malwareScore": [ + { + "key": "x-proofpoint-msgevents.malwareScore", + "transformer": "ToInteger", + "object": "msgevents" + } + ], + "messageID": [ + { + "key": "x-proofpoint-msgevents.messageID", + "object": "msgevents" + } + ], + "messageParts": { + "key": "email-message.body_multipart", + "object": "email-message", + + "is_multipart": true, + "filename": [ + { + "key": "file.name", + "object": "fl" + } + + ], + "md5": [ + { + "key": "file.hashes.MD5", + "object": "fl" + } + ], + "sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "fl" + } + ] + } + , + "messageSize": [ + { + "key": "x-proofpoint-msgevents.size", + "object": "msgevents", + "transformer": "ToInteger" + } + ], + "messageTime": [ + { + "key": "email-message.date", + "transformer": "EpochToTimestamp", + "object": "email-message" + } + ], + "modulesRun": [ + { + "key": "x-proofpoint-msgevents.modulesRun", + "object": "msgevents" + } + ], + "phishScore": [ + { + "key": "x-proofpoint-msgevents.phishScore", + "transformer": "ToInteger", + "object": "msgevents" + } + ], + "policyRoutes": [ + { + "key": "x-proofpoint-msgevents.policyRoutes", + "object": "msgevents" + } + ], + "quarantineFolder": [ + { + "key": "x-proofpoint-msgevents.quarantineFolder", + "object": "msgevents" + } + ], + "quarantineRule": [ + { + "key": "x-proofpoint-msgevents.quarantineRule", + "object": "msgevents" + } + ], + "recipient": [ + { + "key": "email-addr.value", + "object": "to_refs" + }, + { + "key": "email-message.to_refs", + "object": "email-message", + "references": "to_refs" + } + ], + "sender": [ + { + "key": "email-addr.value", + "object": "sender_ref" + }, + { + "key": "email-message.sender_ref", + "object": "email-message", + "references": "sender_ref" + } + ], + "senderIP": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-proofpoint-msgevents.senderIP", + "object": "msgevents", + "references": "src_ip" + } + ], + "spamScore": [ + { + "key": "x-proofpoint-msgevents.spamScore", + "transformer": "ToInteger", + "object": "msgevents" + } + ], + "subject": [ + { + "key": "email-message.subject", + "object": "email-message" + } + ], + + "clickIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "dst_ip" + } + ], + + "headerTo": [ + { + "key": "email-addr.value", + "object": "to_refs" + }, + { + "key": "email-message.to_refs", + "object": "email-message", + "references": "to_refs" + } + ], + + "headerCC": [ + { + "key": "email-addr.value", + "object": "email-addr" + }, + { + "key": "email-message.cc_refs", + "object": "email-message", + "references": "email-addr" + } + ], + "threatsInfoMap": { + "campaignID": [ + { + "key": "x-proofpoint-msgevents.campaignID", + "object": "msgevents" + } + ], + "classification": [ + { + "key": "x-proofpoint-msgevents.classification", + "object": "msgevents" + } + ], + "threat": [ + { + "key": "x-proofpoint-msgevents.threat", + "object": "msgevents" + } + ], + "threatID": [ + { + "key": "x-proofpoint-msgevents.threatID", + "object": "msgevents" + } + ], + "threatStatus": [ + { + "key": "x-proofpoint-msgevents.threatStatus", + "object": "msgevents" + } + ], + "threatTime": [ + { + "key": "x-proofpoint-msgevents.threatTime", + "transformer": "EpochToTimestamp", + "object": "msgevents" + } + ], + "threatType": [ + { + "key": "x-proofpoint-msgevents.threatType", + "object": "msgevents" + } + ], + "threatUrl": [ + { + "key": "url.value", + "object": "url" + } + ] + }, + "clickTime": [ + { + "key": "x-proofpoint-msgevents.clickTime", + "transformer": "EpochToTimestamp", + "object": "msgevents" + } + ], + "QID": [ + { + "key": "x-proofpoint-msgevents.ID", + "object": "msgevents" + } + ], + "clusterId": [ + { + "key": "x-proofpoint-msgevents.clusterId", + "object": "msgevents" + } + ], + "url": [ + { + "key": "url.value", + "object": "url" + } + ], + "userAgent": [ + { + "key": "x-proofpoint-msgevents.userAgent", + "object": "msgevents" + } + ] +} diff --git a/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py b/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py new file mode 100755 index 000000000..1e0a0cb0b --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py @@ -0,0 +1,240 @@ +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \ + ComparisonExpressionOperators, ComparisonComparators, Pattern, \ + CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators +from stix_shifter_utils.stix_translation.src.utils.transformers import TimestampToMilliseconds +from stix_shifter_utils.stix_translation.src.json_to_stix import observable +import logging +import re + +# Source and destination reference mapping for ip and mac addresses. +# Change the keys to match the data source fields. The value array indicates the possible data type that can come into from field. +REFERENCE_DATA_TYPES = {"SourceIpV4": ["ipv4", "ipv4_cidr"], + "SourceIpV6": ["ipv6"], + "DestinationIpV4": ["ipv4", "ipv4_cidr"], + "DestinationIpV6": ["ipv6"]} + +logger = logging.getLogger(__name__) + +param_delimiter="&" + + +class QueryStringPatternTranslator: + # Change comparator values to match with supported data source operators + comparator_lookup = { + ComparisonExpressionOperators.And: "AND", + ComparisonExpressionOperators.Or: "OR", + ComparisonComparators.GreaterThan: ">", + ComparisonComparators.GreaterThanOrEqual: ">=", + ComparisonComparators.LessThan: "<", + ComparisonComparators.LessThanOrEqual: "<=", + ComparisonComparators.Equal: "=", + ComparisonComparators.NotEqual: "!=", + ComparisonComparators.Like: "LIKE", + ComparisonComparators.In: "IN", + ComparisonComparators.Matches: 'LIKE', + # ComparisonComparators.IsSubSet: '', + # ComparisonComparators.IsSuperSet: '', + ObservationOperators.Or: 'OR', + # Treat AND's as OR's -- Unsure how two ObsExps wouldn't cancel each other out. + ObservationOperators.And: 'OR' + } + + + + def __init__(self, pattern: Pattern, data_model_mapper): + self.dmm = data_model_mapper + self.pattern = pattern + self.translated = self.parse_expression(pattern) + + @staticmethod + def _format_set(values) -> str: + gen = values.element_iterator() + return "({})".format(' OR '.join([QueryStringPatternTranslator._escape_value(value) for value in gen])) + + @staticmethod + def _format_match(value) -> str: + raw = QueryStringPatternTranslator._escape_value(value) + if raw[0] == "^": + raw = raw[1:] + else: + raw = ".*" + raw + if raw[-1] == "$": + raw = raw[0:-1] + else: + raw = raw + ".*" + return "\'{}\'".format(raw) + + @staticmethod + def _format_equality(value) -> str: + # return '\'{}\''.format(value) + return '{}'.format(value) + + @staticmethod + def _format_like(value) -> str: + value = "'%{value}%'".format(value=value) + return QueryStringPatternTranslator._escape_value(value) + + @staticmethod + def _escape_value(value, comparator=None) -> str: + if isinstance(value, str): + return '{}'.format(value.replace('\\', '\\\\').replace('\"', '\\"').replace('(', '\\(').replace(')', '\\)')) + else: + return value + + @staticmethod + def _negate_comparison(comparison_string): + return "NOT ({})".format(comparison_string) + + @staticmethod + def _check_value_type(value): + value = str(value) + for key, pattern in observable.REGEX.items(): + if key != 'date' and bool(re.search(pattern, value)): + return key + return None + + @staticmethod + def _parse_reference(self, stix_field, value_type, mapped_field, value, comparator): + if value_type not in REFERENCE_DATA_TYPES["{}".format(mapped_field)]: + return None + else: + return "{mapped_field} {comparator} {value}".format( + mapped_field=mapped_field, comparator=comparator, value=value) + + @staticmethod + def _parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array): + comparison_string = "" + is_reference_value = self._is_reference_value(stix_field) + # Need to use expression.value to match against regex since the passed-in value has already been formated. + value_type = self._check_value_type(expression.value) if is_reference_value else None + mapped_fields_count = 1 if is_reference_value else len(mapped_fields_array) + + for mapped_field in mapped_fields_array: + if is_reference_value: + parsed_reference = self._parse_reference(self, stix_field, value_type, mapped_field, value, comparator) + if not parsed_reference: + continue + comparison_string += parsed_reference + else: + comparison_string += "{mapped_field}{comparator}{value}".format(mapped_field=mapped_field, comparator=comparator, value=value) + + if (mapped_fields_count > 1): + comparison_string += " OR " + mapped_fields_count -= 1 + return comparison_string + + @staticmethod + def _is_reference_value(stix_field): + return stix_field == 'src_ref.value' or stix_field == 'dst_ref.value' + + @staticmethod + def _lookup_comparison_operator(self, expression_operator): + if expression_operator not in self.comparator_lookup: + raise NotImplementedError("Comparison operator {} unsupported for Dummy connector".format(expression_operator.name)) + return self.comparator_lookup[expression_operator] + + def _parse_expression(self, expression, qualifier=None) -> str: + if isinstance(expression, ComparisonExpression): # Base Case + # Resolve STIX Object Path to a field in the target Data Model + stix_object, stix_field = expression.object_path.split(':') + # Multiple data source fields may map to the same STIX Object + mapped_fields_array = self.dmm.map_field(stix_object, stix_field) + # Resolve the comparison symbol to use in the query string (usually just ':') + comparator = self._lookup_comparison_operator(self, expression.comparator) + + if expression.comparator == ComparisonComparators.Equal or expression.comparator == ComparisonComparators.NotEqual: + # Should be in single-quotes + value = self._format_equality(expression.value) + # # '%' -> '*' wildcard, '_' -> '?' single wildcard + # elif expression.comparator == ComparisonComparators.Like: + # value = self._format_like(expression.value) + else: + value = self._escape_value(expression.value) + + comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array) + if(len(mapped_fields_array) > 1 and not self._is_reference_value(stix_field)): + # More than one data source field maps to the STIX attribute, so group comparisons together. + grouped_comparison_string = "(" + comparison_string + ")" + comparison_string = grouped_comparison_string + + if expression.negated: + comparison_string = self._negate_comparison(comparison_string) + if qualifier is not None: + qualifier = self._parse_time_range(qualifier) + return "{}{}{}".format(comparison_string, param_delimiter ,qualifier) + else: + return "{}".format(comparison_string) + + elif isinstance(expression, CombinedComparisonExpression): + operator = param_delimiter + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + if not expression_01 or not expression_02: + return '' + if isinstance(expression.expr1, CombinedComparisonExpression): + expression_01 = "({})".format(expression_01) + if isinstance(expression.expr2, CombinedComparisonExpression): + expression_02 = "{}{}".format(expression_02, operator) + + query_string = "{}{}{}".format(expression_01, operator, expression_02) + if qualifier is not None: + qualifier = self._parse_time_range(qualifier) + return "{}{}".format(query_string, qualifier) + else: + return "{}".format(query_string) + elif isinstance(expression, ObservationExpression): + return self._parse_expression(expression.comparison_expression, qualifier) + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + if isinstance(expression.observation_expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(self, expression.observation_expression.operator) + expression_01 = self._parse_expression(expression.observation_expression.expr1) + # qualifier only needs to be passed into the parse expression once since it will be the same for both expressions + expression_02 = self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + return "{} {} {}".format(expression_01, operator, expression_02) + else: + return self._parse_expression(expression.observation_expression.comparison_expression, expression.qualifier) + elif isinstance(expression, CombinedObservationExpression): + operator = self._lookup_comparison_operator(self, expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + if expression_01 and expression_02: + return "({}) {} ({})".format(expression_01, operator, expression_02) + elif expression_01: + return "{}".format(expression_01) + elif expression_02: + return "{}".format(expression_02) + else: + return '' + elif isinstance(expression, Pattern): + return "{expr}".format(expr=self._parse_expression(expression.expression)) + else: + raise RuntimeError("Unknown Recursion Case for expression={}, type(expression)={}".format( + expression, type(expression))) + + def parse_expression(self, pattern: Pattern): + return self._parse_expression(pattern) + + + @staticmethod + def _parse_time_range(time_range): + if time_range: + try: + startstr = "STARTt'" + stopstr = "'STOPt'" + indexstart = time_range.index(startstr) + time_range = time_range[indexstart+len(startstr):-1] + # remove stopstr + time_range = re.sub(stopstr, "/", time_range) + time_range = "interval="+time_range + + except: + pass + return time_range + + +def translate_pattern(pattern: Pattern, data_model_mapping, options): + # Query result limit and time range can be passed into the QueryStringPatternTranslator if supported by the data source. + # result_limit = options['result_limit'] + # time_range = options['time_range'] + query = QueryStringPatternTranslator(pattern, data_model_mapping).translated + return [query] diff --git a/stix_shifter_modules/proofpoint/stix_translation/query_translator.py b/stix_shifter_modules/proofpoint/stix_translation/query_translator.py new file mode 100755 index 000000000..575372d07 --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_translation/query_translator.py @@ -0,0 +1,26 @@ +import logging + +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator +from . import query_constructor + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file + :param antlr_parsing_object: Antlr parsing objects for the STIX pattern + :type antlr_parsing_object: object + :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into another format. This should default to something if one isn't passed in + :type mapping: str (filepath) + :return: transformed query string + :rtype: str + """ + + logger.info("Converting STIX2 Pattern to data source query") + + query_string = query_constructor.translate_pattern( + antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/proofpoint/stix_translation/results_translator.py b/stix_shifter_modules/proofpoint/stix_translation/results_translator.py new file mode 100755 index 000000000..1a176b05d --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_translation/results_translator.py @@ -0,0 +1,62 @@ +from stix_shifter_utils.stix_translation.src.json_to_stix.json_to_stix import JSONToStix +from os import path +import json +import ast + +class ResultsTranslator(JSONToStix): + + def __init__(self, options, dialect): + super().__init__(options, dialect, path.dirname(__file__)) + + @staticmethod + def process_multipart(objectsdata, body_multipart, fileobjects): + for ele in body_multipart: + filename = ele.get('filename') + if filename: + for i in fileobjects: + try: + name = objectsdata[i]['name'] + MD5 = objectsdata[i]['hashes']['MD5'] + SHA256 = objectsdata[i]['hashes']['SHA-256'] + if filename==name and MD5==ele.get('md5') and SHA256==ele.get('sha256'): + #add body_raw_ref + ele["body_raw_ref"] = i + #del redundant data + del ele['filename'] + del ele['md5'] + del ele['sha256'] + except: + continue + + @staticmethod + def update_bodymultipart(results): + objectsdata = results['objects'][1]['objects'] + fileobjects = [i for i, j in objectsdata.items() if j['type'] == "file"] + body_multipartobjs = [i for i, j in objectsdata.items() if j.get('body_multipart')] + + if body_multipartobjs: + body_multipart = objectsdata[body_multipartobjs[0]].get('body_multipart') + ResultsTranslator.process_multipart(objectsdata, body_multipart, fileobjects) + + def translate_results(self, data_source, data): + """ + Translates JSON data into STIX results based on a mapping file + :param data: JSON formatted data to translate into STIX format + :type data: str + :param mapping: The mapping file path to use as instructions on how to translate the given JSON data to STIX. + Defaults the path to whatever is passed into the constructor for JSONToSTIX (This should be the to_stix_map.json in the module's json directory) + :type mapping: str (filepath) + :return: STIX formatted results + :rtype: str + """ + results = super().translate_results(data_source, data) + json_data = json.loads(data) + + if len(results['objects']) - 1 == len(json_data): + for i in range(1, len(results['objects'])): + results['objects'][i]['num ber_observed'] = 1 + else: + raise RuntimeError("Incorrect number of result objects after translation. Found: {}, expected: {}.".format( + len(results['objects']) - 1, len(json_data))) + + return results \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_translation/transformers.py b/stix_shifter_modules/proofpoint/stix_translation/transformers.py new file mode 100755 index 000000000..8fdcab99b --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_translation/transformers.py @@ -0,0 +1,15 @@ +from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer +from stix_shifter_utils.utils import logger + +LOGGER = logger.set_logger(__name__) + +class prrofpoint_bodymultipart_transformer(ValueTransformer): + @staticmethod + def transform(multipart): + + # print("transformer multipart :", multipart) + for part in multipart: + part['content_type']=part.pop('contentType') + part['content_disposition'] = part.pop('disposition') + + return multipart \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_transmission/__init__.py b/stix_shifter_modules/proofpoint/stix_transmission/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/stix_shifter_modules/proofpoint/stix_transmission/api_client.py b/stix_shifter_modules/proofpoint/stix_transmission/api_client.py new file mode 100755 index 000000000..a85df196b --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_transmission/api_client.py @@ -0,0 +1,45 @@ +import base64 +from stix_shifter_utils.stix_transmission.utils.RestApiClient import RestApiClient + +ENDPOINT_SINCESEC = 'all?format=syslog&sinceSeconds=3600' +ENDPOINT_ALL = 'all?format=json&' + +class APIClient(): + + def __init__(self, connection, configuration): + # Uncomment when implementing data source API client. + auth = configuration.get('auth') + headers = dict() + cred = tuple() + if auth: + if 'username' in auth and 'password' in auth: + headers['Authorization'] = b"Basic " + base64.b64encode( + (auth['username'] + ':' + auth['password']).encode('ascii')) + self.client = RestApiClient(connection.get('host'), + port=None, + headers=headers, url_modifier_function=None, cert_verify=True, sni=None, auth=None + ) + + def ping_data_source(self): + # Pings the data source + pingresult = self.client.call_api(endpoint=ENDPOINT_SINCESEC, method='GET') + return pingresult + + def create_search(self, query_expression): + # Queries the data source + return {"code": 200, "query_id": "uuid_1234567890"} + + def get_search_status(self, search_id): + # Check the current status of the search + return {"code": 200, "status": "COMPLETED"} + + def get_search_results(self, search_id): + # Return the search results. Results must be in JSON format before being translated into STIX + resultdata = self.client.call_api(endpoint=ENDPOINT_ALL+search_id, method='GET') + # Check the current status of the search + return resultdata + + def delete_search(self, search_id): + # Optional since this may not be supported by the data source API + # Delete the search + return {"code": 200, "success": True} diff --git a/stix_shifter_modules/proofpoint/stix_transmission/delete_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/delete_connector.py new file mode 100755 index 000000000..3c22dda86 --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_transmission/delete_connector.py @@ -0,0 +1,24 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_delete_connector import BaseDeleteConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + +class DeleteConnector(BaseDeleteConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def delete_query_connection(self, search_id): + try: + response_dict = self.api_client.delete_search(search_id) + response_code = response_dict["code"] + + # Construct a response object + return_obj = dict() + if response_code == 200: + return_obj['success'] = response_code['success'] + else: + ErrorResponder.fill_error(return_obj, response_dict, ['message']) + return return_obj + except Exception as err: + self.logger.error('error when deleting search {}:'.format(err)) + raise diff --git a/stix_shifter_modules/proofpoint/stix_transmission/error_mapper.py b/stix_shifter_modules/proofpoint/stix_transmission/error_mapper.py new file mode 100755 index 000000000..6566f5a6a --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_transmission/error_mapper.py @@ -0,0 +1,44 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + # These are only examples. Change the keys to reflect the error codes that come back from the data source API. + # search does not exist + 1002: ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS, + # The search cannot be created. The requested search ID that was provided in the query expression is already in use. + # Please use a unique search ID (or allow one to be generated). + 1004: ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR.value, + # A request parameter is not valid + 1005: ErrorCode.TRANSMISSION_INVALID_PARAMETER, + # The server might be temporarily unavailable or offline. Please try again later. + 1010: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + # An error occurred during the attempt + 1020: ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR.value, + #error in query + 2000: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR +} + + +class ErrorMapper(): + + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + logger = logger.set_logger(__name__) + + @staticmethod + def set_error_code(json_data, return_obj): + code = None + try: + code = int(json_data['code']) + except Exception: + pass + + error_code = ErrorMapper.DEFAULT_ERROR + + if code in error_mapping: + error_code = error_mapping[code] + + if error_code == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: " + str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_code) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py new file mode 100755 index 000000000..5a72dd5d4 --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py @@ -0,0 +1,23 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger + +class PingConnector(BasePingConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def ping_connection(self): + try: + response_dict = self.api_client.ping_data_source() + response_code = response_dict.code + # Construct a response object + return_obj = dict() + if response_code >= 200 and response_code <= 204: + return_obj['success'] = True + else: + ErrorResponder.fill_error(return_obj, str(response_code)+ ":"+str(response_dict.read().decode("utf-8")), ['message']) + return return_obj + except Exception as err: + self.logger.error('error when pinging datasource {}:'.format(err)) + raise diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py new file mode 100755 index 000000000..7edbaaecb --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -0,0 +1,55 @@ +import json + +from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +import ast + +class ResultsConnector(BaseResultsConnector): + def __init__(self, api_client): + self.api_client = api_client + self.logger = logger.set_logger(__name__) + + def create_results_connection(self, search_id, offset, length): + try: + min_range = offset + max_range = offset + length + # Grab the response, extract the response code, and convert it to readable json + response = self.api_client.get_search_results(search_id) + #update response with is_multipart : True + response_code = response.code + response_txt = response.read() + # Construct a response object + return_obj = dict() + if response_code == 200: + return_obj['success'] = True + try: + try: + response_txt = response.read().decode('utf-8') + except: + pass + data= json.loads(response_txt) + newdata = list() + for key, value in data.items(): + if isinstance(value, list) and value: + newdata+=value + + # slice of the data count according to offset values + if newdata and max_range > 0 and len(newdata) > max_range: + newdata = newdata[:max_range] + + return_obj['data'] = newdata + + except json.decoder.JSONDecodeError as e: + return_obj['success'] = False + + else: + return_obj['success'] = False + # ErrorResponder.fill_error(return_obj, response_dict, ['message']) + ErrorResponder.fill_error(return_obj, + str(response_code) + ":" + str(response_txt.decode("utf-8")), + ['message']) + return return_obj + except Exception as err: + self.logger.error('error when getting search results: {}'.format(err)) + raise diff --git a/stix_shifter_modules/proofpoint/tests/__init__.py b/stix_shifter_modules/proofpoint/tests/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/__init__.py b/stix_shifter_modules/proofpoint/tests/stix_translation/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py new file mode 100755 index 000000000..7e5b984c9 --- /dev/null +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py @@ -0,0 +1,120 @@ +import unittest +from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator +from stix_shifter_modules.proofpoint.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers +import json +from stix_shifter.stix_translation import stix_translation + +MODULE = "proofpoint" +entry_point = EntryPoint() +map_data = entry_point.get_results_translator().map_data +data_source = { + "type": "identity", + "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "name": "Proofpoint", + "identity_class": "events" +} +options = {} + +event_data=[{ + "is_multipart": True, + "GUID": "Ggfsdfsdf", + + "Header": { + "ccAddresses": [], + "fromAddress": [ + "Header@xxx.com" + ], + "headerFrom": "\"j.\" ", + "headerReplyTo": None, + "replyToAddress": [], + "toAddresses": [], + "xmailer": None + }, + "cluster": "hosted", + "completelyRewritten": True, + "id": "1828003vsdv05566e842", + "impostorScore": 0, + "malwareScore": 0, + "messageID": "", + "messageParts": [ + { + "contentType": "text/html", + "disposition": "inline", + "filename": "text.html", + "md5": "fcfa9b21f43fbdf02965263c63e", + "oContentType": "text/html", + "sandboxStatus": "None", + "sha256": "72d3dc7a01dfbdbe8e871536864f56bf235ba08ff259105ac" + }, +{ + "contentType": "text/html", + "disposition": "inline", + "filename": "text2.html", + "md5": "fcfa9b21f43fbdf02965263c63e", + "oContentType": "text/html", + "sandboxStatus": None, + "sha256": "72d3dc7a01dfbdbe8e871536864f56bf235ba08ff259105bd" + } + ], + "messageSize": 10171, + "messageTime": "2021-06-02T13:41:32.000Z", + "modulesRun": [ + "av", + "spf", + "dkimv", + "spam", + "dmarc", + "urldefense" + ], + "phishScore": 0, + "policyRoutes": [ + "default_inbound", + "allow_relay" + ], + "quarantineFolder": None, + "quarantineRule": None, + "recipient": [ + "recipient@xxx.com" + ], + "sender": "sender@xxx.com", + "senderIP": "400.000.000", + "spamScore": 43, + "subject": "=", + "threatsInfoMap": [ + { + "campaignID": None, + "classification": "phish", + "threat": "https://bit.ly", + "threatID": "45fe3b35ghkk2b8916934b6c0a536cc9b2603d03", + "threatStatus": "active", + "threatTime": "2021-06-03T07:17:11.000Z", + "threatType": "url", + "threatUrl": "https://threatinsight.proofpoint.com" + }, +{ + "campaignID": None, + "classification": "phish", + "threat": "https://bit123.ly", + "threatID": "45fe3b35ghkk2b8916934b6c0a536cc9b2603d04", + "threatStatus": "active", + "threatTime": "2021-06-04T07:17:11.000Z", + "threatType": "url", + "threatUrl": "https://threatinsight.proofpoint.com" + } + ] + }] +class TestProofpointResultsToStix(unittest.TestCase): + """ + class to perform unit test case for proofpoint translate results + """ + + def test_custom_mapping(self): + # data_source_string = json.dumps(data_source) + # data_string = json.dumps(message_data) + # translation = stix_translation.StixTranslation() + # result_bundle = translation.translate('proofpoint', 'results', data_source_string, data_string, options) + + result_bundle = entry_point.translate_results(json.dumps(data_source), json.dumps([event_data])) + + print('result_bundle :', result_bundle) \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py new file mode 100755 index 000000000..6c905586c --- /dev/null +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py @@ -0,0 +1,86 @@ +from stix_shifter_modules.proofpoint.entry_point import EntryPoint +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +from stix_shifter.stix_translation import stix_translation +from stix_shifter_utils.utils.error_response import ErrorCode +import unittest +import datetime +import re + + +translation = stix_translation.StixTranslation() + +def _test_query_assertions(translated_query, test_query): + assert translated_query['queries'] == test_query + +def _remove_timestamp_from_query(queries): + pattern = r'\s*AND\s*\(\@timestamp:\["\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z"\s*TO\s*"\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z"\]\)' + if isinstance(queries, list): + return [re.sub(pattern, "", query) for query in queries] + elif isinstance(queries, str): + return re.sub(pattern, "", queries) + + +class TestAsyncDummyConnection(unittest.TestCase, object): + + def connection(self): + return { + "host": "hostbla", + "port": 8080, + } + + def configuration(self): + return { + "auth": { + "username": "u", + "password": "p" + } + } + + def test_dummy_async_query(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + query = "placeholder query text" + query_response = entry_point.create_query_connection(query) + + assert query_response['search_id'] == "uuid_1234567890" + + def test_dummy_async_status(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + query_id = "uuid_1234567890" + status_response = entry_point.create_status_connection(query_id) + + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.COMPLETED.value + + def test_dummy_async_results(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + # query_id = "uuid_1234567890" + query_id = 'all?format=syslog&sinceSeconds=3600' + results_response = entry_point.create_results_connection(query_id, 1, 1) + print('results_response :', results_response) + + success = results_response["success"] + assert success + data = results_response["data"] + # assert data == "Results from search" + + def test_is_async(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + check_async = entry_point.is_async() + assert check_async + + def test_ping(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_result = entry_point.ping_connection() + assert ping_result["success"] is True + +class TestStixtoQuery(unittest.TestCase, object): + + def test_query(self): + stix_pattern = "[threatStatus:value = 'active' OR threatStatus:value = 'positive' OR threatStatus:value = 'falsepositive'] START t'2021-09-15T16:13:00.00Z' STOP t'2021-09-15T17:13:00.00Z'" + stix_pattern = "[threatStatus:value = 'active'] START t'2021-08-22T07:24:00.000Z' STOP t'2022-08-22T08:20:00.000Z'" + translated_query = translation.translate('proofpoint', 'query', '{}', stix_pattern) + translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries']) + test_query = ['hreatStatus=active&interval=2021-08-22T07:24:00.000Z/2022-08-22T08:20:00.000Z'] + _test_query_assertions(translated_query, test_query) diff --git a/stix_shifter_modules/proofpoint/tests/test_async_dummy.py b/stix_shifter_modules/proofpoint/tests/test_async_dummy.py new file mode 100755 index 000000000..a62d4ca9d --- /dev/null +++ b/stix_shifter_modules/proofpoint/tests/test_async_dummy.py @@ -0,0 +1,57 @@ +from stix_shifter_modules.async_dummy.entry_point import EntryPoint +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +import unittest + + +class TestAsyncDummyConnection(unittest.TestCase, object): + + def connection(self): + return { + "host": "hostbla", + "port": 8080, + } + + def configuration(self): + return { + "auth": { + "username": "u", + "password": "p" + } + } + + def test_dummy_async_query(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + query = "placeholder query text" + query_response = entry_point.create_query_connection(query) + + assert query_response['search_id'] == "uuid_1234567890" + + def test_dummy_async_status(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + query_id = "uuid_1234567890" + status_response = entry_point.create_status_connection(query_id) + + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.COMPLETED.value + + def test_dummy_async_results(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + query_id = "uuid_1234567890" + results_response = entry_point.create_results_connection(query_id, 1, 1) + + success = results_response["success"] + assert success + data = results_response["data"] + assert data == "Results from search" + + def test_is_async(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + check_async = entry_point.is_async() + assert check_async + + def test_ping(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_result = entry_point.ping_connection() + assert ping_result["success"] is True From ae43a6cc9cdeb877436d142e3de6ca876f5fed4b Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 8 Oct 2021 12:04:40 +0530 Subject: [PATCH 02/30] i) Updated code for first code review comments ii)Added code for error handling in results_connector --- .../proofpoint/entry_point.py | 2 +- .../json/dialect1_from_stix_map.json | 28 ------------------- .../stix_translation/json/from_stix_map.json | 8 ++++++ .../stix_transmission/api_client.py | 1 - .../stix_transmission/error_mapper.py | 21 ++++---------- .../stix_transmission/results_connector.py | 24 +++++++++++++--- .../test_proofpoint_stix_to_query.py | 6 ++-- 7 files changed, 38 insertions(+), 52 deletions(-) delete mode 100755 stix_shifter_modules/proofpoint/stix_translation/json/dialect1_from_stix_map.json create mode 100755 stix_shifter_modules/proofpoint/stix_translation/json/from_stix_map.json diff --git a/stix_shifter_modules/proofpoint/entry_point.py b/stix_shifter_modules/proofpoint/entry_point.py index 326e245e9..c27b9fb74 100755 --- a/stix_shifter_modules/proofpoint/entry_point.py +++ b/stix_shifter_modules/proofpoint/entry_point.py @@ -37,7 +37,7 @@ def __init__(self, connection={}, configuration={}, options={}): filepath = os.path.abspath( os.path.join(basepath, "stix_translation")) - dialect = 'dialect1' + dialect = 'default' query_translator = QueryTranslator(options, dialect, filepath) results_translator = ResultsTranslator(options, dialect) self.add_dialect(dialect, query_translator=query_translator, results_translator=results_translator, default=True) \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_translation/json/dialect1_from_stix_map.json b/stix_shifter_modules/proofpoint/stix_translation/json/dialect1_from_stix_map.json deleted file mode 100755 index 5be842db2..000000000 --- a/stix_shifter_modules/proofpoint/stix_translation/json/dialect1_from_stix_map.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - - "format": { - "fields": { - "value": ["format"] - } - }, - "threatStatus": { - "fields": { - "value": ["threatStatus"] - } - }, - "sinceSeconds": { - "fields": { - "value": ["sinceSeconds"] - } - }, - "sinceTime": { - "fields": { - "value": ["sinceTime"] - } - }, - "threatType": { - "fields": { - "value": ["threatType"] - } - } -} \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_translation/json/from_stix_map.json b/stix_shifter_modules/proofpoint/stix_translation/json/from_stix_map.json new file mode 100755 index 000000000..86300f938 --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_translation/json/from_stix_map.json @@ -0,0 +1,8 @@ +{ + "x-proofpoint": { + "fields": { + "threatstatus": ["threatStatus"], + "threattype": ["threatType"] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_transmission/api_client.py b/stix_shifter_modules/proofpoint/stix_transmission/api_client.py index a85df196b..d0b5554e6 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/api_client.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/api_client.py @@ -10,7 +10,6 @@ def __init__(self, connection, configuration): # Uncomment when implementing data source API client. auth = configuration.get('auth') headers = dict() - cred = tuple() if auth: if 'username' in auth and 'password' in auth: headers['Authorization'] = b"Basic " + base64.b64encode( diff --git a/stix_shifter_modules/proofpoint/stix_transmission/error_mapper.py b/stix_shifter_modules/proofpoint/stix_transmission/error_mapper.py index 6566f5a6a..ae77475a7 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/error_mapper.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/error_mapper.py @@ -3,20 +3,11 @@ from stix_shifter_utils.utils import logger error_mapping = { - # These are only examples. Change the keys to reflect the error codes that come back from the data source API. - # search does not exist - 1002: ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS, - # The search cannot be created. The requested search ID that was provided in the query expression is already in use. - # Please use a unique search ID (or allow one to be generated). - 1004: ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR.value, - # A request parameter is not valid - 1005: ErrorCode.TRANSMISSION_INVALID_PARAMETER, - # The server might be temporarily unavailable or offline. Please try again later. - 1010: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, - # An error occurred during the attempt - 1020: ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR.value, - #error in query - 2000: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR + 2000: ErrorCode.TRANSMISSION_RESPONSE_EMPTY_RESULT, + 4000: ErrorCode.TRANSMISSION_SEARCH_DOES_NOT_EXISTS, + 4010: ErrorCode.TRANSMISSION_AUTH_SSL, + 4020: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + 7000: ErrorCode.TRANSMISSION_UNKNOWN } @@ -41,4 +32,4 @@ def set_error_code(json_data, return_obj): if error_code == ErrorMapper.DEFAULT_ERROR: ErrorMapper.logger.error("failed to map: " + str(json_data)) - ErrorMapperBase.set_error_code(return_obj, error_code) + ErrorMapperBase.set_error_code(return_obj, error_code, json_data['message']) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index 7edbaaecb..d36da60d1 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -21,6 +21,7 @@ def create_results_connection(self, search_id, offset, length): response_txt = response.read() # Construct a response object return_obj = dict() + error_obj = dict() if response_code == 200: return_obj['success'] = True try: @@ -43,13 +44,28 @@ def create_results_connection(self, search_id, offset, length): except json.decoder.JSONDecodeError as e: return_obj['success'] = False - else: - return_obj['success'] = False - # ErrorResponder.fill_error(return_obj, response_dict, ['message']) + elif response_code > 200 and response_code <= 204:#empty results + error_obj['code'] = 2000 + elif response_code == 400:#error from data source + error_obj['code'] = 4000 + elif response_code == 401:#Authentication error + error_obj['code'] = 4010 + elif response_code >= 402 and response_code <= 499:#All other client side errors + error_obj['code'] = 4020 + else:#unknown errors + error_obj['code'] = 7000 + if error_obj: + error_obj['message'] = str(response_txt.decode("utf-8")) ErrorResponder.fill_error(return_obj, - str(response_code) + ":" + str(response_txt.decode("utf-8")), + error_obj, ['message']) + err = 'error when getting search results: {}:{}'.format(str(response_code),str(response_txt.decode("utf-8"))) + self.logger.error(err) + # raise NoResultsFoundError(err) return return_obj except Exception as err: self.logger.error('error when getting search results: {}'.format(err)) raise + +class NoResultsFoundError(Exception): + pass diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py index 6c905586c..19c8b6598 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py @@ -78,9 +78,9 @@ def test_ping(self): class TestStixtoQuery(unittest.TestCase, object): def test_query(self): - stix_pattern = "[threatStatus:value = 'active' OR threatStatus:value = 'positive' OR threatStatus:value = 'falsepositive'] START t'2021-09-15T16:13:00.00Z' STOP t'2021-09-15T17:13:00.00Z'" - stix_pattern = "[threatStatus:value = 'active'] START t'2021-08-22T07:24:00.000Z' STOP t'2022-08-22T08:20:00.000Z'" + # stix_pattern = "[x-proofpoint:threatstatus = 'active' OR x-proofpoint:threatstatus = 'positive' OR x-proofpoint:threatstatuss = 'falsepositive'] START t'2021-09-15T16:13:00.00Z' STOP t'2021-09-15T17:13:00.00Z'" + stix_pattern = "[x-proofpoint:threatstatus = 'active'] START t'2021-08-22T07:24:00.000Z' STOP t'2022-08-22T08:20:00.000Z'" translated_query = translation.translate('proofpoint', 'query', '{}', stix_pattern) translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries']) - test_query = ['hreatStatus=active&interval=2021-08-22T07:24:00.000Z/2022-08-22T08:20:00.000Z'] + test_query = ['threatStatus=active&interval=2021-08-22T07:24:00.000Z/2022-08-22T08:20:00.000Z'] _test_query_assertions(translated_query, test_query) From 224625bb1e07a1b15b55e55ee7a6fae1052fcd8d Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 12 Oct 2021 12:13:48 +0530 Subject: [PATCH 03/30] Updated code for second code review comments i)Added code for url and end point ii) Corrected configuration attributes in config file iii) Added a function to append default time range if not present in input STIX pattern --- .../proofpoint/configuration/config.json | 4 ++-- .../stix_translation/query_translator.py | 23 +++++++++++++++++++ .../stix_transmission/api_client.py | 16 +++++++------ .../stix_transmission/results_connector.py | 9 ++++++-- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/stix_shifter_modules/proofpoint/configuration/config.json b/stix_shifter_modules/proofpoint/configuration/config.json index 03150aef0..849c53546 100755 --- a/stix_shifter_modules/proofpoint/configuration/config.json +++ b/stix_shifter_modules/proofpoint/configuration/config.json @@ -28,10 +28,10 @@ "configuration": { "type" : "fields", "auth": { - "username": { + "principal": { "type": "password" }, - "password": { + "secret": { "type": "password" } } diff --git a/stix_shifter_modules/proofpoint/stix_translation/query_translator.py b/stix_shifter_modules/proofpoint/stix_translation/query_translator.py index 575372d07..40c568bf5 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/query_translator.py +++ b/stix_shifter_modules/proofpoint/stix_translation/query_translator.py @@ -3,6 +3,8 @@ from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator from . import query_constructor +from datetime import datetime, timedelta + logger = logging.getLogger(__name__) @@ -20,7 +22,28 @@ def transform_antlr(self, data, antlr_parsing_object): """ logger.info("Converting STIX2 Pattern to data source query") + logger.info(data) query_string = query_constructor.translate_pattern( antlr_parsing_object, self, self.options) return query_string + + + def transform_query(self, data): + # check if time range is present, add and call super() + if data and 'START' not in data and 'STOP' not in data: + #append data with default time range + data = data + self._append_time_range() + logger.info(data) + return BaseQueryTranslator.transform_query(self, data) + + @staticmethod + def _append_time_range(): + #get current system time + endtime = datetime.now() - timedelta(hours=3) + stop = (endtime.strftime("%Y-%m-%dT%H:%M:%S%Z.00Z")) + starttime = endtime - timedelta(hours=1) + start = (starttime.strftime("%Y-%m-%dT%H:%M:%S%Z.00Z")) + time_range = "START t'{}' STOP t'{}'".format(start, stop) + return time_range + diff --git a/stix_shifter_modules/proofpoint/stix_transmission/api_client.py b/stix_shifter_modules/proofpoint/stix_transmission/api_client.py index d0b5554e6..cd88a76f4 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/api_client.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/api_client.py @@ -1,8 +1,7 @@ import base64 from stix_shifter_utils.stix_transmission.utils.RestApiClient import RestApiClient -ENDPOINT_SINCESEC = 'all?format=syslog&sinceSeconds=3600' -ENDPOINT_ALL = 'all?format=json&' +ENDPOINT_ALL = 'v2/siem/all' class APIClient(): @@ -11,9 +10,9 @@ def __init__(self, connection, configuration): auth = configuration.get('auth') headers = dict() if auth: - if 'username' in auth and 'password' in auth: + if 'principal' in auth and 'secret' in auth: headers['Authorization'] = b"Basic " + base64.b64encode( - (auth['username'] + ':' + auth['password']).encode('ascii')) + (auth['principal'] + ':' + auth['secret']).encode('ascii')) self.client = RestApiClient(connection.get('host'), port=None, headers=headers, url_modifier_function=None, cert_verify=True, sni=None, auth=None @@ -21,12 +20,13 @@ def __init__(self, connection, configuration): def ping_data_source(self): # Pings the data source - pingresult = self.client.call_api(endpoint=ENDPOINT_SINCESEC, method='GET') + endpoint = ENDPOINT_ALL + "?format=syslog&sinceSeconds=3600" + pingresult = self.client.call_api(endpoint=endpoint, method='GET') return pingresult def create_search(self, query_expression): # Queries the data source - return {"code": 200, "query_id": "uuid_1234567890"} + return {"code": 200, "query_id": query_expression} def get_search_status(self, search_id): # Check the current status of the search @@ -34,7 +34,9 @@ def get_search_status(self, search_id): def get_search_results(self, search_id): # Return the search results. Results must be in JSON format before being translated into STIX - resultdata = self.client.call_api(endpoint=ENDPOINT_ALL+search_id, method='GET') + #resultdata = self.client.call_api(endpoint=ENDPOINT_ALL+search_id, method='GET')#working + endpoint = ENDPOINT_ALL+"?format=json&" + resultdata = self.client.call_api(endpoint=endpoint, method='GET', urldata=search_id) # Check the current status of the search return resultdata diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index d36da60d1..a289e59d4 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -55,11 +55,16 @@ def create_results_connection(self, search_id, offset, length): else:#unknown errors error_obj['code'] = 7000 if error_obj: - error_obj['message'] = str(response_txt.decode("utf-8")) + error_msg = "" + try: + error_msg = str(response_txt.decode("utf-8")) + except: + pass + error_obj['message'] = error_msg ErrorResponder.fill_error(return_obj, error_obj, ['message']) - err = 'error when getting search results: {}:{}'.format(str(response_code),str(response_txt.decode("utf-8"))) + err = 'error when getting search results: {}:{}'.format(str(response_code),error_msg) self.logger.error(err) # raise NoResultsFoundError(err) return return_obj From 642d920661614ccf7701954e0ca30435c32ac868 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 19 Oct 2021 10:17:39 +0530 Subject: [PATCH 04/30] Updated code for code review comments - code optimization --- .../proofpoint/configuration/config.json | 8 --- .../proofpoint/configuration/lang_en.json | 12 +--- .../proofpoint/entry_point.py | 36 +----------- .../stix_translation/json/to_stix_map.json | 3 - .../stix_translation/query_constructor.py | 51 +++++++++++------ .../stix_translation/query_translator.py | 22 ------- .../stix_translation/results_translator.py | 36 +----------- .../stix_transmission/query_connector.py | 5 ++ .../stix_transmission/results_connector.py | 21 ++++--- .../stix_transmission/status_connector.py | 5 ++ .../test_proofpoint_json_to_stix.py | 4 +- .../proofpoint/tests/test_async_dummy.py | 57 ------------------- 12 files changed, 64 insertions(+), 196 deletions(-) create mode 100644 stix_shifter_modules/proofpoint/stix_transmission/query_connector.py create mode 100644 stix_shifter_modules/proofpoint/stix_transmission/status_connector.py delete mode 100755 stix_shifter_modules/proofpoint/tests/test_async_dummy.py diff --git a/stix_shifter_modules/proofpoint/configuration/config.json b/stix_shifter_modules/proofpoint/configuration/config.json index 849c53546..51dca298e 100755 --- a/stix_shifter_modules/proofpoint/configuration/config.json +++ b/stix_shifter_modules/proofpoint/configuration/config.json @@ -15,14 +15,6 @@ "help": { "type": "link", "default": "https://www.ibm.com/support/knowledgecenter/SSTDPP_1.6.0/platform/docs/scp-core/data-sources.html" - }, - "sni": { - "type": "text", - "optional": true - }, - "selfSignedCert": { - "type": "password", - "optional": true } }, "configuration": { diff --git a/stix_shifter_modules/proofpoint/configuration/lang_en.json b/stix_shifter_modules/proofpoint/configuration/lang_en.json index fa014b803..998a213ae 100755 --- a/stix_shifter_modules/proofpoint/configuration/lang_en.json +++ b/stix_shifter_modules/proofpoint/configuration/lang_en.json @@ -12,23 +12,15 @@ "help": { "label": "Need additional help?", "description": "More details on the data source setting can be found in the specified link" - }, - "selfSignedCert": { - "label": " Connection Certificate", - "description": "Use self-signed SSL certificate and CA content(root and intermediate) of data source" - }, - "sni": { - "label": "Server Name Indicator", - "description": "The Server Name Indicator (SNI) enables a separate hostname to be provided for SSL authentication" } }, "configuration": { "auth": { - "username": { + "principal": { "label": "Username", "description": "Username with access to the search API" }, - "password": { + "secret": { "label": "Password", "description": "Password of the user with access to the search API" } diff --git a/stix_shifter_modules/proofpoint/entry_point.py b/stix_shifter_modules/proofpoint/entry_point.py index c27b9fb74..258fb7284 100755 --- a/stix_shifter_modules/proofpoint/entry_point.py +++ b/stix_shifter_modules/proofpoint/entry_point.py @@ -1,43 +1,11 @@ from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector -from .stix_transmission.api_client import APIClient -from .stix_transmission.ping_connector import PingConnector -from .stix_transmission.results_connector import ResultsConnector -from .stix_transmission.delete_connector import DeleteConnector -from .stix_translation.query_translator import QueryTranslator -from .stix_translation.results_translator import ResultsTranslator -import os - class EntryPoint(BaseEntryPoint): - # python main.py translate async_dummy query '{}' "[ipv4-addr:value = '127.0.0.1']" - # python main.py translate async_dummy:dialect1 query '{}' "[ipv4-addr:value = '127.0.0.1']" - # python main.py translate async_dummy:dialect2 query '{}' "[ipv4-addr:value = '127.0.0.1']" - def __init__(self, connection={}, configuration={}, options={}): super().__init__(connection, configuration, options) self.set_async(False) if connection: - api_client = APIClient(connection, configuration) - base_sync_connector = BaseSyncConnector() - ping_connector = PingConnector(api_client) - query_connector = base_sync_connector - status_connector = base_sync_connector - results_connector = ResultsConnector(api_client) - delete_connector = DeleteConnector(api_client) - - self.set_ping_connector(ping_connector) - self.set_query_connector(query_connector) - self.set_status_connector(status_connector) - self.set_results_connector(results_connector) - self.set_delete_connector(delete_connector) - - basepath = os.path.dirname(__file__) - filepath = os.path.abspath( - os.path.join(basepath, "stix_translation")) + self.setup_transmission_simple(connection, configuration) - dialect = 'default' - query_translator = QueryTranslator(options, dialect, filepath) - results_translator = ResultsTranslator(options, dialect) - self.add_dialect(dialect, query_translator=query_translator, results_translator=results_translator, default=True) \ No newline at end of file + self.setup_translation_simple(dialect_default='default') \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json b/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json index e2eebfd5d..15dc3af4c 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json @@ -156,7 +156,6 @@ "messageTime": [ { "key": "email-message.date", - "transformer": "EpochToTimestamp", "object": "email-message" } ], @@ -320,7 +319,6 @@ "threatTime": [ { "key": "x-proofpoint-msgevents.threatTime", - "transformer": "EpochToTimestamp", "object": "msgevents" } ], @@ -340,7 +338,6 @@ "clickTime": [ { "key": "x-proofpoint-msgevents.clickTime", - "transformer": "EpochToTimestamp", "object": "msgevents" } ], diff --git a/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py b/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py index 1e0a0cb0b..4c41b4b59 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py +++ b/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py @@ -1,9 +1,9 @@ from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, ComparisonExpression, \ ComparisonExpressionOperators, ComparisonComparators, Pattern, \ CombinedComparisonExpression, CombinedObservationExpression, ObservationOperators -from stix_shifter_utils.stix_translation.src.utils.transformers import TimestampToMilliseconds from stix_shifter_utils.stix_translation.src.json_to_stix import observable import logging +from datetime import datetime, timedelta import re # Source and destination reference mapping for ip and mac addresses. @@ -13,6 +13,8 @@ "DestinationIpV4": ["ipv4", "ipv4_cidr"], "DestinationIpV6": ["ipv6"]} +START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)" + logger = logging.getLogger(__name__) param_delimiter="&" @@ -43,6 +45,7 @@ class QueryStringPatternTranslator: def __init__(self, pattern: Pattern, data_model_mapper): self.dmm = data_model_mapper + self.qualifier_string = '' self.pattern = pattern self.translated = self.parse_expression(pattern) @@ -183,7 +186,9 @@ def _parse_expression(self, expression, qualifier=None) -> str: else: return "{}".format(query_string) elif isinstance(expression, ObservationExpression): + self.qualifier_string = self._parse_time_range(qualifier) return self._parse_expression(expression.comparison_expression, qualifier) + # return "{}".format(self.qualifier_string) elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): if isinstance(expression.observation_expression, CombinedObservationExpression): operator = self._lookup_comparison_operator(self, expression.observation_expression.operator) @@ -212,24 +217,38 @@ def _parse_expression(self, expression, qualifier=None) -> str: expression, type(expression))) def parse_expression(self, pattern: Pattern): - return self._parse_expression(pattern) + query = self._parse_expression(pattern) + if self.qualifier_string: + query = "{query}{param_delimiter}{qualifier_string}".format(query=query, param_delimiter=param_delimiter, qualifier_string=self.qualifier_string) + return query @staticmethod - def _parse_time_range(time_range): - if time_range: - try: - startstr = "STARTt'" - stopstr = "'STOPt'" - indexstart = time_range.index(startstr) - time_range = time_range[indexstart+len(startstr):-1] - # remove stopstr - time_range = re.sub(stopstr, "/", time_range) - time_range = "interval="+time_range - - except: - pass - return time_range + def _parse_time_range(qualifier, time_range=1): + """ + :param qualifier: str, input time range i.e START t'2019-04-10T08:43:10.003Z' STOP t'2019-04-20T10:43:10.003Z' + :param time_range: int, value available from main.py in options variable + :return: str, format_string bound with time range provided + """ + try: + compile_timestamp_regex = re.compile(START_STOP_PATTERN) + mapped_field = "interval" + if qualifier and compile_timestamp_regex.search(qualifier): + time_range_iterator = compile_timestamp_regex.finditer(qualifier) + time_range_list = [each.group() for each in time_range_iterator] + # Default time range Start time = Now - 1 hours and Stop time = Now + else: + stop_time = datetime.utcnow() + start_time = stop_time - timedelta(hours=time_range) + converted_starttime = start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + converted_stoptime = stop_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + time_range_list = [converted_starttime, converted_stoptime] + value = "{start_time}/{stop_time}".format(start_time=time_range_list[0], stop_time=time_range_list[1]) + format_string = '{mapped_field}={value}'.format(mapped_field=mapped_field, value=value) + + except (KeyError, IndexError, TypeError) as e: + raise e + return format_string def translate_pattern(pattern: Pattern, data_model_mapping, options): diff --git a/stix_shifter_modules/proofpoint/stix_translation/query_translator.py b/stix_shifter_modules/proofpoint/stix_translation/query_translator.py index 40c568bf5..bbf7d2201 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/query_translator.py +++ b/stix_shifter_modules/proofpoint/stix_translation/query_translator.py @@ -2,9 +2,6 @@ from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator from . import query_constructor - -from datetime import datetime, timedelta - logger = logging.getLogger(__name__) @@ -28,22 +25,3 @@ def transform_antlr(self, data, antlr_parsing_object): antlr_parsing_object, self, self.options) return query_string - - def transform_query(self, data): - # check if time range is present, add and call super() - if data and 'START' not in data and 'STOP' not in data: - #append data with default time range - data = data + self._append_time_range() - logger.info(data) - return BaseQueryTranslator.transform_query(self, data) - - @staticmethod - def _append_time_range(): - #get current system time - endtime = datetime.now() - timedelta(hours=3) - stop = (endtime.strftime("%Y-%m-%dT%H:%M:%S%Z.00Z")) - starttime = endtime - timedelta(hours=1) - start = (starttime.strftime("%Y-%m-%dT%H:%M:%S%Z.00Z")) - time_range = "START t'{}' STOP t'{}'".format(start, stop) - return time_range - diff --git a/stix_shifter_modules/proofpoint/stix_translation/results_translator.py b/stix_shifter_modules/proofpoint/stix_translation/results_translator.py index 1a176b05d..b1f88c149 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/results_translator.py +++ b/stix_shifter_modules/proofpoint/stix_translation/results_translator.py @@ -1,42 +1,12 @@ from stix_shifter_utils.stix_translation.src.json_to_stix.json_to_stix import JSONToStix from os import path import json -import ast class ResultsTranslator(JSONToStix): - def __init__(self, options, dialect): - super().__init__(options, dialect, path.dirname(__file__)) + def __init__(self, options, dialect, base_file_path=path.dirname(__file__), callback=None): + super().__init__(options, dialect, base_file_path=base_file_path) - @staticmethod - def process_multipart(objectsdata, body_multipart, fileobjects): - for ele in body_multipart: - filename = ele.get('filename') - if filename: - for i in fileobjects: - try: - name = objectsdata[i]['name'] - MD5 = objectsdata[i]['hashes']['MD5'] - SHA256 = objectsdata[i]['hashes']['SHA-256'] - if filename==name and MD5==ele.get('md5') and SHA256==ele.get('sha256'): - #add body_raw_ref - ele["body_raw_ref"] = i - #del redundant data - del ele['filename'] - del ele['md5'] - del ele['sha256'] - except: - continue - - @staticmethod - def update_bodymultipart(results): - objectsdata = results['objects'][1]['objects'] - fileobjects = [i for i, j in objectsdata.items() if j['type'] == "file"] - body_multipartobjs = [i for i, j in objectsdata.items() if j.get('body_multipart')] - - if body_multipartobjs: - body_multipart = objectsdata[body_multipartobjs[0]].get('body_multipart') - ResultsTranslator.process_multipart(objectsdata, body_multipart, fileobjects) def translate_results(self, data_source, data): """ @@ -54,7 +24,7 @@ def translate_results(self, data_source, data): if len(results['objects']) - 1 == len(json_data): for i in range(1, len(results['objects'])): - results['objects'][i]['num ber_observed'] = 1 + results['objects'][i]['number_observed'] = 1 else: raise RuntimeError("Incorrect number of result objects after translation. Found: {}, expected: {}.".format( len(results['objects']) - 1, len(json_data))) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/query_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/query_connector.py new file mode 100644 index 000000000..ee6f57cef --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_transmission/query_connector.py @@ -0,0 +1,5 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector + +class QueryConnector(BaseSyncConnector): + def __init__(self, api_client): + self.api_client = api_client \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index a289e59d4..26a5985eb 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -3,7 +3,6 @@ from stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -import ast class ResultsConnector(BaseResultsConnector): def __init__(self, api_client): @@ -16,7 +15,6 @@ def create_results_connection(self, search_id, offset, length): max_range = offset + length # Grab the response, extract the response code, and convert it to readable json response = self.api_client.get_search_results(search_id) - #update response with is_multipart : True response_code = response.code response_txt = response.read() # Construct a response object @@ -25,10 +23,7 @@ def create_results_connection(self, search_id, offset, length): if response_code == 200: return_obj['success'] = True try: - try: - response_txt = response.read().decode('utf-8') - except: - pass + response_txt = response.read().decode('utf-8') data= json.loads(response_txt) newdata = list() for key, value in data.items(): @@ -41,8 +36,11 @@ def create_results_connection(self, search_id, offset, length): return_obj['data'] = newdata - except json.decoder.JSONDecodeError as e: + except json.decoder.JSONDecodeError as err: return_obj['success'] = False + except Exception as err: + return_obj['success'] = False + self.logger.error('Response decode error: {}'.format(err)) elif response_code > 200 and response_code <= 204:#empty results error_obj['code'] = 2000 @@ -52,19 +50,20 @@ def create_results_connection(self, search_id, offset, length): error_obj['code'] = 4010 elif response_code >= 402 and response_code <= 499:#All other client side errors error_obj['code'] = 4020 - else:#unknown errors + else: + #unknown errors error_obj['code'] = 7000 if error_obj: error_msg = "" try: error_msg = str(response_txt.decode("utf-8")) - except: - pass + except Exception as err: + self.logger.error('Response decode error: {}'.format(err)) error_obj['message'] = error_msg ErrorResponder.fill_error(return_obj, error_obj, ['message']) - err = 'error when getting search results: {}:{}'.format(str(response_code),error_msg) + err = 'error when getting search results: {}:{}'.format(str(response_code), error_msg) self.logger.error(err) # raise NoResultsFoundError(err) return return_obj diff --git a/stix_shifter_modules/proofpoint/stix_transmission/status_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/status_connector.py new file mode 100644 index 000000000..d52a8f9a0 --- /dev/null +++ b/stix_shifter_modules/proofpoint/stix_transmission/status_connector.py @@ -0,0 +1,5 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector + +class StatusConnector(BaseSyncConnector): + def __init__(self, api_client): + self.api_client = api_client \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py index 7e5b984c9..cdb55c843 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py @@ -16,7 +16,7 @@ } options = {} -event_data=[{ +event_data={"queryEndTime" : "2021-09-14T13:30:00Z", "clicksPermitted" : [], "clicksBlocked" : [], "messagesDelivered" :[{ "is_multipart": True, "GUID": "Ggfsdfsdf", @@ -103,7 +103,7 @@ "threatUrl": "https://threatinsight.proofpoint.com" } ] - }] + }]} class TestProofpointResultsToStix(unittest.TestCase): """ class to perform unit test case for proofpoint translate results diff --git a/stix_shifter_modules/proofpoint/tests/test_async_dummy.py b/stix_shifter_modules/proofpoint/tests/test_async_dummy.py deleted file mode 100755 index a62d4ca9d..000000000 --- a/stix_shifter_modules/proofpoint/tests/test_async_dummy.py +++ /dev/null @@ -1,57 +0,0 @@ -from stix_shifter_modules.async_dummy.entry_point import EntryPoint -from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status -import unittest - - -class TestAsyncDummyConnection(unittest.TestCase, object): - - def connection(self): - return { - "host": "hostbla", - "port": 8080, - } - - def configuration(self): - return { - "auth": { - "username": "u", - "password": "p" - } - } - - def test_dummy_async_query(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query = "placeholder query text" - query_response = entry_point.create_query_connection(query) - - assert query_response['search_id'] == "uuid_1234567890" - - def test_dummy_async_status(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query_id = "uuid_1234567890" - status_response = entry_point.create_status_connection(query_id) - - success = status_response["success"] - assert success - status = status_response["status"] - assert status == Status.COMPLETED.value - - def test_dummy_async_results(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query_id = "uuid_1234567890" - results_response = entry_point.create_results_connection(query_id, 1, 1) - - success = results_response["success"] - assert success - data = results_response["data"] - assert data == "Results from search" - - def test_is_async(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - check_async = entry_point.is_async() - assert check_async - - def test_ping(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - ping_result = entry_point.ping_connection() - assert ping_result["success"] is True From f40212b7789757b0d88d256660001eb296c87445 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 19 Oct 2021 18:53:52 +0530 Subject: [PATCH 05/30] Updated code for unit tests in tests folder --- .../stix_translation/query_constructor.py | 2 +- .../test_proofpoint_json_to_stix.py | 58 +++++++++++++++++-- .../test_proofpoint_stix_to_query.py | 32 +++++++--- 3 files changed, 78 insertions(+), 14 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py b/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py index 4c41b4b59..08e3f88bf 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py +++ b/stix_shifter_modules/proofpoint/stix_translation/query_constructor.py @@ -182,7 +182,7 @@ def _parse_expression(self, expression, qualifier=None) -> str: query_string = "{}{}{}".format(expression_01, operator, expression_02) if qualifier is not None: qualifier = self._parse_time_range(qualifier) - return "{}{}".format(query_string, qualifier) + return "{}{}{}".format(query_string, param_delimiter, qualifier) else: return "{}".format(query_string) elif isinstance(expression, ObservationExpression): diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py index cdb55c843..46d9bdc6e 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py @@ -2,8 +2,12 @@ from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator from stix_shifter_modules.proofpoint.entry_point import EntryPoint from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers +from stix_shifter.stix_translation import stix_translation import json from stix_shifter.stix_translation import stix_translation +from stix_shifter_utils.utils.error_response import ErrorCode + +translation = stix_translation.StixTranslation() MODULE = "proofpoint" entry_point = EntryPoint() @@ -104,17 +108,59 @@ } ] }]} + +def _test_query_assertions(query, queries): + assert query['queries'] == [queries] + + class TestProofpointResultsToStix(unittest.TestCase): """ class to perform unit test case for proofpoint translate results """ - def test_custom_mapping(self): - # data_source_string = json.dumps(data_source) - # data_string = json.dumps(message_data) - # translation = stix_translation.StixTranslation() - # result_bundle = translation.translate('proofpoint', 'results', data_source_string, data_string, options) + @staticmethod + def get_first(itr, constraint): + return next( + (obj for obj in itr if constraint(obj)), + None + ) + + @staticmethod + def get_first_of_type(itr, typ): + return TestProofpointResultsToStix.get_first(itr, lambda o: type(o) == dict and o.get('type') == typ) + + def test_common_mapping(self): result_bundle = entry_point.translate_results(json.dumps(data_source), json.dumps([event_data])) + assert (result_bundle['type'] == 'bundle') + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert (result_bundle_identity['type'] == data_source['type']) + assert (result_bundle_identity['id'] == data_source['id']) + assert (result_bundle_identity['name'] == data_source['name']) + assert (result_bundle_identity['identity_class'] + == data_source['identity_class']) + + observed_data = result_bundle_objects[1] + + assert (observed_data['id'] is not None) + assert (observed_data['type'] == "observed-data") + assert (observed_data['created_by_ref'] == result_bundle_identity['id']) - print('result_bundle :', result_bundle) \ No newline at end of file + assert (observed_data['number_observed'] == 5) + assert (observed_data['created'] is not None) + assert (observed_data['modified'] is not None) + assert (observed_data['first_observed'] is not None) + assert (observed_data['last_observed'] is not None) + + def test_custom_mapping(self): + result_bundle = entry_point.translate_results(json.dumps(data_source), json.dumps([event_data])) + result_bundle_objects = result_bundle['objects'] + observed_data = result_bundle_objects[1] + assert ('objects' in observed_data) + objects = observed_data['objects'] + curr_obj = TestProofpointResultsToStix.get_first_of_type(objects.values(), 'ipv4-addr') + assert (curr_obj is not None), 'ipv4-addr object type not found' + assert (curr_obj.keys() == {'type', 'value'}) + assert (curr_obj['value'] == "127.0.0.1") \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py index 19c8b6598..518c146f2 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py @@ -7,6 +7,7 @@ import re +MODULE = "proofpoint" translation = stix_translation.StixTranslation() def _test_query_assertions(translated_query, test_query): @@ -77,10 +78,27 @@ def test_ping(self): class TestStixtoQuery(unittest.TestCase, object): - def test_query(self): - # stix_pattern = "[x-proofpoint:threatstatus = 'active' OR x-proofpoint:threatstatus = 'positive' OR x-proofpoint:threatstatuss = 'falsepositive'] START t'2021-09-15T16:13:00.00Z' STOP t'2021-09-15T17:13:00.00Z'" - stix_pattern = "[x-proofpoint:threatstatus = 'active'] START t'2021-08-22T07:24:00.000Z' STOP t'2022-08-22T08:20:00.000Z'" - translated_query = translation.translate('proofpoint', 'query', '{}', stix_pattern) - translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries']) - test_query = ['threatStatus=active&interval=2021-08-22T07:24:00.000Z/2022-08-22T08:20:00.000Z'] - _test_query_assertions(translated_query, test_query) + def test_url_params_query(self): + stix_pattern = "[x-proofpoint:threatstatus = 'active'] START t'2021-09-29T06:00:00.00Z' STOP t'2021-09-29T06:30:00.00Z'" + query = translation.translate(MODULE, 'query', '{}', stix_pattern) + queries = ["threatStatus=active&interval=2021-09-29T06:00:00.00Z/2021-09-29T06:30:00.00Z"] + _test_query_assertions(query, queries) + + def test_default_timerange_query(self): + stix_pattern = "[x-proofpoint:threatstatus = 'active']" + query = translation.translate(MODULE, 'query', '{}', stix_pattern) + queries = ["threatStatus=active&interval=2021-09-29T06:00:00.00Z/2021-09-29T06:30:00.00Z"] + _test_query_assertions(query, queries) + + def test_query_from_multiple_comparison_expressions(self): + stix_pattern = "[x-proofpoint:threatstatus = 'active' AND x-proofpoint:threatstatus = 'cleared'] START t'2021-09-29T06:00:00.00Z' STOP t'2021-09-29T06:30:00.00Z'" + query = translation.translate(MODULE, 'query', '{}', stix_pattern) + queries = ["threatStatus=cleared&threatStatus=active&interval=2021-09-29T06:00:00.00Z/2021-09-29T06:30:00.00Z"] + _test_query_assertions(query, queries) + + def test_query_unmapped_attribute(self): + stix_pattern = "[x-proofpoint:threatstatus = 'active' AND x-proofpoint:threatstatus = 'cleared' AND unmapped:attribute = 'something']" + query = translation.translate(MODULE, 'query', '{}', stix_pattern) + assert query['success'] == False + assert ErrorCode.TRANSLATION_MAPPING_ERROR.value == query['code'] + assert 'Unable to map the following STIX objects and properties to data source fields' in query['error'] From 77a96b95449f51baa808ac825247904a904e9af1 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Wed, 20 Oct 2021 15:23:55 +0530 Subject: [PATCH 06/30] Added README.md for Proofpoint module --- stix_shifter_modules/proofpoint/README.md | 184 ++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 stix_shifter_modules/proofpoint/README.md diff --git a/stix_shifter_modules/proofpoint/README.md b/stix_shifter_modules/proofpoint/README.md new file mode 100644 index 000000000..aa9c6a814 --- /dev/null +++ b/stix_shifter_modules/proofpoint/README.md @@ -0,0 +1,184 @@ +# Proofpoint + +REST Web Service APIs: https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API + +### Format for making STIX translation calls via the CLI + +`python3 main.py ` + +This example input pattern: + +'python3 main.py translate "proofpoint" query '{}' "[x-proofpoint:threatstatus = 'active' AND x-proofpoint:threatstatus = 'cleared'] START t'2021-09-29T06:00:00.00Z' STOP t'2021-09-29T06:30:00.00Z'"' + +will return +``` +{ + "queries": [ + "threatStatus=cleared&threatStatus=activeinterval=2021-09-29T06:00:00.00Z/2021-09-29T06:30:00.00Z" + ] +} +``` +## Converting from Proofpoint events STIX + +Proofpoint data to STIX mapping is defined in `to_stix_map.json` + +This example Proofpoint data: + +python3 main.py transmit proofpoint '{"host":""}' '{"auth":{"principal":"","secret":""}}' results "threatStatus=falsePositive&threatStatus=active&threatStatus=cleared&interval=2021-10-06T09:00:00Z/2021-10-06T10:00:00Z=" 1 2 + +Will return the following STIX observable: + { + "success": true, + "data": [ + { + "spamScore": 0, + "phishScore": 0, + "threatsInfoMap": [ + { + "threatID": "f5faf4e8cc8617d40fa2062bc53adb476d66c90ed85ccdfad56dbf6a913617cc", + "threatStatus": "cleared", + "classification": "malware", + "threatUrl": "https://threatinsight.proofpoint.com/a7929edb-9295-4c75-8767-8a8ddf8a5807/threat/email/f5faf4e8cc8617d40fa2062bc53adb476d66c90ed85ccdfad56dbf6a913617cc", + "threatTime": "2021-09-29T07:30:46.000Z", + "threat": "http://theteflacademy.co.uk", + "campaignID": null, + "threatType": "url" + } + ], + "messageTime": "2021-10-06T09:30:39.000Z", + "impostorScore": 0.0, + "malwareScore": 0, + "cluster": "hcltechnologies_hosted", + "subject": null, + "quarantineFolder": null, + "quarantineRule": null, + "policyRoutes": [ + "default_inbound" + ], + "modulesRun": [ + "smtpsrv", + "av", + "zerohour", + "spf", + "dkimv", + "sandbox", + "banner", + "dmarc", + "pdr", + "urldefense" + ], + "messageSize": 4305, + "headerFrom": "deepak das ", + "headerReplyTo": null, + "fromAddress": [ + "ab90das@gmail.com" + ], + "ccAddresses": [], + "replyToAddress": [], + "toAddresses": [], + "xmailer": null, + "messageParts": [ + { + "disposition": "inline", + "sha256": "eb4f9cc9dd8e84166f4e711f0196e87063daa379f8a6a9daddef700536a887d3", + "md5": "98180d02ca5eb2d4646928543b53e8ae", + "filename": "text.txt", + "sandboxStatus": null, + "oContentType": "text/plain", + "contentType": "text/plain" + }, + { + "disposition": "inline", + "sha256": "0a7f71ca3e9d1203e2857e51ae0d099b976e724828496592e86ce6fb083f7f0c", + "md5": "e4ad4da3a624b996a1c04a95a0e6588c", + "filename": "text.html", + "sandboxStatus": null, + "oContentType": "text/html", + "contentType": "text/html" + } + ], + "completelyRewritten": true, + "id": "5569b4b2-f0cc-eab7-bf8d-e54d5dcbcd01", + "QID": "1969UFnu029601", + "GUID": "ZNMbDB0FdlMIB8xRaL1ytpktHOGb9bb6", + "sender": "ab90das@gmail.com", + "recipient": [ + "deepak-sd@presentfortesting.com" + ], + "senderIP": "209.85.208.46", + "messageID": "" + } + ] + } + +python3 main.py execute proofpoint proofpoint '{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "proofpoint", "identity_class": "system"}' '{"host":""}' '{"auth":{"principal":"","secret":""}}' "[x-proofpoint:threatstatus = 'active'] START t'2021-09-29T06:00:00.00Z' STOP t'2021-09-29T06:30:00.00Z'" + + + +```json +{ + "type": "bundle", + "id": "bundle--65b8b0e8-3e97-4749-bc42-75b98842e32b", + "spec_version": "2.0", + "objects": [ + { + "id": "identity--d06281b8-b746-447b-bc22-15eaf23dee91", + "name": "proofpoint_async", + "type": "identity", + "identity_class": "system", + "created": "2021-06-29T03:27:48.694Z", + "modified": "2021-06-29T03:27:48.694Z" + }, + { + "id": "observed-data--495f50ac-b43f-4e9f-952c-38c88d2b8cf9", + "type": "observed-data", + "created_by_ref": "identity--d06281b8-b746-447b-bc22-15eaf23dee91", + "created": "2021-10-04T06:39:15.034Z", + "modified": "2021-10-04T06:39:15.034Z", + "objects": { + "0": { + "type": "url", + "value": "https://www.mozilla.org/en-US/firefox" + }, + "1": { + "type": "x-proofpoint-msgevents", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", + "ID": "448cceab-52ce-4638-9ad2-843c9b038c7b", + "senderIP": "7", + "GUID": "IEXA9uKty7y0XtTtXfEGlXdg1pZcutRU" + }, + "2": { + "type": "ipv4-addr", + "value": "34.239.248.48" + }, + "3": { + "type": "network-traffic", + "src_ref": "2" + }, + "4": { + "type": "email-addr", + "value": "0100017c3019caeb-8d083606-04bf-41ff-9d24-85682bff3328-000000@amazonses.com" + }, + "5": { + "type": "email-message", + "sender_ref": "4", + "to_refs": "6" + }, + "6": { + "type": "email-addr", + "value": "subashuser@iscgalaxy.com" + }, + "7": { + "type": "ipv4-addr", + "value": "54.240.11.32" + } + }, + "first_observed": "2021-10-04T06:39:15.034Z", + "last_observed": "2021-10-04T06:39:15.034Z", + "number_observed": 1, + "num ber_observed": 1 + } + ] +} + +``` From c212bff21ef9f5c84038b7a793b6d4430003782e Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 22 Oct 2021 14:51:05 +0530 Subject: [PATCH 07/30] Updated changes for validator errors --- .../stix_translation/json/to_stix_map.json | 97 ++++++++++--------- .../stix_translation/transformers.py | 15 ++- 2 files changed, 66 insertions(+), 46 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json b/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json index 15dc3af4c..42babf22a 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json @@ -2,7 +2,7 @@ "GUID": [ { - "key": "x-proofpoint-msgevents.GUID", + "key": "x-proofpoint-msgevents.guid", "object": "msgevents" } ], @@ -11,7 +11,8 @@ "ccAddresses": [ { "key": "email-addr.value", - "object": "email-addr" + "object": "email-addr", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.cc_refs", @@ -22,7 +23,8 @@ "fromAddress": [ { "key": "email-addr.value", - "object": "from_ref" + "object": "from_ref", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.from_ref", @@ -33,7 +35,8 @@ "headerFrom": [ { "key": "email-addr.value", - "object": "email-addr" + "object": "email-addr", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.from_ref", @@ -44,7 +47,8 @@ "headerReplyTo": [ { "key": "email-addr.value", - "object": "email-addr" + "object": "email-addr", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.from_ref", @@ -55,23 +59,25 @@ "replyToAddress": [ { "key": "email-addr.value", - "object": "email-addr" + "object": "email-addr", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.from_ref", "object": "email-message", - "references": "email-addr" + "references": ["email-addr"] } ], "toAddresses": [ { "key": "email-addr.value", - "object": "to_refs" + "object": "to_refs", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.to_refs", "object": "email-message", - "references": "to_refs" + "references": ["to_refs"] } ], "xmailer": [ @@ -90,41 +96,34 @@ ], "completelyRewritten": [ { - "key": "x-proofpoint-msgevents.Rewritten-Status", - "object": "msgevents" - } - ], - "id": [ - { - "key": "x-proofpoint-msgevents.ID", + "key": "x-proofpoint-msgevents.rewrittenstatus", "object": "msgevents" } ], + "impostorScore": [ { - "key": "x-proofpoint-msgevents.impostorScore", + "key": "x-proofpoint-msgevents.impostorscore", "transformer": "ToInteger", "object": "msgevents" } ], "malwareScore": [ { - "key": "x-proofpoint-msgevents.malwareScore", + "key": "x-proofpoint-msgevents.malwarescore", "transformer": "ToInteger", "object": "msgevents" } ], "messageID": [ { - "key": "x-proofpoint-msgevents.messageID", + "key": "x-proofpoint-msgevents.messageid", "object": "msgevents" } ], "messageParts": { "key": "email-message.body_multipart", "object": "email-message", - - "is_multipart": true, "filename": [ { "key": "file.name", @@ -146,6 +145,7 @@ ] } , + "messageSize": [ { "key": "x-proofpoint-msgevents.size", @@ -161,50 +161,52 @@ ], "modulesRun": [ { - "key": "x-proofpoint-msgevents.modulesRun", + "key": "x-proofpoint-msgevents.modulesrun", "object": "msgevents" } ], "phishScore": [ { - "key": "x-proofpoint-msgevents.phishScore", + "key": "x-proofpoint-msgevents.phishscore", "transformer": "ToInteger", "object": "msgevents" } ], "policyRoutes": [ { - "key": "x-proofpoint-msgevents.policyRoutes", + "key": "x-proofpoint-msgevents.policyroutes", "object": "msgevents" } ], "quarantineFolder": [ { - "key": "x-proofpoint-msgevents.quarantineFolder", + "key": "x-proofpoint-msgevents.quarantinefolder", "object": "msgevents" } ], "quarantineRule": [ { - "key": "x-proofpoint-msgevents.quarantineRule", + "key": "x-proofpoint-msgevents.quarantinerule", "object": "msgevents" } ], "recipient": [ { "key": "email-addr.value", - "object": "to_refs" + "object": "to_refs", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.to_refs", "object": "email-message", - "references": "to_refs" + "references": ["to_refs"] } ], "sender": [ { "key": "email-addr.value", - "object": "sender_ref" + "object": "sender_ref", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.sender_ref", @@ -227,14 +229,14 @@ "references": "src_ip" }, { - "key": "x-proofpoint-msgevents.senderIP", + "key": "x-proofpoint-msgevents.senderip", "object": "msgevents", "references": "src_ip" } ], "spamScore": [ { - "key": "x-proofpoint-msgevents.spamScore", + "key": "x-proofpoint-msgevents.spamscore", "transformer": "ToInteger", "object": "msgevents" } @@ -246,6 +248,11 @@ } ], + "is_multipart": [{ + "key": "email-message.is_multipart", + "object": "email-message" + }], + "clickIP": [ { "key": "ipv4-addr.value", @@ -265,30 +272,32 @@ "headerTo": [ { "key": "email-addr.value", - "object": "to_refs" + "object": "to_refs", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.to_refs", "object": "email-message", - "references": "to_refs" + "references": ["to_refs"] } ], "headerCC": [ { "key": "email-addr.value", - "object": "email-addr" + "object": "email-addr", + "transformer": "proofpoint_emailid_transformer" }, { "key": "email-message.cc_refs", "object": "email-message", - "references": "email-addr" + "references": ["email-addr"] } ], "threatsInfoMap": { "campaignID": [ { - "key": "x-proofpoint-msgevents.campaignID", + "key": "x-proofpoint-msgevents.campaignid", "object": "msgevents" } ], @@ -306,25 +315,25 @@ ], "threatID": [ { - "key": "x-proofpoint-msgevents.threatID", + "key": "x-proofpoint-msgevents.threatid", "object": "msgevents" } ], "threatStatus": [ { - "key": "x-proofpoint-msgevents.threatStatus", + "key": "x-proofpoint-msgevents.threatstatus", "object": "msgevents" } ], "threatTime": [ { - "key": "x-proofpoint-msgevents.threatTime", + "key": "x-proofpoint-msgevents.threattime", "object": "msgevents" } ], "threatType": [ { - "key": "x-proofpoint-msgevents.threatType", + "key": "x-proofpoint-msgevents.threattype", "object": "msgevents" } ], @@ -337,19 +346,19 @@ }, "clickTime": [ { - "key": "x-proofpoint-msgevents.clickTime", + "key": "x-proofpoint-msgevents.clicktime", "object": "msgevents" } ], "QID": [ { - "key": "x-proofpoint-msgevents.ID", + "key": "x-proofpoint-msgevents.qid", "object": "msgevents" } ], "clusterId": [ { - "key": "x-proofpoint-msgevents.clusterId", + "key": "x-proofpoint-msgevents.clusterid", "object": "msgevents" } ], @@ -361,7 +370,7 @@ ], "userAgent": [ { - "key": "x-proofpoint-msgevents.userAgent", + "key": "x-proofpoint-msgevents.useragent", "object": "msgevents" } ] diff --git a/stix_shifter_modules/proofpoint/stix_translation/transformers.py b/stix_shifter_modules/proofpoint/stix_translation/transformers.py index 8fdcab99b..bebc1d8a5 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/transformers.py +++ b/stix_shifter_modules/proofpoint/stix_translation/transformers.py @@ -3,7 +3,7 @@ LOGGER = logger.set_logger(__name__) -class prrofpoint_bodymultipart_transformer(ValueTransformer): +class proofpoint_bodymultipart_transformer(ValueTransformer): @staticmethod def transform(multipart): @@ -12,4 +12,15 @@ def transform(multipart): part['content_type']=part.pop('contentType') part['content_disposition'] = part.pop('disposition') - return multipart \ No newline at end of file + return multipart + +class proofpoint_emailid_transformer(ValueTransformer): + @staticmethod + def transform(emailid): + if isinstance(emailid, list): emailid = emailid[0] + if "<" in emailid and ">" in emailid: + startindex = emailid.index('<') + 1 + endindex = emailid.index('>') + emailid = emailid[startindex: endindex] + + return emailid \ No newline at end of file From cb3a2d92e953182f94f2deaf49a48d82fdd4ce38 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Mon, 25 Oct 2021 16:27:33 +0530 Subject: [PATCH 08/30] Updated changes for validator errors and README --- stix_shifter_modules/proofpoint/README.md | 10 +++++----- .../proofpoint/stix_translation/json/to_stix_map.json | 1 + .../proofpoint/stix_translation/transformers.py | 2 +- .../proofpoint/stix_transmission/results_connector.py | 1 + 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/stix_shifter_modules/proofpoint/README.md b/stix_shifter_modules/proofpoint/README.md index aa9c6a814..ca1a00c06 100644 --- a/stix_shifter_modules/proofpoint/README.md +++ b/stix_shifter_modules/proofpoint/README.md @@ -68,10 +68,10 @@ Will return the following STIX observable: "urldefense" ], "messageSize": 4305, - "headerFrom": "deepak das ", + "headerFrom": "header ", "headerReplyTo": null, "fromAddress": [ - "ab90das@gmail.com" + "from@gmail.com" ], "ccAddresses": [], "replyToAddress": [], @@ -101,9 +101,9 @@ Will return the following STIX observable: "id": "5569b4b2-f0cc-eab7-bf8d-e54d5dcbcd01", "QID": "1969UFnu029601", "GUID": "ZNMbDB0FdlMIB8xRaL1ytpktHOGb9bb6", - "sender": "ab90das@gmail.com", + "sender": "xxx@gmail.com", "recipient": [ - "deepak-sd@presentfortesting.com" + "xxx@presentfortesting.com" ], "senderIP": "209.85.208.46", "messageID": "" @@ -166,7 +166,7 @@ python3 main.py execute proofpoint proofpoint '{"type": "identity", "id": "ident }, "6": { "type": "email-addr", - "value": "subashuser@iscgalaxy.com" + "value": "xxx@abc.com" }, "7": { "type": "ipv4-addr", diff --git a/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json b/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json index 42babf22a..b3d08b2ef 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/proofpoint/stix_translation/json/to_stix_map.json @@ -124,6 +124,7 @@ "messageParts": { "key": "email-message.body_multipart", "object": "email-message", + "transformer": "proofpoint_bodymultipart_transformer", "filename": [ { "key": "file.name", diff --git a/stix_shifter_modules/proofpoint/stix_translation/transformers.py b/stix_shifter_modules/proofpoint/stix_translation/transformers.py index bebc1d8a5..70b852629 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/transformers.py +++ b/stix_shifter_modules/proofpoint/stix_translation/transformers.py @@ -10,7 +10,7 @@ def transform(multipart): # print("transformer multipart :", multipart) for part in multipart: part['content_type']=part.pop('contentType') - part['content_disposition'] = part.pop('disposition') + # part['content_disposition'] = part.pop('disposition') return multipart diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index 26a5985eb..5cee6d9fc 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -9,6 +9,7 @@ def __init__(self, api_client): self.api_client = api_client self.logger = logger.set_logger(__name__) + def create_results_connection(self, search_id, offset, length): try: min_range = offset From a03ab964e596720aae6cbfae9a5f47132543f569 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Mon, 25 Oct 2021 16:37:23 +0530 Subject: [PATCH 09/30] Updated changes for transmission module unit tests --- .../proofpoint/stix_transmission/api_client.py | 2 +- .../test_proofpoint_stix_to_query.py | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/api_client.py b/stix_shifter_modules/proofpoint/stix_transmission/api_client.py index cd88a76f4..65fcf54e3 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/api_client.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/api_client.py @@ -20,7 +20,7 @@ def __init__(self, connection, configuration): def ping_data_source(self): # Pings the data source - endpoint = ENDPOINT_ALL + "?format=syslog&sinceSeconds=3600" + endpoint = ENDPOINT_ALL + "?format=json&sinceSeconds=3600" pingresult = self.client.call_api(endpoint=endpoint, method='GET') return pingresult diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py index 518c146f2..dd670640d 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py @@ -21,7 +21,7 @@ def _remove_timestamp_from_query(queries): return re.sub(pattern, "", queries) -class TestAsyncDummyConnection(unittest.TestCase, object): +class TestProofpointConnection(unittest.TestCase, object): def connection(self): return { @@ -37,14 +37,14 @@ def configuration(self): } } - def test_dummy_async_query(self): + def test_proofpoint_query(self): entry_point = EntryPoint(self.connection(), self.configuration()) query = "placeholder query text" query_response = entry_point.create_query_connection(query) assert query_response['search_id'] == "uuid_1234567890" - def test_dummy_async_status(self): + def test_proofpoint_status(self): entry_point = EntryPoint(self.connection(), self.configuration()) query_id = "uuid_1234567890" status_response = entry_point.create_status_connection(query_id) @@ -54,17 +54,13 @@ def test_dummy_async_status(self): status = status_response["status"] assert status == Status.COMPLETED.value - def test_dummy_async_results(self): + def test_proofpoint_results(self): entry_point = EntryPoint(self.connection(), self.configuration()) - # query_id = "uuid_1234567890" - query_id = 'all?format=syslog&sinceSeconds=3600' + query_id = 'sinceSeconds=3600' results_response = entry_point.create_results_connection(query_id, 1, 1) - print('results_response :', results_response) success = results_response["success"] assert success - data = results_response["data"] - # assert data == "Results from search" def test_is_async(self): entry_point = EntryPoint(self.connection(), self.configuration()) From 472fdfac6034f29c53e90c655dc2498426c70019 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Mon, 25 Oct 2021 18:13:08 +0530 Subject: [PATCH 10/30] Code changes after pytest run on unittests files --- .../stix_transmission/ping_connector.py | 9 ++++++++- .../stix_transmission/results_connector.py | 5 +++++ .../test_proofpoint_json_to_stix.py | 16 ++++++++-------- .../test_proofpoint_stix_to_query.py | 18 +++++++----------- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py index 5a72dd5d4..070975f4b 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py @@ -13,10 +13,17 @@ def ping_connection(self): response_code = response_dict.code # Construct a response object return_obj = dict() + error_obj = dict() if response_code >= 200 and response_code <= 204: return_obj['success'] = True else: - ErrorResponder.fill_error(return_obj, str(response_code)+ ":"+str(response_dict.read().decode("utf-8")), ['message']) + error_msg = "" + try: + error_msg = str(response_dict.read().decode("utf-8")) + except Exception as err: + self.logger.error('Response decode error: {}'.format(err)) + error_obj['message'] = error_msg + ErrorResponder.fill_error(return_obj,error_obj,['message']) return return_obj except Exception as err: self.logger.error('error when pinging datasource {}:'.format(err)) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index 5cee6d9fc..bd4d0b242 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -35,6 +35,11 @@ def create_results_connection(self, search_id, offset, length): if newdata and max_range > 0 and len(newdata) > max_range: newdata = newdata[:max_range] + for msg in newdata: + if "messageParts" in msg: + msg["is_multipart"] = True + else: msg["is_multipart"] = False + return_obj['data'] = newdata except json.decoder.JSONDecodeError as err: diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py index 46d9bdc6e..c9f85f1cf 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py @@ -20,7 +20,7 @@ } options = {} -event_data={"queryEndTime" : "2021-09-14T13:30:00Z", "clicksPermitted" : [], "clicksBlocked" : [], "messagesDelivered" :[{ +event_data={ "is_multipart": True, "GUID": "Ggfsdfsdf", @@ -107,7 +107,7 @@ "threatUrl": "https://threatinsight.proofpoint.com" } ] - }]} + } def _test_query_assertions(query, queries): assert query['queries'] == [queries] @@ -131,7 +131,7 @@ def get_first_of_type(itr, typ): def test_common_mapping(self): - result_bundle = entry_point.translate_results(json.dumps(data_source), json.dumps([event_data])) + result_bundle = entry_point.translate_results(json.dumps(data_source), json.dumps(event_data)) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] @@ -148,7 +148,7 @@ def test_common_mapping(self): assert (observed_data['type'] == "observed-data") assert (observed_data['created_by_ref'] == result_bundle_identity['id']) - assert (observed_data['number_observed'] == 5) + assert (observed_data['number_observed'] == 1) assert (observed_data['created'] is not None) assert (observed_data['modified'] is not None) assert (observed_data['first_observed'] is not None) @@ -160,7 +160,7 @@ def test_custom_mapping(self): observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] - curr_obj = TestProofpointResultsToStix.get_first_of_type(objects.values(), 'ipv4-addr') - assert (curr_obj is not None), 'ipv4-addr object type not found' - assert (curr_obj.keys() == {'type', 'value'}) - assert (curr_obj['value'] == "127.0.0.1") \ No newline at end of file + curr_obj = TestProofpointResultsToStix.get_first_of_type(objects.values(), 'email-message') + assert (curr_obj is not None), 'email-message object type not found' + assert ("type" in curr_obj.keys()) + assert ("date" in curr_obj.keys()) \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py index dd670640d..2a26221d1 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py @@ -25,15 +25,15 @@ class TestProofpointConnection(unittest.TestCase, object): def connection(self): return { - "host": "hostbla", + "host": "tap-api-v2.proofpoint.com/v2/siem", "port": 8080, } def configuration(self): return { "auth": { - "username": "u", - "password": "p" + "principal": "96dbfb1e-e6bf-f298-3080-3cab5b1cfe19", + "secret": "1ee4776401dc372e608cb9ae4810cec5420c2689ff38befa1ac46bd17da20904" } } @@ -42,11 +42,11 @@ def test_proofpoint_query(self): query = "placeholder query text" query_response = entry_point.create_query_connection(query) - assert query_response['search_id'] == "uuid_1234567890" + assert query_response['search_id'] == query def test_proofpoint_status(self): entry_point = EntryPoint(self.connection(), self.configuration()) - query_id = "uuid_1234567890" + query_id = "placeholder query text" status_response = entry_point.create_status_connection(query_id) success = status_response["success"] @@ -62,11 +62,6 @@ def test_proofpoint_results(self): success = results_response["success"] assert success - def test_is_async(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - check_async = entry_point.is_async() - assert check_async - def test_ping(self): entry_point = EntryPoint(self.connection(), self.configuration()) ping_result = entry_point.ping_connection() @@ -84,7 +79,8 @@ def test_default_timerange_query(self): stix_pattern = "[x-proofpoint:threatstatus = 'active']" query = translation.translate(MODULE, 'query', '{}', stix_pattern) queries = ["threatStatus=active&interval=2021-09-29T06:00:00.00Z/2021-09-29T06:30:00.00Z"] - _test_query_assertions(query, queries) + assert "threatStatus=active" in queries[0] + assert "interval" in queries[0] def test_query_from_multiple_comparison_expressions(self): stix_pattern = "[x-proofpoint:threatstatus = 'active' AND x-proofpoint:threatstatus = 'cleared'] START t'2021-09-29T06:00:00.00Z' STOP t'2021-09-29T06:30:00.00Z'" From 080c5df437a475606337fa1c0ab29d321bb3e894 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Mon, 25 Oct 2021 18:20:48 +0530 Subject: [PATCH 11/30] Line correction --- stix_shifter_modules/proofpoint/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stix_shifter_modules/proofpoint/README.md b/stix_shifter_modules/proofpoint/README.md index ca1a00c06..aaa9c8ada 100644 --- a/stix_shifter_modules/proofpoint/README.md +++ b/stix_shifter_modules/proofpoint/README.md @@ -175,8 +175,7 @@ python3 main.py execute proofpoint proofpoint '{"type": "identity", "id": "ident }, "first_observed": "2021-10-04T06:39:15.034Z", "last_observed": "2021-10-04T06:39:15.034Z", - "number_observed": 1, - "num ber_observed": 1 + "number_observed": 1 } ] } From 851cfa9b000d33c20f844600bb97be4b8b3d8fc8 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Mon, 25 Oct 2021 18:23:53 +0530 Subject: [PATCH 12/30] Removed principal and secret from unittest file --- .../tests/stix_translation/test_proofpoint_stix_to_query.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py index 2a26221d1..bb1ecc7e2 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py @@ -25,15 +25,15 @@ class TestProofpointConnection(unittest.TestCase, object): def connection(self): return { - "host": "tap-api-v2.proofpoint.com/v2/siem", + "host": "host", "port": 8080, } def configuration(self): return { "auth": { - "principal": "96dbfb1e-e6bf-f298-3080-3cab5b1cfe19", - "secret": "1ee4776401dc372e608cb9ae4810cec5420c2689ff38befa1ac46bd17da20904" + "principal": "principal", + "secret": "secret" } } From 85ed5f2eca5cb34eaec718202b76ae04f33ef3a8 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 26 Oct 2021 13:04:15 +0530 Subject: [PATCH 13/30] decode error resolved --- .../proofpoint/stix_transmission/ping_connector.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py index 070975f4b..0ed15de31 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/ping_connector.py @@ -1,6 +1,8 @@ from stix_shifter_utils.modules.base.stix_transmission.base_ping_connector import BasePingConnector from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger +import re +import string class PingConnector(BasePingConnector): def __init__(self, api_client): @@ -19,11 +21,14 @@ def ping_connection(self): else: error_msg = "" try: - error_msg = str(response_dict.read().decode("utf-8")) + valid_characters = string.printable + error_msg = str(response_dict.read().decode("utf-8", errors='ignore')) + error_msg = ''.join(i for i in error_msg if i in valid_characters) + except Exception as err: self.logger.error('Response decode error: {}'.format(err)) error_obj['message'] = error_msg - ErrorResponder.fill_error(return_obj,error_obj,['message']) + ErrorResponder.fill_error(return_obj,error_obj,'message') return return_obj except Exception as err: self.logger.error('error when pinging datasource {}:'.format(err)) From e37008fce23af72f78561aeb46a5b134ad148955 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 26 Oct 2021 13:04:43 +0530 Subject: [PATCH 14/30] decode error resolved hostname added for pytest --- .../tests/stix_translation/test_proofpoint_stix_to_query.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py index bb1ecc7e2..de12fc84c 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py @@ -25,15 +25,15 @@ class TestProofpointConnection(unittest.TestCase, object): def connection(self): return { - "host": "host", + "host": "tap-api-v2.proofpoint.com", "port": 8080, } def configuration(self): return { "auth": { - "principal": "principal", - "secret": "secret" + "principal": "96dbfb1e-e6bf-f298-3080-3cab5b1cfe19", + "secret": "1ee4776401dc372e608cb9ae4810cec5420c2689ff38befa1ac46bd17da20904" } } From 5c3255db9ba48e2539d33189909f96b5460b2b76 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 26 Oct 2021 13:59:23 +0530 Subject: [PATCH 15/30] Separated transmission test module --- .../test_proofpoint_json_to_stix.py | 4 -- .../test_proofpoint_stix_to_query.py | 49 ------------------ .../stix_transmission/test_proofpoint.py | 50 +++++++++++++++++++ 3 files changed, 50 insertions(+), 53 deletions(-) create mode 100644 stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py index c9f85f1cf..18da1a7ba 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_json_to_stix.py @@ -1,11 +1,7 @@ import unittest -from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator from stix_shifter_modules.proofpoint.entry_point import EntryPoint -from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers -from stix_shifter.stix_translation import stix_translation import json from stix_shifter.stix_translation import stix_translation -from stix_shifter_utils.utils.error_response import ErrorCode translation = stix_translation.StixTranslation() diff --git a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py index de12fc84c..7051918a7 100755 --- a/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py +++ b/stix_shifter_modules/proofpoint/tests/stix_translation/test_proofpoint_stix_to_query.py @@ -1,9 +1,6 @@ -from stix_shifter_modules.proofpoint.entry_point import EntryPoint -from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status from stix_shifter.stix_translation import stix_translation from stix_shifter_utils.utils.error_response import ErrorCode import unittest -import datetime import re @@ -21,52 +18,6 @@ def _remove_timestamp_from_query(queries): return re.sub(pattern, "", queries) -class TestProofpointConnection(unittest.TestCase, object): - - def connection(self): - return { - "host": "tap-api-v2.proofpoint.com", - "port": 8080, - } - - def configuration(self): - return { - "auth": { - "principal": "96dbfb1e-e6bf-f298-3080-3cab5b1cfe19", - "secret": "1ee4776401dc372e608cb9ae4810cec5420c2689ff38befa1ac46bd17da20904" - } - } - - def test_proofpoint_query(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query = "placeholder query text" - query_response = entry_point.create_query_connection(query) - - assert query_response['search_id'] == query - - def test_proofpoint_status(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query_id = "placeholder query text" - status_response = entry_point.create_status_connection(query_id) - - success = status_response["success"] - assert success - status = status_response["status"] - assert status == Status.COMPLETED.value - - def test_proofpoint_results(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query_id = 'sinceSeconds=3600' - results_response = entry_point.create_results_connection(query_id, 1, 1) - - success = results_response["success"] - assert success - - def test_ping(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - ping_result = entry_point.ping_connection() - assert ping_result["success"] is True - class TestStixtoQuery(unittest.TestCase, object): def test_url_params_query(self): diff --git a/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py b/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py new file mode 100644 index 000000000..5a1fcb2cf --- /dev/null +++ b/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py @@ -0,0 +1,50 @@ +from stix_shifter_modules.proofpoint.entry_point import EntryPoint +from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status +import unittest + + +class TestProofpointConnection(unittest.TestCase, object): + + def connection(self): + return { + "host": "tap-api-v2.proofpoint.com", + "port": 8080, + } + + def configuration(self): + return { + "auth": { + "principal": "96dbfb1e-e6bf-f298-3080-3cab5b1cfe19", + "secret": "1ee4776401dc372e608cb9ae4810cec5420c2689ff38befa1ac46bd17da20904" + } + } + + def test_proofpoint_query(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + query = "placeholder query text" + query_response = entry_point.create_query_connection(query) + + assert query_response['search_id'] == query + + def test_proofpoint_status(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + query_id = "placeholder query text" + status_response = entry_point.create_status_connection(query_id) + + success = status_response["success"] + assert success + status = status_response["status"] + assert status == Status.COMPLETED.value + + def test_proofpoint_results(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + query_id = 'sinceSeconds=3600' + results_response = entry_point.create_results_connection(query_id, 1, 1) + + success = results_response["success"] + assert success + + def test_ping(self): + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_result = entry_point.ping_connection() + assert ping_result["success"] is True \ No newline at end of file From 80c98053dcb6bd7098377dccc07ce08e37b3e7c5 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 26 Oct 2021 19:34:00 +0530 Subject: [PATCH 16/30] Removed credentials --- .../proofpoint/tests/stix_transmission/test_proofpoint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py b/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py index 5a1fcb2cf..5c2109680 100644 --- a/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py +++ b/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py @@ -7,15 +7,15 @@ class TestProofpointConnection(unittest.TestCase, object): def connection(self): return { - "host": "tap-api-v2.proofpoint.com", + "host": "host", "port": 8080, } def configuration(self): return { "auth": { - "principal": "96dbfb1e-e6bf-f298-3080-3cab5b1cfe19", - "secret": "1ee4776401dc372e608cb9ae4810cec5420c2689ff38befa1ac46bd17da20904" + "principal": "principal", + "secret": "secret" } } From adf400d93a0232912d16fe1bc17846b06daf5b8c Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Wed, 27 Oct 2021 15:09:39 +0530 Subject: [PATCH 17/30] Added 'body' field --- .../proofpoint/stix_translation/transformers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stix_shifter_modules/proofpoint/stix_translation/transformers.py b/stix_shifter_modules/proofpoint/stix_translation/transformers.py index 70b852629..1aead9ab3 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/transformers.py +++ b/stix_shifter_modules/proofpoint/stix_translation/transformers.py @@ -10,7 +10,13 @@ def transform(multipart): # print("transformer multipart :", multipart) for part in multipart: part['content_type']=part.pop('contentType') - # part['content_disposition'] = part.pop('disposition') + part['content_disposition'] = part.pop('disposition') + if part['content_type'] == 'text/plain': + part['body'] = part['filename'][:-4] + if 'filename' in part: + del part['filename'] + if 'md5' in part: del part['md5'] + if 'sha256' in part: del part['sha256'] return multipart From 8e60a6c5c3df4aca9cd857abaf0752797bf4c356 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Wed, 27 Oct 2021 18:10:19 +0530 Subject: [PATCH 18/30] Added 'body' field Updated unittest transmission file --- .../stix_translation/transformers.py | 6 +- .../api_response/proofpoint_result_empty.json | 1 + .../api_response/proofpoint_results.json | 1 + .../stix_transmission/test_proofpoint.py | 203 +++++++++++++++--- 4 files changed, 172 insertions(+), 39 deletions(-) create mode 100644 stix_shifter_modules/proofpoint/tests/stix_transmission/api_response/proofpoint_result_empty.json create mode 100644 stix_shifter_modules/proofpoint/tests/stix_transmission/api_response/proofpoint_results.json diff --git a/stix_shifter_modules/proofpoint/stix_translation/transformers.py b/stix_shifter_modules/proofpoint/stix_translation/transformers.py index 1aead9ab3..bd3391945 100755 --- a/stix_shifter_modules/proofpoint/stix_translation/transformers.py +++ b/stix_shifter_modules/proofpoint/stix_translation/transformers.py @@ -6,8 +6,6 @@ class proofpoint_bodymultipart_transformer(ValueTransformer): @staticmethod def transform(multipart): - - # print("transformer multipart :", multipart) for part in multipart: part['content_type']=part.pop('contentType') part['content_disposition'] = part.pop('disposition') @@ -15,8 +13,8 @@ def transform(multipart): part['body'] = part['filename'][:-4] if 'filename' in part: del part['filename'] - if 'md5' in part: del part['md5'] - if 'sha256' in part: del part['sha256'] + if 'md5' in part: del part['md5'] + if 'sha256' in part: del part['sha256'] return multipart diff --git a/stix_shifter_modules/proofpoint/tests/stix_transmission/api_response/proofpoint_result_empty.json b/stix_shifter_modules/proofpoint/tests/stix_transmission/api_response/proofpoint_result_empty.json new file mode 100644 index 000000000..845d652c4 --- /dev/null +++ b/stix_shifter_modules/proofpoint/tests/stix_transmission/api_response/proofpoint_result_empty.json @@ -0,0 +1 @@ +{"queryEndTime" : "2021-09-14T13:30:00Z", "clicksPermitted" : [], "clicksBlocked" : [], "messagesDelivered" :[]} \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/tests/stix_transmission/api_response/proofpoint_results.json b/stix_shifter_modules/proofpoint/tests/stix_transmission/api_response/proofpoint_results.json new file mode 100644 index 000000000..73b8ce053 --- /dev/null +++ b/stix_shifter_modules/proofpoint/tests/stix_transmission/api_response/proofpoint_results.json @@ -0,0 +1 @@ +{"queryEndTime" : "2021-09-14T13:30:00Z", "clicksPermitted" : [], "clicksBlocked" : [], "messagesDelivered" :[{"is_multipart": true, "GUID": "Ggfsdfsdf", "Header": {"ccAddresses": [], "fromAddress": ["Header@xxx.com"], "headerFrom": "\"j.\" ", "headerReplyTo": null, "replyToAddress": [], "toAddresses": [], "xmailer": null}, "cluster": "hosted", "completelyRewritten": true, "id": "1828003vsdv05566e842", "impostorScore": 0, "malwareScore": 0, "messageID": "", "messageParts": [{"contentType": "text/html", "disposition": "inline", "filename": "text.html", "md5": "fcfa9b21f43fbdf02965263c63e", "oContentType": "text/html", "sandboxStatus": "None", "sha256": "72d3dc7a01dfbdbe8e871536864f56bf235ba08ff259105ac"}, {"contentType": "text/html", "disposition": "inline", "filename": "text2.html", "md5": "fcfa9b21f43fbdf02965263c63e", "oContentType": "text/html", "sandboxStatus": null, "sha256": "72d3dc7a01dfbdbe8e871536864f56bf235ba08ff259105bd"}], "messageSize": 10171, "messageTime": "2021-06-02T13:41:32.000Z", "modulesRun": ["av", "spf", "dkimv", "spam", "dmarc", "urldefense"], "phishScore": 0, "policyRoutes": ["default_inbound", "allow_relay"], "quarantineFolder": null, "quarantineRule": null, "recipient": ["recipient@xxx.com"], "sender": "sender@xxx.com", "senderIP": "400.000.000", "spamScore": 43, "subject": "=", "threatsInfoMap": [{"campaignID": null, "classification": "phish", "threat": "https://bit.ly", "threatID": "45fe3b35ghkk2b8916934b6c0a536cc9b2603d03", "threatStatus": "active", "threatTime": "2021-06-03T07:17:11.000Z", "threatType": "url", "threatUrl": "https://threatinsight.proofpoint.com"}, {"campaignID": null, "classification": "phish", "threat": "https://bit123.ly", "threatID": "45fe3b35ghkk2b8916934b6c0a536cc9b2603d04", "threatStatus": "active", "threatTime": "2021-06-04T07:17:11.000Z", "threatType": "url", "threatUrl": "https://threatinsight.proofpoint.com"}]}]} \ No newline at end of file diff --git a/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py b/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py index 5c2109680..0e4e99ade 100644 --- a/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py +++ b/stix_shifter_modules/proofpoint/tests/stix_transmission/test_proofpoint.py @@ -1,50 +1,183 @@ from stix_shifter_modules.proofpoint.entry_point import EntryPoint from stix_shifter_utils.modules.base.stix_transmission.base_status_connector import Status import unittest +from unittest.mock import patch +import json +from stix_shifter.stix_transmission import stix_transmission +from stix_shifter_utils.utils.error_response import ErrorCode +import os -class TestProofpointConnection(unittest.TestCase, object): +CONNECTION= { + "host": "host", + "port": 8080, + } - def connection(self): - return { - "host": "host", - "port": 8080, - } - def configuration(self): - return { - "auth": { - "principal": "principal", - "secret": "secret" - } +CONFIG = { + "auth": { + "principal": "principal", + "secret": "secret" } + } + +searchid = "sinceSeconds=3600" +query_mock = "?format=json&interval=PT30M/2016-05-01T12:30:00Z&threatStatus=falsePositive&threatStatus=active&threatStatus=cleared" + +class ProofpointMockResponse: + def __init__(self, response_code, obj): + self.code = response_code + self.object = obj + + def read(self): + return bytearray(self.object, 'utf-8') + +@patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.__init__') +class TestProofpointConnection(unittest.TestCase, object): + + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_endpoint(self, mock_ping_response, mock_api_client): + mock_api_client.return_value = None + mocked_return_value = '["mock", "placeholder"]' + mock_ping_response.return_value = ProofpointMockResponse(200, mocked_return_value) + + transmission = stix_transmission.StixTransmission('proofpoint', CONNECTION, CONFIG) + ping_response = transmission.ping() + + assert ping_response is not None + assert ping_response['success'] + + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.ping_data_source') + def test_ping_endpoint_exception(self, mock_ping_response, mock_api_client): + mock_api_client.return_value = None + mocked_return_value = '["mock", "placeholder"]' + mock_ping_response.return_value = ProofpointMockResponse(200, mocked_return_value) + mock_ping_response.side_effect = Exception('exception') + + transmission = stix_transmission.StixTransmission('proofpoint', CONNECTION, CONFIG) + ping_response = transmission.ping() + + assert ping_response is not None + assert ping_response['success'] is False + assert ping_response['code'] == ErrorCode.TRANSMISSION_UNKNOWN.value + + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.get_search_status', autospec=True) + def test_status_response(self, mock_status_response, mock_api_client): + mock_api_client.return_value = None + response = {"success": True, "status": "COMPLETED", "progress": 100} + mock_status_response.return_value = ProofpointMockResponse(200, json.dumps(response)) + + entry_point = EntryPoint(CONNECTION, CONFIG) + status_response = entry_point.create_status_connection(searchid) + + assert status_response is not None + assert 'status' in status_response + assert status_response['status'] == 'COMPLETED' + assert 'progress' in status_response + assert status_response['progress'] == 100 + assert 'success' in status_response + assert status_response['success'] is True + + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_results_response(self, mock_results_response, mock_api_client): + mock_api_client.return_value = None + + dir_path = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(dir_path, 'api_response', 'proofpoint_results.json') + mocked_return_value = open(file_path, 'r').read() + + mock_results_response.return_value = ProofpointMockResponse(200, mocked_return_value) + offset = 0 + length = 1 + + transmission = stix_transmission.StixTransmission('proofpoint', CONNECTION, CONFIG) + results_response = transmission.results(query_mock, offset, length) + + assert 'success' in results_response + assert results_response['success'] is True + assert 'data' in results_response + assert len(results_response['data']) > 0 + + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_results_response_empty(self, mock_results_response, mock_api_client): + mock_api_client.return_value = None + + dir_path = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(dir_path, 'api_response', 'proofpoint_result_empty.json') + mocked_return_value = open(file_path, 'r').read() + + mock_results_response.return_value = ProofpointMockResponse(200, mocked_return_value) + offset = 0 + length = 1 + + transmission = stix_transmission.StixTransmission('proofpoint', CONNECTION, CONFIG) + results_response = transmission.results(searchid, offset, length) + + assert 'success' in results_response + assert results_response['success'] is True + assert 'data' in results_response + assert len(results_response['data']) == 0 + + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_results_response_exception(self, mock_results_response, mock_api_client): + mock_api_client.return_value = None + + dir_path = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(dir_path, 'api_response', 'proofpoint_results.json') + mocked_return_value = open(file_path, 'r').read() + + mock_results_response.return_value = ProofpointMockResponse(200, mocked_return_value) + mock_results_response.side_effect = Exception('exception') + offset = 0 + length = 1 + + transmission = stix_transmission.StixTransmission('proofpoint', CONNECTION, CONFIG) + results_response = transmission.results(searchid, offset, length) + + assert 'success' in results_response + assert results_response['success'] is False + assert results_response['code'] == ErrorCode.TRANSMISSION_UNKNOWN.value + + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.create_search', autospec=True) + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.get_search_status', autospec=True) + @patch('stix_shifter_modules.proofpoint.stix_transmission.api_client.APIClient.get_search_results', autospec=True) + def test_query_flow(self, mock_results_response, mock_status_response, mock_query_response, mock_api_client): + mock_api_client.return_value = None + + mock_query_response.return_value = ProofpointMockResponse(201, query_mock) + + dir_path = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(dir_path, 'api_response', 'proofpoint_results.json') + results_mock = open(file_path, 'r').read() + mock_results_response.return_value = ProofpointMockResponse(200, results_mock) + + status_mock = {"success": True, "status": "COMPLETED", "progress": 100} + mock_status_response.return_value = ProofpointMockResponse(200, status_mock) - def test_proofpoint_query(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query = "placeholder query text" - query_response = entry_point.create_query_connection(query) + entry_point = EntryPoint(CONNECTION, CONFIG) + query_response = entry_point.create_query_connection(searchid) - assert query_response['search_id'] == query + assert query_response is not None + assert query_response['success'] is True + assert 'search_id' in query_response + assert query_response['search_id'] == searchid - def test_proofpoint_status(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query_id = "placeholder query text" - status_response = entry_point.create_status_connection(query_id) + status_response = entry_point.create_status_connection(query_mock) - success = status_response["success"] - assert success - status = status_response["status"] - assert status == Status.COMPLETED.value + assert status_response is not None + assert 'status' in status_response + assert status_response['status'] == 'COMPLETED' + assert 'progress' in status_response + assert status_response['progress'] == 100 + assert 'success' in status_response + assert status_response['success'] is True - def test_proofpoint_results(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - query_id = 'sinceSeconds=3600' - results_response = entry_point.create_results_connection(query_id, 1, 1) - success = results_response["success"] - assert success + offset = 0 + length = 1 + results_response = entry_point.create_results_connection(searchid, offset, length) - def test_ping(self): - entry_point = EntryPoint(self.connection(), self.configuration()) - ping_result = entry_point.ping_connection() - assert ping_result["success"] is True \ No newline at end of file + assert 'success' in results_response + assert results_response['success'] is True + assert 'data' in results_response + assert len(results_response['data']) > 0 \ No newline at end of file From 00c80adda8363c896de10e4d6da5159e5c51409b Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Thu, 18 Nov 2021 14:41:57 +0530 Subject: [PATCH 19/30] Updated code for data type of offset and length --- .../proofpoint/stix_transmission/results_connector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index bd4d0b242..3f5f798cc 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -12,13 +12,15 @@ def __init__(self, api_client): def create_results_connection(self, search_id, offset, length): try: - min_range = offset - max_range = offset + length + min_range = int(offset) - 1 + max_range = min_range + int(length) + # Grab the response, extract the response code, and convert it to readable json response = self.api_client.get_search_results(search_id) response_code = response.code response_txt = response.read() # Construct a response object + # print("response code={}, text={}".format(response_code, response_txt)) return_obj = dict() error_obj = dict() if response_code == 200: @@ -31,9 +33,9 @@ def create_results_connection(self, search_id, offset, length): if isinstance(value, list) and value: newdata+=value - # slice of the data count according to offset values + # slice off the data count according to offset values if newdata and max_range > 0 and len(newdata) > max_range: - newdata = newdata[:max_range] + newdata = newdata[min_range:max_range] for msg in newdata: if "messageParts" in msg: From f97fec426a4a2dd9d69672f292784dea467f7ee1 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Thu, 18 Nov 2021 15:28:05 +0530 Subject: [PATCH 20/30] Updated code for data type of offset and length --- .../proofpoint/stix_transmission/results_connector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index 3f5f798cc..7859835a5 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -12,7 +12,7 @@ def __init__(self, api_client): def create_results_connection(self, search_id, offset, length): try: - min_range = int(offset) - 1 + min_range = int(offset) max_range = min_range + int(length) # Grab the response, extract the response code, and convert it to readable json @@ -42,6 +42,7 @@ def create_results_connection(self, search_id, offset, length): msg["is_multipart"] = True else: msg["is_multipart"] = False + print("newdata :", newdata) return_obj['data'] = newdata except json.decoder.JSONDecodeError as err: From 3a440a19c79aa8bdb540ce10398b9701457e527e Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 19 Nov 2021 09:23:52 +0530 Subject: [PATCH 21/30] Updated code for data type of offset and length --- .../proofpoint/stix_transmission/results_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index 7859835a5..9e6a03e72 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -34,7 +34,7 @@ def create_results_connection(self, search_id, offset, length): newdata+=value # slice off the data count according to offset values - if newdata and max_range > 0 and len(newdata) > max_range: + if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range: newdata = newdata[min_range:max_range] for msg in newdata: From 897a910dc178cbc011447317c096c5ee9b990c86 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 19 Nov 2021 21:42:45 +0530 Subject: [PATCH 22/30] Revert "Updated code for data type of offset and length" This reverts commit 3a440a19 --- .../stix_transmission/results_connector.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index bd4d0b242..9e6a03e72 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -12,13 +12,15 @@ def __init__(self, api_client): def create_results_connection(self, search_id, offset, length): try: - min_range = offset - max_range = offset + length + min_range = int(offset) + max_range = min_range + int(length) + # Grab the response, extract the response code, and convert it to readable json response = self.api_client.get_search_results(search_id) response_code = response.code response_txt = response.read() # Construct a response object + # print("response code={}, text={}".format(response_code, response_txt)) return_obj = dict() error_obj = dict() if response_code == 200: @@ -31,15 +33,16 @@ def create_results_connection(self, search_id, offset, length): if isinstance(value, list) and value: newdata+=value - # slice of the data count according to offset values - if newdata and max_range > 0 and len(newdata) > max_range: - newdata = newdata[:max_range] + # slice off the data count according to offset values + if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range: + newdata = newdata[min_range:max_range] for msg in newdata: if "messageParts" in msg: msg["is_multipart"] = True else: msg["is_multipart"] = False + print("newdata :", newdata) return_obj['data'] = newdata except json.decoder.JSONDecodeError as err: From 98f1b735fc0dbcf49f7c4927ccf985589d9e284f Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 19 Nov 2021 21:53:25 +0530 Subject: [PATCH 23/30] Revert "Revert "Updated code for data type of offset and length"" This reverts commit 897a910dc178cbc011447317c096c5ee9b990c86. --- .../stix_transmission/results_connector.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index 9e6a03e72..bd4d0b242 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -12,15 +12,13 @@ def __init__(self, api_client): def create_results_connection(self, search_id, offset, length): try: - min_range = int(offset) - max_range = min_range + int(length) - + min_range = offset + max_range = offset + length # Grab the response, extract the response code, and convert it to readable json response = self.api_client.get_search_results(search_id) response_code = response.code response_txt = response.read() # Construct a response object - # print("response code={}, text={}".format(response_code, response_txt)) return_obj = dict() error_obj = dict() if response_code == 200: @@ -33,16 +31,15 @@ def create_results_connection(self, search_id, offset, length): if isinstance(value, list) and value: newdata+=value - # slice off the data count according to offset values - if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range: - newdata = newdata[min_range:max_range] + # slice of the data count according to offset values + if newdata and max_range > 0 and len(newdata) > max_range: + newdata = newdata[:max_range] for msg in newdata: if "messageParts" in msg: msg["is_multipart"] = True else: msg["is_multipart"] = False - print("newdata :", newdata) return_obj['data'] = newdata except json.decoder.JSONDecodeError as err: From 8cbd5e0ecd5cd448197c325950941c357bb2f635 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 19 Nov 2021 22:00:48 +0530 Subject: [PATCH 24/30] Revert "Revert "Revert "Updated code for data type of offset and length""" This reverts commit 98f1b735fc0dbcf49f7c4927ccf985589d9e284f. --- .../stix_transmission/results_connector.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index bd4d0b242..9e6a03e72 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -12,13 +12,15 @@ def __init__(self, api_client): def create_results_connection(self, search_id, offset, length): try: - min_range = offset - max_range = offset + length + min_range = int(offset) + max_range = min_range + int(length) + # Grab the response, extract the response code, and convert it to readable json response = self.api_client.get_search_results(search_id) response_code = response.code response_txt = response.read() # Construct a response object + # print("response code={}, text={}".format(response_code, response_txt)) return_obj = dict() error_obj = dict() if response_code == 200: @@ -31,15 +33,16 @@ def create_results_connection(self, search_id, offset, length): if isinstance(value, list) and value: newdata+=value - # slice of the data count according to offset values - if newdata and max_range > 0 and len(newdata) > max_range: - newdata = newdata[:max_range] + # slice off the data count according to offset values + if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range: + newdata = newdata[min_range:max_range] for msg in newdata: if "messageParts" in msg: msg["is_multipart"] = True else: msg["is_multipart"] = False + print("newdata :", newdata) return_obj['data'] = newdata except json.decoder.JSONDecodeError as err: From b18fe3f13d696609a5d177c495a0e3375b344ab2 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 19 Nov 2021 22:02:40 +0530 Subject: [PATCH 25/30] Updated code for data type of offset and length --- .../proofpoint/stix_transmission/results_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py index 9e6a03e72..7859835a5 100755 --- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py @@ -34,7 +34,7 @@ def create_results_connection(self, search_id, offset, length): newdata+=value # slice off the data count according to offset values - if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range: + if newdata and max_range > 0 and len(newdata) > max_range: newdata = newdata[min_range:max_range] for msg in newdata: From 987feee497bc8642527a3a477bb5b9e009898262 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 26 Nov 2021 16:40:26 +0530 Subject: [PATCH 26/30] Resolved merge conflicts --- .github/workflows/main.yml | 2 +- .idea/.gitignore | 0 .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 7 + .idea/modules.xml | 8 + .../shelved.patch | 19 + ...mitted_changes_before_rebase__Changes_.xml | 4 + .idea/stix-shifter.iml | 12 + .idea/vcs.xml | 6 + .idea/workspace.xml | 424 +++ OVERVIEW.md | 22 +- README.md | 6 +- .../connectors/datadog_supported_stix.md | 34 +- .../connectors/elastic_ecs_supported_stix.md | 3 +- .../connectors/proofpoint_supported_stix.md | 69 + .../connectors/qradar_supported_stix.md | 17 +- .../connectors/sumologic_supported_stix.md | 31 + adapter-guide/develop-stix-adapter.md | 2 +- adapter-guide/supported-mappings.md | 4 + .../ibm_cloud_pak_for_security/Dockerfile | 2 +- .../ibm_cloud_pak_for_security/README.md | 10 +- .../ibm_cloud_pak_for_security/_build.sh | 13 +- .../ibm_cloud_pak_for_security/deploy.sh | 27 +- stix_shifter/scripts/stix_shifter.py | 23 +- .../scripts/supported_property_exporter.py | 6 +- .../json/guardduty_from_stix_map.json | 3 +- .../json/vpcflow_from_stix_map.json | 3 +- .../stix_translation/query_constructor.py | 8 +- .../stix_translation/json/from_stix_map.json | 4 +- .../json/stix_2_1/from_stix_map.json | 161 ++ .../json/stix_2_1/to_stix_map.json | 595 +++++ .../test_azure_sentinel_stix_to_query.py | 5 +- .../json/stix_2_1/from_stix_map.json | 48 + .../json/stix_2_1/to_stix_map.json | 140 + .../stix_translation/json/from_stix_map.json | 8 +- .../json/stix_2_1/from_stix_map.json | 174 ++ .../json/stix_2_1/to_stix_map.json | 251 ++ .../json/stix_2_1/to_stix_map_events.json | 391 +++ .../stix_translation/json/from_stix_map.json | 4 +- .../json/stix_2_1/from_stix_map.json | 134 + .../json/stix_2_1/to_stix_map.json | 182 ++ .../stix_translation/json/from_stix_map.json | 8 +- .../json/stix_2_1/from_stix_map.json | 125 + .../json/stix_2_1/to_stix_map.json | 389 +++ .../test_datadog_stix_to_query.py | 8 +- .../stix_translation/json/from_stix_map.json | 4 +- .../json/stix_2_1/from_stix_map.json | 545 ++++ .../json/stix_2_1/to_stix_map.json | 2331 +++++++++++++++++ .../stix_translation/json/to_stix_map.json | 3 +- stix_shifter_modules/lang_en.json | 4 + .../stix_translation/json/from_stix_map.json | 14 +- .../json/stix_2_1/from_stix_map.json | 453 ++++ .../json/stix_2_1/to_stix_map.json | 996 +++++++ .../stix_translation/json/from_stix_map.json | 19 +- .../json/stix_2_1/from_stix_map.json | 65 + .../json/stix_2_1/to_stix_map.json | 153 ++ .../stix_translation/json/to_stix_map.json | 74 + .../stix_translation/query_constructor.py | 10 + .../test_mysql_stix_to_query.py | 81 +- .../json/aql_events_fields.json | 6 - .../json/events_from_stix_map.json | 19 +- .../json/stix_2_1/events_from_stix_map.json | 156 ++ .../json/stix_2_1/flows_from_stix_map.json | 92 + .../json/stix_2_1/to_stix_map.json | 885 +++++++ .../stix_translation/json/to_stix_map.json | 100 +- .../stix_transmission/results_connector.py | 5 +- .../json/aql_events_fields.json | 6 - .../json/events_from_stix_map.json | 19 +- .../json/stix_2_1/events_from_stix_map.json | 156 ++ .../json/stix_2_1/to_stix_map.json | 869 ++++++ .../stix_translation/json/to_stix_map.json | 85 +- .../json/stix_2_1/from_stix_map.json | 166 ++ .../json/stix_2_1/to_stix_map.json | 330 +++ .../messageActivityData_from_stix_map.json | 2 +- .../endpointActivityData_from_stix_map.json | 53 + .../messageActivityData_from_stix_map.json | 41 + .../json/stix_2_1/to_stix_map.json | 387 +++ .../stix_translation/base_query_translator.py | 16 +- .../base_results_translator.py | 7 +- .../id_contributing_properties.json | 20 + .../json_to_stix/json_to_stix_translator.py | 36 +- .../src/utils/transformers.py | 2 +- .../test_results_translation.py | 127 +- tests/stix_translation/test_transformers.py | 19 + 84 files changed, 11457 insertions(+), 297 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/shelf/Uncommitted_changes_before_rebase_[Changes]/shelved.patch create mode 100644 .idea/shelf/Uncommitted_changes_before_rebase__Changes_.xml create mode 100644 .idea/stix-shifter.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 adapter-guide/connectors/proofpoint_supported_stix.md create mode 100644 adapter-guide/connectors/sumologic_supported_stix.md create mode 100644 stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/bigfix/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/bigfix/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/to_stix_map_events.json create mode 100644 stix_shifter_modules/cbcloud/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/cbcloud/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/crowdstrike/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/crowdstrike/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/msatp/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/msatp/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/mysql/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/mysql/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/qradar/stix_translation/json/stix_2_1/events_from_stix_map.json create mode 100644 stix_shifter_modules/qradar/stix_translation/json/stix_2_1/flows_from_stix_map.json create mode 100644 stix_shifter_modules/qradar/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/qradar_perf_test/stix_translation/json/stix_2_1/events_from_stix_map.json create mode 100644 stix_shifter_modules/qradar_perf_test/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/splunk/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/splunk/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/endpointActivityData_from_stix_map.json create mode 100644 stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/messageActivityData_from_stix_map.json create mode 100644 stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_utils/stix_translation/src/json_to_stix/id_contributing_properties.json create mode 100644 tests/stix_translation/test_transformers.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 82594ad65..45df1c808 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 000000000..105ce2da2 --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 000000000..21b7dee60 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000..0a6a40f57 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/shelf/Uncommitted_changes_before_rebase_[Changes]/shelved.patch b/.idea/shelf/Uncommitted_changes_before_rebase_[Changes]/shelved.patch new file mode 100644 index 000000000..35fa70b12 --- /dev/null +++ b/.idea/shelf/Uncommitted_changes_before_rebase_[Changes]/shelved.patch @@ -0,0 +1,19 @@ +Index: stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP +<+>import json\n\nfrom stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector\nfrom stix_shifter_utils.utils.error_response import ErrorResponder\nfrom stix_shifter_utils.utils import logger\n\nclass ResultsConnector(BaseResultsConnector):\n def __init__(self, api_client):\n self.api_client = api_client\n self.logger = logger.set_logger(__name__)\n\n\n def create_results_connection(self, search_id, offset, length):\n try:\n min_range = int(offset)\n max_range = min_range + int(length)\n\n # Grab the response, extract the response code, and convert it to readable json\n response = self.api_client.get_search_results(search_id)\n response_code = response.code\n response_txt = response.read()\n # Construct a response object\n # print(\"response code={}, text={}\".format(response_code, response_txt))\n return_obj = dict()\n error_obj = dict()\n if response_code == 200:\n return_obj['success'] = True\n try:\n response_txt = response.read().decode('utf-8')\n data= json.loads(response_txt)\n newdata = list()\n for key, value in data.items():\n if isinstance(value, list) and value:\n newdata+=value\n\n # slice off the data count according to offset values\n if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range:\n newdata = newdata[min_range:max_range]\n\n for msg in newdata:\n if \"messageParts\" in msg:\n msg[\"is_multipart\"] = True\n else: msg[\"is_multipart\"] = False\n\n print(\"newdata :\", newdata)\n return_obj['data'] = newdata\n\n except json.decoder.JSONDecodeError as err:\n return_obj['success'] = False\n except Exception as err:\n return_obj['success'] = False\n self.logger.error('Response decode error: {}'.format(err))\n\n elif response_code > 200 and response_code <= 204:#empty results\n error_obj['code'] = 2000\n elif response_code == 400:#error from data source\n error_obj['code'] = 4000\n elif response_code == 401:#Authentication error\n error_obj['code'] = 4010\n elif response_code >= 402 and response_code <= 499:#All other client side errors\n error_obj['code'] = 4020\n else:\n #unknown errors\n error_obj['code'] = 7000\n if error_obj:\n error_msg = \"\"\n try:\n error_msg = str(response_txt.decode(\"utf-8\"))\n except Exception as err:\n self.logger.error('Response decode error: {}'.format(err))\n error_obj['message'] = error_msg\n ErrorResponder.fill_error(return_obj,\n error_obj,\n ['message'])\n err = 'error when getting search results: {}:{}'.format(str(response_code), error_msg)\n self.logger.error(err)\n # raise NoResultsFoundError(err)\n return return_obj\n except Exception as err:\n self.logger.error('error when getting search results: {}'.format(err))\n raise\n\nclass NoResultsFoundError(Exception):\n pass\n +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py +--- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py (revision ca9282c041bd9ba8581136708abe77290802c1d9) ++++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py (date 1637338725740) +@@ -34,7 +34,7 @@ + newdata+=value + + # slice off the data count according to offset values +- if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range: ++ if newdata and max_range > 0 and len(newdata) > max_range: + newdata = newdata[min_range:max_range] + + for msg in newdata: diff --git a/.idea/shelf/Uncommitted_changes_before_rebase__Changes_.xml b/.idea/shelf/Uncommitted_changes_before_rebase__Changes_.xml new file mode 100644 index 000000000..5bc59623a --- /dev/null +++ b/.idea/shelf/Uncommitted_changes_before_rebase__Changes_.xml @@ -0,0 +1,4 @@ + + \ No newline at end of file diff --git a/.idea/stix-shifter.iml b/.idea/stix-shifter.iml new file mode 100644 index 000000000..4871eb071 --- /dev/null +++ b/.idea/stix-shifter.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000..94a25f7f4 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 000000000..cfd13f936 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,424 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1633433539700 + + + 1633436100542 + + + 1633674880650 + + + 1634021028673 + + + 1634618859315 + + + 1634649832103 + + + 1634723635852 + + + 1634894465227 + + + 1635159453857 + + + 1635160043136 + + + 1635165788579 + + + 1635166248747 + + + 1635166433818 + + + 1635233655540 + + + 1635233683219 + + + 1635236963324 + + + 1635257040425 + + + 1635327579105 + + + 1635338419503 + + + 1637226717777 + + + 1637229485097 + + + 1637294032908 + + + 1637338365309 + + + 1637339560253 + + + 1637832674207 + + + 1637847209146 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/OVERVIEW.md b/OVERVIEW.md index eb009fe96..973bdf402 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -123,7 +123,7 @@ Each connector supports a set of STIX objects and properties as defined in the c Stix-shifter currently offers connector support for the following cybersecurity products. Click on a connector name in the following table to see a list of STIX attributes and properties it supports. -List updated: September 28, 2021 +List updated: October 29, 2021 | Connector | Module Name | Data Model | Developer | Translation | Transmission | Availability | | :------------------------: | :------------------: | :--------: | :----------: | :---------: | :----------: | :----------: | @@ -148,6 +148,10 @@ List updated: September 28, 2021 | [Secret Server](adapter-guide/connectors/secretserver_supported_stix.md) | secretserver | Default | IBM | Yes | Yes | Released | | [One Login](adapter-guide/connectors/onelogin_supported_stix.md) | onelogin | Default | GS Lab | Yes | Yes | Released | | MySQL | mysql | Default | IBM | Yes | Yes | Released | +| [Sumo Logic](adapter-guide/connectors/sumologic_supported_stix.md) | sumologic | Default | GS Lab | Yes | Yes | Released | +| [Datadog](adapter-guide/connectors/datadog_supported_stix.md) | datadog | Default | GS Lab | Yes | Yes | Released | +| [Infoblox BloxOne Threat Defense](adapter-guide/connectors/infoblox_supported_stix.md) | infoblox | Default | Infoblox | Yes | Yes | Released | +| [Proofpoint (SIEM API)](adapter-guide/connectors/proofpoint_supported_stix.md) | proofpoint | Default | IBM Security | Yes | Yes | Released | @@ -502,9 +506,21 @@ The `execute` command tests all steps of the translation-transmission flow: ### Debug -You can add `--debug` option at the end of your CLI command to see more logs. +You can add the `--debug` option to your CLI command to see more logs. -`stix-shifter execute '' '' '' '' --debug` +`stix-shifter --debug execute '' '' '' ''` + +### Change max returned results + +You can add the `--results` option with an integer value at the end of your CLI command to limit the maximum number of returned search results (default 10). + +`stix-shifter execute '' '' '' '' --results 50` + +### Save the STIX results to a file + +You can redirect the output of your CLI command to a file to save the STIX results. + +`stix-shifter execute '' '' '' '' > results.json` ## Modules diff --git a/README.md b/README.md index 2dc2fbecf..cd2d783ac 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ This stix-shifter has the following dependencies: - [antlr4-python3-runtime==4.8](https://pypi.org/project/antlr4-python3-runtime/) - [python-dateutil==2.8.1](https://pypi.org/project/python-dateutil/) -Your development environment must use Python 3 +Your development environment must use Python version: 3.6, 3.7, 3.8 or 3.9 ## Installation @@ -78,11 +78,11 @@ print(response) We are thrilled you are considering contributing! We welcome all contributors. -Please read our [guidelines for contributing](https://github.com/opencybersecurityalliance/stix-shifter/blob/master/CONTRIBUTING.md). +Please read our [guidelines for contributing](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CONTRIBUTING.md). ## Guide for creating new connectors -If you want to create a new connector for STIX-shifter, see the [developer guide](https://github.com/opencybersecurityalliance/stix-shifter/blob/master/adapter-guide/develop-stix-adapter.md) +If you want to create a new connector for STIX-shifter, see the [developer guide](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/develop-stix-adapter.md) ## Licensing diff --git a/adapter-guide/connectors/datadog_supported_stix.md b/adapter-guide/connectors/datadog_supported_stix.md index cf899a21c..7b74ba24f 100644 --- a/adapter-guide/connectors/datadog_supported_stix.md +++ b/adapter-guide/connectors/datadog_supported_stix.md @@ -1,37 +1,35 @@ +##### Updated on 10/29/21 ## Datadog | STIX Object | STIX Property | Data Source Field | |--|--|--| +| artifact | payload_bin | text | |
| | | | domain-name | value | host | |
| | | -| artifact | payload_bin | text | +| process | created_time | start | +| process | command_line | cmdline | +| process | pid | pid | +| process | created_time | timestamp | +| process | creator_user_ref | user | |
| | | | user-account | user_id | user | |
| | | -| process | command_line | cmdline -| process | pid | pid -| process | creator_user_ref | user -| process | created_time | timestamp -| process | created_time | start -| process | parent_ref | ppid -| process | host | host +| x-datadog-event | priority | priority | +| x-datadog-event | monitor_id | monitor_id | +| x-datadog-event | tags | tags | +| x-datadog-event | unaggregated | is_aggregate | +| x-datadog-event | alert_type | alert_type | |
| | | +| x-ibm-finding | time_observed | date_happened | | x-ibm-finding | start | start | | x-ibm-finding | end | end | -| x-ibm-finding | time_observed | date_happened | |
| | | +| x-oca-event | domain_ref | host | +| x-oca-event | original_ref | text | +| x-oca-event | module | source | | x-oca-event | code | id | | x-oca-event | code | id_str | | x-oca-event | outcome | title | -| x-oca-event | module | source | | x-oca-event | agent | device_name | | x-oca-event | created | date_happened | -| x-oca-event | original_ref.payload_bin | text | -| x-oca-event | domain_ref.value | host | -|
| | | -| x-datadog-event | priority | priority | -| x-datadog-event | monitor_id | monitor_id | -| x-datadog-event | tags | tags | -| x-datadog-event | is_aggregate | unaggregated | -| x-datadog-event | alert_type | alert_type | |
| | | diff --git a/adapter-guide/connectors/elastic_ecs_supported_stix.md b/adapter-guide/connectors/elastic_ecs_supported_stix.md index 54c89db59..d8c9c5ada 100644 --- a/adapter-guide/connectors/elastic_ecs_supported_stix.md +++ b/adapter-guide/connectors/elastic_ecs_supported_stix.md @@ -1,4 +1,4 @@ -##### Updated on 09/28/21 +##### Updated on 10/29/21 ## Elasticsearch ECS | STIX Object | STIX Property | Data Source Field | |--|--|--| @@ -80,6 +80,7 @@ | process | parent_ref | pid | | process | creator_user_ref | name | | process | creator_user_ref | id | +| process | x_ttp_tags | tags | |
| | | | software | name | name | | software | vendor | type | diff --git a/adapter-guide/connectors/proofpoint_supported_stix.md b/adapter-guide/connectors/proofpoint_supported_stix.md new file mode 100644 index 000000000..086b5ee59 --- /dev/null +++ b/adapter-guide/connectors/proofpoint_supported_stix.md @@ -0,0 +1,69 @@ +##### Updated on 10/29/21 +## Proofpoint (SIEM API) +| STIX Object | STIX Property | Data Source Field | +|--|--|--| +| email-addr | value | ccAddresses | +| email-addr | value | fromAddress | +| email-addr | value | headerFrom | +| email-addr | value | headerReplyTo | +| email-addr | value | replyToAddress | +| email-addr | value | toAddresses | +| email-addr | value | recipient | +| email-addr | value | sender | +| email-addr | value | headerTo | +| email-addr | value | headerCC | +|
| | | +| email-message | cc_refs | ccAddresses | +| email-message | from_ref | fromAddress | +| email-message | from_ref | headerFrom | +| email-message | from_ref | headerReplyTo | +| email-message | from_ref | replyToAddress | +| email-message | to_refs | toAddresses | +| email-message | body_multipart | messageParts | +| email-message | date | messageTime | +| email-message | to_refs | recipient | +| email-message | sender_ref | sender | +| email-message | subject | subject | +| email-message | is_multipart | is_multipart | +| email-message | to_refs | headerTo | +| email-message | cc_refs | headerCC | +|
| | | +| ipv4-addr | value | senderIP | +| ipv4-addr | value | clickIP | +|
| | | +| ipv6-addr | value | senderIP | +| ipv6-addr | value | clickIP | +|
| | | +| network-traffic | src_ref | senderIP | +| network-traffic | src_ref | clickIP | +|
| | | +| url | value | threatUrl | +| url | value | url | +|
| | | +| x-proofpoint-msgevents | guid | GUID | +| x-proofpoint-msgevents | xmailer | xmailer | +| x-proofpoint-msgevents | cluster | cluster | +| x-proofpoint-msgevents | rewrittenstatus | completelyRewritten | +| x-proofpoint-msgevents | impostorscore | impostorScore | +| x-proofpoint-msgevents | malwarescore | malwareScore | +| x-proofpoint-msgevents | messageid | messageID | +| x-proofpoint-msgevents | size | messageSize | +| x-proofpoint-msgevents | modulesrun | modulesRun | +| x-proofpoint-msgevents | phishscore | phishScore | +| x-proofpoint-msgevents | policyroutes | policyRoutes | +| x-proofpoint-msgevents | quarantinefolder | quarantineFolder | +| x-proofpoint-msgevents | quarantinerule | quarantineRule | +| x-proofpoint-msgevents | senderip | senderIP | +| x-proofpoint-msgevents | spamscore | spamScore | +| x-proofpoint-msgevents | campaignid | campaignID | +| x-proofpoint-msgevents | classification | classification | +| x-proofpoint-msgevents | threat | threat | +| x-proofpoint-msgevents | threatid | threatID | +| x-proofpoint-msgevents | threatstatus | threatStatus | +| x-proofpoint-msgevents | threattime | threatTime | +| x-proofpoint-msgevents | threattype | threatType | +| x-proofpoint-msgevents | clicktime | clickTime | +| x-proofpoint-msgevents | qid | QID | +| x-proofpoint-msgevents | clusterid | clusterId | +| x-proofpoint-msgevents | useragent | userAgent | +|
| | | diff --git a/adapter-guide/connectors/qradar_supported_stix.md b/adapter-guide/connectors/qradar_supported_stix.md index a37adc40f..f8017e26b 100644 --- a/adapter-guide/connectors/qradar_supported_stix.md +++ b/adapter-guide/connectors/qradar_supported_stix.md @@ -1,4 +1,4 @@ -##### Updated on 09/28/21 +##### Updated on 11/09/21 ## IBM QRadar | STIX Object | STIX Property | Data Source Field | |--|--|--| @@ -36,6 +36,10 @@ | file | parent_directory_ref | ParentImage | | file | name | ServiceFileName | | file | parent_directory_ref | ServiceFileName | +| file | name | TargetImage | +| file | parent_directory_ref | TargetImage | +| file | name | SourceImage | +| file | parent_directory_ref | SourceImage | |
| | | | ipv4-addr | value | identityip | | ipv4-addr | value | destinationaddress | @@ -79,6 +83,7 @@ | process | creator_user_ref | username | | process | binary_ref | Image | | process | binary_ref | ParentImage | +| process | binary_ref | TargetImage | | process | parent_ref | ParentImage | | process | command_line | "Process CommandLine" | | process | command_line | ParentCommandLine | @@ -116,16 +121,6 @@ | x-ibm-finding | name | "CRE Name" | | x-ibm-finding | description | "CRE Description" | |
| | | -| x-ibm-windows | targetimage | TargetImage | -| x-ibm-windows | granted_access | "Granted Access" | -| x-ibm-windows | call_trace | "Call Trace" | -| x-ibm-windows | source_image | SourceImage | -| x-ibm-windows | pipe_name | PipeName | -| x-ibm-windows | start_module | StartModule | -| x-ibm-windows | start_function | StartFunction | -| x-ibm-windows | signed | Signed | -| x-ibm-windows | imphash | "IMP Hash" | -|
| | | | x-oca-asset | ip_refs | identityip | | x-oca-asset | hostname | identityhostname | | x-oca-asset | ip_refs | sourceaddress | diff --git a/adapter-guide/connectors/sumologic_supported_stix.md b/adapter-guide/connectors/sumologic_supported_stix.md new file mode 100644 index 000000000..c628abcc9 --- /dev/null +++ b/adapter-guide/connectors/sumologic_supported_stix.md @@ -0,0 +1,31 @@ +##### Updated on 10/29/21 +## Sumo Logic +| STIX Object | STIX Property | Data Source Field | +|--|--|--| +| artifact | payload_bin | _raw | +|
| | | +| domain-name | value | _sourcehost | +|
| | | +| user-account | user_id | id | +| user-account | account_login | email | +| user-account | display_name | displayName | +| user-account | account_created | createdAt | +| user-account | account_last_login | lastLoginTimestamp | +|
| | | +| x-ibm-finding | event_count | _messagecount | +| x-ibm-finding | time_observed | _messagetime | +| x-ibm-finding | src_device | _collector | +| x-ibm-finding | start | _receipttime | +|
| | | +| x-oca-event | original_ref | _raw | +| x-oca-event | domain_ref | _sourcehost | +| x-oca-event | created | _messagetime | +| x-oca-event | code | _messageid | +| x-oca-event | agent | _collector | +| x-oca-event | module | _source | +| x-oca-event | provider | _sourcecategory | +| x-oca-event | user_ref | email | +|
| | | +| x-sumologic-source | collectorid | _collectorid | +| x-sumologic-source | sourcename | _sourcename | +|
| | | diff --git a/adapter-guide/develop-stix-adapter.md b/adapter-guide/develop-stix-adapter.md index 638d35da1..b93e8e4d6 100644 --- a/adapter-guide/develop-stix-adapter.md +++ b/adapter-guide/develop-stix-adapter.md @@ -24,7 +24,7 @@ By implementing a new connector, _Developer A_ allows _Product A_ to fit into th ## Prerequisites -- Your development environment must use Python 3.6. +- Your development environment must use any of the follow Python versions: 3.6, 3.7, 3.8 and 3.9. - You must have access to the target data source. In the sample scenario, you must have access to Product A data source. - You must be familiar with Product A's query language and APIs. - You must be familiar or understand the following concepts: diff --git a/adapter-guide/supported-mappings.md b/adapter-guide/supported-mappings.md index cb95f4ef5..199e839d1 100644 --- a/adapter-guide/supported-mappings.md +++ b/adapter-guide/supported-mappings.md @@ -28,3 +28,7 @@ Stix-shifter currently offers connector support for the following cybersecurity - [Trend Micro Vision One](connectors/trendmicro_vision_one_supported_stix.md) - [OneLogin](connectors/onelogin_supported_stix.md) - [Secret Server](connectors/secretserver_supported_stix.md) +- [Sumo Logic](connectors/sumologic_supported_stix.md) +- [Datadog](connectors/datadog_supported_stix.md) +- [Proofpoint (SIEM API)](connectors/proofpoint_supported_stix.md) +- [Infoblox BloxOne Threat Defense](connectors/infoblox_supported_stix.md) diff --git a/deployment/ibm_cloud_pak_for_security/Dockerfile b/deployment/ibm_cloud_pak_for_security/Dockerfile index c31188f6e..94f9735b3 100644 --- a/deployment/ibm_cloud_pak_for_security/Dockerfile +++ b/deployment/ibm_cloud_pak_for_security/Dockerfile @@ -12,7 +12,7 @@ RUN microdnf update -y && rm -fr /var/cache/yum && \ COPY ./bundle/ /opt/app/ RUN chown -R 1001 /opt/app/ -RUN pip3 install 'cryptography==2.9.2' +RUN pip3 install 'cryptography==3.3.2' RUN pip3 install 'pyopenssl==19.1.0' USER 1001 diff --git a/deployment/ibm_cloud_pak_for_security/README.md b/deployment/ibm_cloud_pak_for_security/README.md index 78520eb3c..e32acced9 100644 --- a/deployment/ibm_cloud_pak_for_security/README.md +++ b/deployment/ibm_cloud_pak_for_security/README.md @@ -34,17 +34,19 @@ Since the primary use-case for these scripts is to install a new or updated conn `cloudctl login -a -u -p -n ` + Note: there is a known issue when logged in as `kubeadmin` user via oc command, `oc login -u kubeadmin` + 9. Run the deployment script based on one of the following scenarios: ### A. Build the connector image and then deploy into your Kubernetes cluster ``` - ./deploy.sh remote + ./deploy.sh remote [-n ] ``` - (Ex: `./deploy.sh elastic_ecs remote`) + (Ex: `./deploy.sh elastic_ecs remote -n cp4s`) ### B. Deploy an existing connector image from a registry into your Kubernetes cluster ``` - ./deploy.sh remote + ./deploy.sh remote [-n ] ``` (Ex: `./deploy.sh elastic_ecs remote docker.io//stix_shifter_modules_CONNECTORNAME:tag`) @@ -54,3 +56,5 @@ Since the primary use-case for these scripts is to install a new or updated conn ``` (Ex: `./deploy.sh elastic_ecs local`) + When making a remote build, [-n < NAMESPACE >] flag is optional. If not supplied, the namespace will be obtained from the current namespace project of the cluster you are logged in. + diff --git a/deployment/ibm_cloud_pak_for_security/_build.sh b/deployment/ibm_cloud_pak_for_security/_build.sh index b8e251b67..c4f42d53a 100755 --- a/deployment/ibm_cloud_pak_for_security/_build.sh +++ b/deployment/ibm_cloud_pak_for_security/_build.sh @@ -11,6 +11,7 @@ validate_cmd () { } IMAGE_URL="$1" +NAMESPACE="$2" validate_cmd openssl validate_cmd python3 @@ -28,7 +29,12 @@ validate_cmd kubectl validate_cmd oc FILE_PREFIX=stix_shifter_modules_ -NAMESPACE=cp4s +if [ "X$NAMESPACE" == "X" ]; then + NAMESPACE=`kubectl config view --minify --output 'jsonpath={..namespace}' | awk '{print $1}'` +fi +if [ "${IMAGE_URL}" == "no-repository" ] ; then + IMAGE_URL="" +fi TIMESTAMP=`date '+%Y%m%d%H%M%S'` if [ ! -z "${IMAGE_URL}" ]; then @@ -156,8 +162,9 @@ if [ -z "${IMAGE_URL}" ]; then fi fi -echo "Logging in into internal registry..." -docker login -u `oc whoami` -p `oc whoami -t` $REPOSITORY +DOCKER_USER=`oc whoami` +echo "Logging in into internal registry $REPOSITORY as $DOCKER_USER ..." +docker login -u $DOCKER_USER -p `oc whoami -t` $REPOSITORY if [ ! -z "${IMAGE_URL}" ]; then echo "Pulling ${IMAGE_URL}" diff --git a/deployment/ibm_cloud_pak_for_security/deploy.sh b/deployment/ibm_cloud_pak_for_security/deploy.sh index 6e4ea4759..ba51a2e95 100755 --- a/deployment/ibm_cloud_pak_for_security/deploy.sh +++ b/deployment/ibm_cloud_pak_for_security/deploy.sh @@ -1,5 +1,28 @@ #!/bin/bash +POSITIONAL=() +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -n) + NAMESPACE="$2" + shift + shift + ;; + -n=*) + NAMESPACE="${1#*=}" + shift + ;; + *) # unknown option + ARGG=$1 + POSITIONAL+=("$ARGG") # save it in an array for later + shift # past argument + ;; + esac +done + +set -- "${POSITIONAL[@]}" # restore positional parameters + cd "$(dirname "$0")" PYTHONIOENCODING='utf8' @@ -57,8 +80,8 @@ if [ $BUILD_LOCATION == "local" ]; then ./_build_local.sh elif [ ! $REPOSITORY == "" ]; then echo "Deploying image from repository: ${REPOSITORY}" - ./_build.sh $REPOSITORY + ./_build.sh $REPOSITORY $NAMESPACE else echo "Building and deploying image" - ./_build.sh + ./_build.sh "no-repository" $NAMESPACE fi diff --git a/stix_shifter/scripts/stix_shifter.py b/stix_shifter/scripts/stix_shifter.py index 21a8b02cd..764b196b5 100644 --- a/stix_shifter/scripts/stix_shifter.py +++ b/stix_shifter/scripts/stix_shifter.py @@ -49,6 +49,8 @@ def main(): # process arguments parent_parser = argparse.ArgumentParser(description='stix_shifter') + parent_parser.add_argument('-d', '--debug', action='store_true', + help='Print detail logs for debugging') parent_subparsers = parent_parser.add_subparsers(dest='command') # translate parser @@ -71,8 +73,6 @@ def main(): # optional arguments translate_parser.add_argument('-x', '--stix-validator', action='store_true', help='Run the STIX 2 validator against the translated results') - translate_parser.add_argument('-d', '--debug', action='store_true', - help='Print detail logs for debugging') # modules parser parent_subparsers.add_parser(MODULES, help='Get modules list') @@ -101,32 +101,25 @@ def main(): type=str, help='Data source authentication' ) - transmit_parser.add_argument('-d', '--debug', action='store_true', - help='Print detail logs for debugging') # operation subparser operation_subparser = transmit_parser.add_subparsers(title="operation", dest="operation_command") operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source") query_operation_parser = operation_subparser.add_parser(stix_transmission.QUERY, help="Executes a query on the data source") query_operation_parser.add_argument('query_string', help='native datasource query string') - query_operation_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') results_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS, help="Fetches the results of the data source query") results_operation_parser.add_argument('search_id', help='uuid of executed query') results_operation_parser.add_argument('offset', help='offset of results') results_operation_parser.add_argument('length', help='length of results') - results_operation_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') resultsstix_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS_STIX, help="Fetches the results of the data source query, response is translated in STIX") resultsstix_operation_parser.add_argument('search_id', help='uuid of executed query') resultsstix_operation_parser.add_argument('offset', help='offset of results') resultsstix_operation_parser.add_argument('length', help='length of results') resultsstix_operation_parser.add_argument('data_source', help='STIX identity object representing a datasource') - resultsstix_operation_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') status_operation_parser = operation_subparser.add_parser(stix_transmission.STATUS, help="Gets the current status of the query") status_operation_parser.add_argument('search_id', help='uuid of executed query') - status_operation_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') delete_operation_parser = operation_subparser.add_parser(stix_transmission.DELETE, help="Delete a running query on the data source") delete_operation_parser.add_argument('search_id', help='id of query to remove') - delete_operation_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') operation_subparser.add_parser(stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous') execute_parser = parent_subparsers.add_parser(EXECUTE, help='Translate and fully execute a query') @@ -159,8 +152,8 @@ def main(): type=str, help='Query String' ) - execute_parser.add_argument('-d', '--debug', action='store_true', - help='Print detail logs for debugging') + execute_parser.add_argument('-r', '--results', type=int, default=10, + help='Maximum number of returned results (default 10)') host_parser = parent_subparsers.add_parser(HOST, help='Host a local query service, for testing and development') host_parser.add_argument( @@ -184,9 +177,6 @@ def main(): help='SSL key filename' ) - host_parser.add_argument('-d', '--debug', action='store_true', - help='Print detail logs for debugging') - args = parent_parser.parse_args() help_and_exit = args.command is None @@ -297,7 +287,7 @@ def is_async(): log.debug(status) else: raise RuntimeError("Fetching status failed") - result = transmission.results(search_id, 0, 9) + result = transmission.results(search_id, 0, args.results - 1) if result["success"]: log.debug("Search {} results is:\n{}".format(search_id, result["data"])) @@ -313,7 +303,8 @@ def is_async(): translation_options = copy.deepcopy(connection_dict.get('options', {})) options['validate_pattern'] = True result = translation.translate(args.module, 'results', args.data_source, json.dumps(results), translation_options) - log.info('STIX Results: \n' + json.dumps(result, indent=4, sort_keys=False)) + log.info('STIX Results (written to stdout):\n') + print(json.dumps(result, indent=4, sort_keys=False)) exit(0) elif args.command == TRANSLATE: diff --git a/stix_shifter/scripts/supported_property_exporter.py b/stix_shifter/scripts/supported_property_exporter.py index 05da62137..443452d50 100644 --- a/stix_shifter/scripts/supported_property_exporter.py +++ b/stix_shifter/scripts/supported_property_exporter.py @@ -27,7 +27,11 @@ "crowdstrike": 'CrowdStrike Falcon', "trendmicro_vision_one": "Trend Micro Vision One", "onelogin": "OneLogin", - "secretserver": "Secret Server" + "secretserver": "Secret Server", + "sumologic": "Sumo Logic", + "datadog": "Datadog", + "proofpoint": "Proofpoint (SIEM API)", + "infoblox": "Infoblox BloxOne Threat Defense" } now = datetime.now() diff --git a/stix_shifter_modules/aws_athena/stix_translation/json/guardduty_from_stix_map.json b/stix_shifter_modules/aws_athena/stix_translation/json/guardduty_from_stix_map.json index dde1b1994..c373c6f2a 100644 --- a/stix_shifter_modules/aws_athena/stix_translation/json/guardduty_from_stix_map.json +++ b/stix_shifter_modules/aws_athena/stix_translation/json/guardduty_from_stix_map.json @@ -73,6 +73,5 @@ "src_os_ref.value": ["platform"], "dst_geolocation": ["country_name"] } - }, - "startstopattr": "updatedat" + } } diff --git a/stix_shifter_modules/aws_athena/stix_translation/json/vpcflow_from_stix_map.json b/stix_shifter_modules/aws_athena/stix_translation/json/vpcflow_from_stix_map.json index 89aed99a4..30312852f 100644 --- a/stix_shifter_modules/aws_athena/stix_translation/json/vpcflow_from_stix_map.json +++ b/stix_shifter_modules/aws_athena/stix_translation/json/vpcflow_from_stix_map.json @@ -35,6 +35,5 @@ "start": ["starttime"], "end": ["endtime"] } - }, - "startstopattr": "starttime" + } } \ No newline at end of file diff --git a/stix_shifter_modules/aws_athena/stix_translation/query_constructor.py b/stix_shifter_modules/aws_athena/stix_translation/query_constructor.py index 39bd85833..6c7e1528e 100644 --- a/stix_shifter_modules/aws_athena/stix_translation/query_constructor.py +++ b/stix_shifter_modules/aws_athena/stix_translation/query_constructor.py @@ -272,8 +272,14 @@ def _parse_time_range(self, qualifier, time_range): [start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z', stop_time.strftime( '%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z']] start_stop_list = [each for each in time_range_list] + + if self.service_type == 'guardduty': + startstopattr = 'updatedat' + elif self.service_type == 'vpcflow': + startstopattr = 'starttime' + qualifier_string = "AND {datetime_field} BETWEEN {starttime} AND " \ - "{stoptime}".format(datetime_field=self.dmm.map_data['startstopattr'], + "{stoptime}".format(datetime_field=startstopattr, starttime=start_stop_list[0], stoptime=start_stop_list[1]) return qualifier_string diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/from_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/from_stix_map.json index b7fed0c39..302d27bce 100644 --- a/stix_shifter_modules/azure_sentinel/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/azure_sentinel/stix_translation/json/from_stix_map.json @@ -1,7 +1,7 @@ { "ipv4-addr": { "fields": { - "value": ["networkConnections.sourceAddress", "networkConnections.destinationAddress", "hostStates.publicIpAddress", "hostStates.privateIpAddress", "userStates.logonIp"] + "value": ["networkConnections.sourceAddress", "networkConnections.destinationAddress"] } }, "ipv6-addr": { @@ -108,6 +108,7 @@ "hostStates.isAzureAadRegistered": ["hostStates.isAzureAadRegistered"], "hostStates.isHybridAzureDomainJoined": ["hostStates.isHybridAzureDomainJoined"], "hostStates.os": ["hostStates.os"], + "hostStates.publicIpAddress": ["hostStates.publicIpAddress"], "hostStates.privateIpAddress": ["hostStates.privateIpAddress"], "hostStates.riskScore": ["hostStates.riskScore"], "id": ["id"], @@ -143,6 +144,7 @@ "triggers.name": ["triggers.name"], "triggers.type": ["triggers.type"], "triggers.value": ["triggers.value"], + "userStates.logonIp": ["userStates.logonIp"], "userStates.aadUserId": ["userStates.aadUserId"], "userStates.emailRole": ["userStates.emailRole"], "userStates.isVpn": ["userStates.isVpn"], diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..1684bffb5 --- /dev/null +++ b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,161 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["networkConnections.sourceAddress", "networkConnections.destinationAddress", "hostStates.publicIpAddress", "hostStates.privateIpAddress", "userStates.logonIp"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["networkConnections.sourceAddress", "networkConnections.destinationAddress"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["networkConnections.sourcePort"], + "dst_port": ["networkConnections.destinationPort"], + "protocols[*]": ["networkConnections.protocol"], + "src_ref.value": ["networkConnections.sourceAddress"], + "dst_ref.value": ["networkConnections.destinationAddress"] + } + }, + "directory": { + "fields": { + "path": ["fileStates.path", "process.path"] + } + }, + "file": { + "fields": { + "parent_directory_ref.path": ["fileStates.path"], + "name": ["fileStates.name"], + "hashes.'SHA-256'": ["fileStates.fileHash.hashValue"], + "hashes.'SHA-1'": ["fileStates.fileHash.hashValue"], + "hashes.MD5": ["fileStates.fileHash.hashValue"], + "hashes.authenticodeHash256": ["fileStates.fileHash.hashValue"], + "hashes.lsHash": ["fileStates.fileHash.hashValue"], + "hashes.ctph": ["fileStates.fileHash.hashValue"], + "hashes.peSha1": ["fileStates.fileHash.hashValue"], + "hashes.peSha256": ["fileStates.fileHash.hashValue"], + "hashes.unknown": ["fileStates.fileHash.hashValue"] + } + }, + "process" : { + "fields": { + "name": ["processes.name", "processes.parentProcessName"], + "parent_ref.name": ["processes.parentProcessName"], + "command_line": ["processes.commandLine"], + "pid": ["processes.processId", "processes.parentProcessId", "registryKeyStates.processId"], + "created_time": ["processes.createdDateTime"], + "parent_ref.pid": ["processes.parentProcessId"], + "image_ref.path": ["processes.path"] + } + }, + "domain-name": { + "fields": { + "value": ["hostStates.fqdn", "hostStates.netBiosName", "networkConnections.destinationDomain", "userStates.domainName"] + } + }, + "user-account": { + "fields": { + "user_id": ["userStates.accountName", "processes.accountName"], + "account_login": ["userStates.logonId"], + "account_type": ["userStates.userAccountType"], + "account_last_login": ["userStates.logonDateTime"] + } + }, + "software": { + "fields": { + "name": ["vendorInformation.provider", "networkConnections.applicationName"], + "vendor": ["vendorInformation.vendor"], + "version": ["vendorInformation.providerVersion"] + } + }, + "url": { + "fields": { + "name": ["networkConnections.destinationUrl"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["registryKeyStates.key"], + "extensions.windows-registry-value-type.valueData": [ "registryKeyStates.valueData" ], + "extensions.windows-registry-value-type.name": [ "registryKeyStates.valueName" ], + "extensions.windows-registry-value-type.valueType": [ "registryKeyStates.valueType" ] + } + }, + "x-msazure-sentinel": { + "fields": { + "tenant_id": ["azureTenantId"], + "subscription_id": ["azureSubscriptionId"] + } + }, + "x-msazure-sentinel-alert": { + "fields": { + "activityGroupName": ["activityGroupName"], + "assignedTo": ["assignedTo"], + "category": ["category"], + "closedDateTime": ["closedDateTime"], + "cloudAppStates.destinationServiceName": ["cloudAppStates.destinationServiceName"], + "cloudAppStates.destinationServiceIp": ["cloudAppStates.destinationServiceIp"], + "cloudAppStates.riskScore": ["cloudAppStates.riskScore"], + "comments": ["comments"], + "confidence": ["confidence"], + "createdDateTime": ["createdDateTime"], + "description": ["description"], + "detectionIds": ["detectionIds"], + "eventDateTime": ["eventDateTime"], + "feedback": ["feedback"], + "hostStates.isAzureAadJoined": ["hostStates.isAzureAadJoined"], + "hostStates.isAzureAadRegistered": ["hostStates.isAzureAadRegistered"], + "hostStates.isHybridAzureDomainJoined": ["hostStates.isHybridAzureDomainJoined"], + "hostStates.os": ["hostStates.os"], + "hostStates.privateIpAddress": ["hostStates.privateIpAddress"], + "hostStates.riskScore": ["hostStates.riskScore"], + "id": ["id"], + "incidentIds": ["incidentIds"], + "lastModifiedDateTime": ["lastModifiedDateTime"], + "malwareStates.category": ["malwareStates.category"], + "malwareStates.family": ["malwareStates.family"], + "malwareStates.name": ["malwareStates.family"], + "malwareStates.severity": ["malwareStates.family"], + "malwareStates.wasRunning": ["malwareStates.family"], + "networkConnections.destinationLocation": ["networkConnections.destinationLocation"], + "networkConnections.direction": ["networkConnections.direction"], + "networkConnections.domainRegisteredDateTime": ["networkConnections.domainRegisteredDateTime"], + "networkConnections.localDnsName": ["networkConnections.localDnsName"], + "networkConnections.natDestinationAddress": ["networkConnections.natDestinationAddress"], + "networkConnections.natDestinationPort": ["networkConnections.natDestinationPort"], + "networkConnections.natSourceAddress": ["networkConnections.natSourceAddress"], + "networkConnections.natSourcePort": ["networkConnections.natSourcePort"], + "networkConnections.riskScore": ["networkConnections.riskScore"], + "networkConnections.sourceLocation": ["networkConnections.sourceLocation"], + "networkConnections.status": ["networkConnections.status"], + "networkConnections.urlParameters": ["networkConnections.urlParameters"], + "processes.integrityLevel": ["processes.integrityLevel"], + "processes.isElevated": ["processes.isElevated"], + "recommendedActions": ["recommendedActions"], + "securityResources.resource": ["securityResources.resource"], + "securityResources.resourceType": ["securityResources.resourceType"], + "severity": ["severity"], + "sourceMaterials": ["sourceMaterials"], + "status": ["status"], + "tags": ["tags"], + "title": ["title"], + "triggers.name": ["triggers.name"], + "triggers.type": ["triggers.type"], + "triggers.value": ["triggers.value"], + "userStates.aadUserId": ["userStates.aadUserId"], + "userStates.emailRole": ["userStates.emailRole"], + "userStates.isVpn": ["userStates.isVpn"], + "userStates.logonLocation": ["userStates.logonLocation"], + "userStates.logonType": ["userStates.logonType"], + "userStates.onPremisesSecurityIdentifier": ["userStates.onPremisesSecurityIdentifier"], + "userStates.riskScore": ["userStates.riskScore"], + "userStates.userAccountType": ["userStates.userAccountType"], + "userStates.userPrincipalName": ["userStates.userPrincipalName"], + "vendorInformation.subProvider": ["vendorInformation.subProvider"], + "vulnerabilityStates.cve": ["vulnerabilityStates.cve"], + "vulnerabilityStates.severity": ["vulnerabilityStates.severity"], + "vulnerabilityStates.wasRunning": ["vulnerabilityStates.wasRunning"] + } + } +} diff --git a/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..9be30c417 --- /dev/null +++ b/stix_shifter_modules/azure_sentinel/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,595 @@ +{ + "eventDateTime": [ + { + "key": "first_observed", + "cybox": false + }, + { + "key": "last_observed", + "cybox": false + } + ], + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "azureTenantId": { + "key": "x-msazure-sentinel.tenant_id", + "object": "sentinel" + }, + "azureSubscriptionId": { + "key": "x-msazure-sentinel.subscription_id", + "object": "sentinel" + }, + "activityGroupName": { + "key": "x-msazure-sentinel-alert.activityGroupName", + "object": "alert" + }, + "assignedTo": { + "key": "x-msazure-sentinel-alert.assignedTo", + "object": "alert" + }, + "category": { + "key": "x-msazure-sentinel-alert.category", + "object": "alert" + }, + "closedDateTime": { + "key": "x-msazure-sentinel-alert.closedDateTime", + "object": "alert" + }, + "cloudAppStates": { + "destinationServiceName": { + "key": "x-msazure-sentinel-alert.cloudAppStates.destinationServiceName", + "object": "alert" + }, + "destinationServiceIp": { + "key": "x-msazure-sentinel-alert.cloudAppStates.destinationServiceIp", + "object": "alert" + }, + "riskScore": { + "key": "x-msazure-sentinel-alert.cloudAppStates.riskScore", + "object": "alert" + } + }, + "comments": { + "key": "x-msazure-sentinel-alert.comments", + "object": "alert", + "transformer": "ToString" + }, + "confidence": { + "key": "x-msazure-sentinel-alert.confidence", + "object": "alert" + }, + "createdDateTime": [ + { + "key": "created", + "cybox": false + }, + { + "key": "x-msazure-sentinel-alert.createddatetime", + "object": "alert" + } + ], + "description": { + "key": "x-msazure-sentinel-alert.description", + "object": "alert" + }, + "detectionIds": { + "key": "x-msazure-sentinel-alert.detectionids", + "object": "alert", + "transformer": "ToString" + }, + "feedback": { + "key": "x-msazure-sentinel-alert.feedback", + "object": "alert", + "transformer": "ToString" + }, + "fileStates": { + "fileHash": { + "sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "md5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "authenticodeHash256": { + "key": "file.hashes.authenticodeHash256", + "object": "file" + }, + "lsHash": { + "key": "file.hashes.lsHash", + "object": "file" + }, + "ctph": { + "key": "file.hashes.ctph", + "object": "file" + }, + "peSha1": { + "key": "file.hashes.peSha1", + "object": "file" + }, + "peSha256": { + "key": "file.hashes.peSha256", + "object": "file" + }, + "unknown": { + "key": "file.hashes.UNKNOWN", + "object": "file" + } + }, + "name": { + "key": "file.name", + "object": "file" + }, + "path": [ + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "riskScore": { + "key": "x-msazure-sentinel-alert.fileStates.riskScore", + "object": "file" + } + }, + "hostStates": { + "fqdn": { + "key": "domain-name.value", + "object": "host" + }, + "isAzureAadJoined": { + "key": "x-msazure-sentinel-alert.hostStates.isAzureAadJoined", + "object": "alert" + }, + "isAzureAadRegistered": { + "key": "x-msazure-sentinel-alert.hostStates.isAzureAadRegistered", + "object": "alert" + }, + "isHybridAzureDomainJoined": { + "key": "x-msazure-sentinel-alert.hostStates.isHybridAzureDomainJoined", + "object": "alert" + }, + "os": { + "key": "x-msazure-sentinel-alert.hostStates.os", + "object": "alert" + }, + "privateIpAddress": { + "key": "ipv4-addr.value" + }, + "publicIpAddress": { + "key": "ipv4-addr.value" + }, + "riskScore": { + "key": "x-msazure-sentinel-alert.hostStates.riskScore", + "object": "alert" + } + }, + "id": { + "key": "x-msazure-sentinel-alert.providerid", + "object": "alert" + }, + "incidentIds": { + "key": "x-msazure-sentinel-alert.incidentIds", + "object": "alert", + "transformer": "ToString" + }, + "lastModifiedDateTime": [ + { + "key": "modified", + "cybox": false + }, + { + "key": "x-msazure-sentinel-alert.lastmodifieddatetime", + "object": "alert" + } + ], + "malwareStates": { + "category": { + "key": "x-msazure-sentinel-alert.malwareStates.category", + "object": "alert" + }, + "family": { + "key": "x-msazure-sentinel-alert.malwareStates.family", + "object": "alert" + }, + "name": { + "key": "x-msazure-sentinel-alert.malwareStates.name", + "object": "alert" + }, + "severity": { + "key": "x-msazure-sentinel-alert.malwareStates.severity", + "object": "alert" + }, + "wasRunning": { + "key": "x-msazure-sentinel-alert.malwareStates.wasRunning", + "object": "alert" + } + }, + "networkConnections": { + "applicationName": { + "key": "software.name" + }, + "destinationAddress": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "destinationLocation": { + "key": "x-msazure-sentinel-alert.networkConnections.destinationLocation", + "object": "alert" + }, + "destinationDomain": { + "key": "domain-name.value", + "transformer": "ToDomainName" + }, + "destinationPort": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "destinationUrl": { + "key": "url.value", + "object": "url" + }, + "direction": { + "key": "x-msazure-sentinel-alert.networkConnections.direction", + "object": "alert" + }, + "domainRegisteredDateTime": { + "key": "x-msazure-sentinel-alert.networkConnections.domainRegisteredDateTime", + "object": "alert" + }, + "localDnsName": { + "key": "x-msazure-sentinel-alert.networkConnections.localDnsName", + "object": "alert" + }, + "natDestinationAddress": { + "key": "x-msazure-sentinel-alert.networkConnections.natDestinationAddress", + "object": "alert" + }, + "natDestinationPort": { + "key": "x-msazure-sentinel-alert.networkConnections.natDestinationPort", + "object": "alert" + }, + "natSourceAddress": { + "key": "x-msazure-sentinel-alert.networkConnections.natSourceAddress", + "object": "alert" + }, + "natSourcePort": { + "key": "x-msazure-sentinel-alert.networkConnections.natSourcePort", + "object": "alert" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "group": "True", + "transformer": "ToLowercaseArray" + }, + "riskScore": { + "key": "x-msazure-sentinel-alert.networkConnections.riskScore", + "object": "alert" + }, + "sourceAddress": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], + "sourceLocation": { + "key": "x-msazure-sentinel-alert.networkConnections.sourceLocation", + "object": "alert" + }, + "sourcePort": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "status": { + "key": "x-msazure-sentinel-alert.networkConnections.status", + "object": "alert" + }, + "urlParameters": { + "key": "x-msazure-sentinel-alert.networkConnections.urlParameters", + "object": "alert" + } + }, + "processes": { + "accountName": [ + { + "key": "user-account.user_id", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user" + } + ], + "commandLine": { + "key": "process.command_line", + "object": "process" + }, + "createdDateTime": { + "key": "process.created_time", + "object": "process" + }, + "fileHash": { + "sha256": { + "key": "file.hashes.SHA-256", + "object": "processType" + }, + "sha1": { + "key": "file.hashes.SHA-1", + "object": "processType" + }, + "md5": { + "key": "file.hashes.MD5", + "object": "processType" + }, + "authenticodeHash256": { + "key": "file.hashes.authenticodeHash256", + "object": "processType" + }, + "lsHash": { + "key": "file.hashes.lsHash", + "object": "processType" + }, + "ctph": { + "key": "file.hashes.ctph", + "object": "processType" + }, + "peSha1": { + "key": "file.hashes.peSha1", + "object": "processType" + }, + "peSha256": { + "key": "file.hashes.peSha256", + "object": "processType" + }, + "unknown": { + "key": "file.hashes.UNKNOWN", + "object": "processType" + } + }, + "integrityLevel": { + "key": "x-msazure-sentinel-alert.processes.integrityLevel", + "object": "alert" + }, + "isElevated": { + "key": "x-msazure-sentinel-alert.processes.isElevated", + "object": "alert" + }, + "parentProcessCreatedDateTime": { + "key": "process.created_time", + "object": "process_parent" + }, + "parentProcessId": [ + { + "key": "process.pid", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "path": { + "key": "directory.path", + "object": "dir" + }, + "processId": { + "key": "process.pid", + "object": "process" + } + }, + "recommendedActions": { + "key": "x-msazure-sentinel-alert.recommendedactions", + "object": "alert", + "transformer": "ToString" + }, + "registryKeyStates": { + "hive": { + "key": "x-msazure-sentinel-alert.registryKeyStates.hive", + "object": "alert", + "transformer": "ToString" + }, + "key": { + "key": "windows-registry-key.key", + "object": "registry" + }, + "oldKey": { + "key": "x-msazure-sentinel-alert.registryKeyStates.oldKey", + "object": "alert" + }, + "oldValueData": { + "key": "x-msazure-sentinel-alert.registryKeyStates.oldValueData", + "object": "alert" + }, + "oldValueName": { + "key": "x-msazure-sentinel-alert.registryKeyStates.oldValueName", + "object": "alert" + }, + "operation": { + "key": "x-msazure-sentinel-alert.registryKeyStates.operation", + "object": "alert", + "transformer": "ToString" + }, + "processId": { + "key": "process.pid" + }, + "valueData": { + "key": "extensions.windows-registry-value-type.valueData", + "object": "registry" + }, + "valueName": { + "key": "extensions.windows-registry-value-type.name", + "object": "registry" + }, + "valueType": { + "key": "extensions.windows-registry-value-type.valuetype", + "object": "registry", + "transformer": "ToString" + } + }, + "securityResources": { + "resource": { + "key": "x-msazure-sentinel-alert.securityresources.resource", + "object": "alert" + }, + "resourceType": { + "key": "x-msazure-sentinel-alert.securityresources.resourcetype", + "object": "alert", + "transformer": "ToString" + } + }, + "severity": { + "key": "x-msazure-sentinel-alert.severity", + "object": "alert" + }, + "sourceMaterials": { + "key": "x-msazure-sentinel-alert.sourcematerials", + "object": "alert", + "transformer": "ToString" + }, + "status": { + "key": "x-msazure-sentinel-alert.status", + "object": "alert", + "transformer": "ToString" + }, + "tags": { + "key": "x-msazure-sentinel-alert.tags", + "object": "alert", + "transformer": "ToString" + }, + "title": { + "key": "x-msazure-sentinel-alert.title", + "object": "alert" + }, + "triggers": { + "name": { + "key": "x-msazure-sentinel-alert.triggers.name", + "object": "alert" + }, + "type": { + "key": "x-msazure-sentinel-alert.triggers.type", + "object": "alert" + }, + "value": { + "key": "x-msazure-sentinel-alert.triggers.value", + "object": "alert" + } + }, + "userStates": { + "aadUserId": { + "key": "x-msazure-sentinel-alert.userStates.aaduserid", + "object": "alert" + }, + "accountName": { + "key": "user-account.user_id", + "object": "user" + }, + "domainName": { + "key": "domain-name.value" + }, + "emailRole": { + "key": "x-msazure-sentinel-alert.userStates.emailrole", + "object": "alert" + }, + "isVpn": { + "key": "x-msazure-sentinel-alert.userStates.isvpn", + "object": "alert" + }, + "logonDateTime": { + "key": "user-account.account_last_login", + "object": "user" + }, + "logonId": { + "key": "user-account.account_login", + "object": "user" + }, + "logonIp": { + "key": "ipv4-addr.value" + }, + "logonLocation": { + "key": "x-msazure-sentinel-alert.userStates.logonLocation", + "object": "alert" + }, + "logonType": { + "key": "x-msazure-sentinel-alert.userStates.logonType", + "object": "alert" + }, + "onPremisesSecurityIdentifier": { + "key": "x-msazure-sentinel-alert.userStates.onpremisessecurityidentifier", + "object": "alert" + }, + "riskScore": { + "key": "x-msazure-sentinel-alert.userStates.riskScore", + "object": "alert" + }, + "userAccountType": { + "key": "x-msazure-sentinel-alert.userStates.useraccounttype", + "object": "alert" + }, + "userPrincipalName": { + "key": "x-msazure-sentinel-alert.userStates.userPrincipalName", + "object": "alert" + } + }, + "vendorInformation": { + "provider": { + "key": "software.name", + "object": "application" + }, + "vendor": { + "key": "software.vendor", + "object": "application" + }, + "providerVersion": { + "key": "software.version", + "object": "application" + }, + "subProvider": { + "key": "x-msazure-sentinel-alert.vendorinformation.subprovider", + "object": "alert" + } + }, + "vulnerabilityStates": { + "cve": { + "key": "x-msazure-sentinel-alert.vulnerabilityStates.cve", + "object": "alert" + }, + "severity": { + "key": "x-msazure-sentinel-alert.vulnerabilityStates.severity", + "object": "alert" + }, + "wasRunning": { + "key": "x-msazure-sentinel-alert.vulnerabilityStates.wasRunning", + "object": "alert" + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_stix_to_query.py b/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_stix_to_query.py index 7e9ca7af3..7967cc552 100644 --- a/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_stix_to_query.py +++ b/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_stix_to_query.py @@ -55,10 +55,7 @@ def test_network_comp_exp(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = ["((networkConnections/any(query1:contains(tolower(query1/sourceAddress), '172.16.2.22')) or " - "networkConnections/any(query1:contains(tolower(query1/destinationAddress), '172.16.2.22')) or " - "hostStates/any(query1:tolower(query1/publicIpAddress) eq '172.16.2.22') or " - "hostStates/any(query1:tolower(query1/privateIpAddress) eq '172.16.2.22') or " - "userStates/any(query1:tolower(query1/logonIp) eq '172.16.2.22'))) " + "networkConnections/any(query1:contains(tolower(query1/destinationAddress), '172.16.2.22')))) " "and (eventDateTime ge 2019-09-10T08:43:10.003Z and eventDateTime le 2019-09-23T10:43:10.453Z)"] queries = _remove_timestamp_from_query(queries) diff --git a/stix_shifter_modules/bigfix/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/bigfix/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..eb45f3b1c --- /dev/null +++ b/stix_shifter_modules/bigfix/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,48 @@ +{ + "file": { + "fields": { + "name": ["file.name", "process.name"], + "hashes.'SHA-256'": ["file.sha256"], + "hashes.'SHA-1'": ["file.sha1"], + "hashes.MD5": ["file.md5"], + "parent_directory_ref.path": ["file.folder"], + "size": ["file.size"] + } + }, + "process": { + "fields": { + "pid": ["process.pid", "process.process id"], + "parent_ref.pid": ["process.ppid"], + "image_ref.name" : ["file.name", "process.name"], + "image_ref.hashes.SHA-256" : ["file.sha256"], + "image_ref.hashes.SHA-1": ["file.sha1"], + "image_ref.hashes.MD5": ["file.md5"], + "image_ref.parent_directory_ref.path": ["file.pathname"], + "image_ref.size": ["file.size"], + "creator_user_ref.user_id": ["process.user", "process.name of user"] + } + }, + "ipv4-addr": { + "fields": { + "value": ["socket.local address","socket.remote address"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["socket.local address","socket.remote address"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["socket.local port"], + "dst_port": ["socket.remote port"], + "src_ref.value": ["socket.local address"], + "dst_ref.value": ["socket.remote address"] + } + }, + "mac-addr": { + "fields": { + "value": ["adapter.mac address"] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/bigfix/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/bigfix/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..fdf89adc2 --- /dev/null +++ b/stix_shifter_modules/bigfix/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,140 @@ +{ + "file_name": { + "key": "file.name", + "object": "file" + }, + "file_path": [ + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file" + } + ], + "sha256hash": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "sha1hash": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "md5hash": { + "key": "file.hashes.MD5", + "object": "file" + }, + "file_size": { + "key" : "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "process_id": { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + "process_ppid": [ + { + "key": "process.pid", + "object": "parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "process_user": [ + { + "key":"user-account.user_id", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user" + } + ], + "timestamp": [ + { + "key": "first_observed", + "transformer": "EpochSecondsToTimestamp", + "cybox": false + }, + { + "key": "last_observed", + "transformer": "EpochSecondsToTimestamp", + "cybox": false + } + ], + "computer_identity": { + "key": "x-bigfix-relevance.computer_identity" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "local_address": + [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], + "remote_address": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "local_port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "remote_port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + "mac": [ + { + "key": "mac-addr.value", + "object": "src_mac" + } + ] +} \ No newline at end of file diff --git a/stix_shifter_modules/carbonblack/stix_translation/json/from_stix_map.json b/stix_shifter_modules/carbonblack/stix_translation/json/from_stix_map.json index 4e4c5c5d5..700d05529 100644 --- a/stix_shifter_modules/carbonblack/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/carbonblack/stix_translation/json/from_stix_map.json @@ -44,12 +44,12 @@ "creator_user_ref.user_id": ["username"], "creator_user_ref.account_login": ["username"], "binary_ref.hashes.MD5": ["md5"], - "binary_ref.hashes.SHA-256": ["sha256"], + "binary_ref.hashes.'SHA-256'": ["sha256"], "parent_ref.pid": ["parent_pid"], "parent_ref.name": ["parent_name"], "parent_ref.command_line": ["cmdline"], "parent_ref.binary_ref.hashes.MD5": ["parent_md5"], - "parent_ref.binary_ref.hashes.SHA-256": ["parent_sha256"] + "parent_ref.binary_ref.hashes.'SHA-256'": ["parent_sha256"] } }, "url": { @@ -89,14 +89,14 @@ "process_ref.parent_ref.pid": ["parent_pid"], "process_ref.parent_ref.command_line": ["cmdline"], "process_ref.parent_ref.binary_ref.hashes.MD5": ["parent_md5"], - "process_ref.parent_ref.binary_ref.hashes.SHA-256": ["parent_sha256"], + "process_ref.parent_ref.binary_ref.hashes.'SHA-256'": ["parent_sha256"], "process_ref.process_ref.creator_user_ref.user_id": ["username"], "process_ref.process_ref.creator_user_ref.account_login": ["username"], "parent_process_ref.name": ["parent_name"], "parent_process_ref.pid": ["parent_pid"], "parent_process_ref.command_line": ["cmdline"], "parent_process_ref.binary_ref.hashes.MD5": ["parent_md5"], - "parent_process_ref.binary_ref.hashes.SHA-256": ["parent_sha256"], + "parent_process_ref.binary_ref.hashes.'SHA-256'": ["parent_sha256"], "parent_process_ref.creator_user_ref.user_id": ["username"], "parent_process_ref.creator_user_ref.account_login": ["username"], "domain_ref.value": ["domain", "hostname"], diff --git a/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..6a00544ce --- /dev/null +++ b/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,174 @@ +{ + "network-traffic": { + "fields": { + "src_port": ["ipport"], + "dst_port": ["ipport"], + "src_ref.value": ["ipaddr", "ipv6addr"], + "dst_ref.value": ["ipaddr", "ipv6addr"] + } + }, + "ipv4-addr": { + "fields": { + "value": ["ipaddr"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["ipv6addr"] + } + }, + "file": { + "fields": { + "name": ["process_name", "childproc_name"], + "hashes.MD5": ["md5"], + "hashes.'SHA-256'": ["sha256"], + "hashes.parent_MD5": ["parent_md5"], + "hashes.parent_SHA-256": ["parent_sha256"], + "hashes.child_MD5": ["childproc_md5"], + "hashes.child_SHA-256": ["childproc_sha256"], + "hashes.filewrite_MD5": ["filewrite_md5"], + "hashes.filewrite_SHA-256": ["filewrite_sha256"], + "hashes.blocked_MD5": ["blocked_md5"], + "hashes.blocked_SHA-256": ["blocked_sha256"], + "hashes.crossproc_MD5": ["crossproc_md5"], + "hashes.crossproc_SHA-256": ["crossproc_sha256"], + "parent_directory_ref.path": ["path", "modload"] + } + }, + "process": { + "fields": { + "command_line": ["cmdline"], + "created_time": ["start"], + "pid": ["process_pid"], + "creator_user_ref.user_id": ["username"], + "creator_user_ref.account_login": ["username"], + "image_ref.hashes.MD5": ["md5"], + "image_ref.hashes.'SHA-256'": ["sha256"], + "parent_ref.pid": ["parent_pid"], + "parent_ref.name": ["parent_name"], + "parent_ref.command_line": ["cmdline"], + "parent_ref.image_ref.hashes.MD5": ["parent_md5"], + "parent_ref.image_ref.hashes.'SHA-256'": ["parent_sha256"] + } + }, + "url": { + "fields": { + "value": ["domain"] + } + }, + "domain-name": { + "fields": { + "value": ["domain", "hostname"] + } + }, + "user-account": { + "fields": { + "user_id": ["username"] + } + }, + "directory": { + "fields": { + "path": ["path"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["regmod"] + } + }, + "x-oca-event": { + "fields": { + "process_ref.pid": ["process_pid"], + "process_ref.name": ["process_name", "crossproc_name"], + "process_ref.command_line": ["cmdline"], + "process_ref.image_ref.name": ["process_name", "childproc_name"], + "process_ref.creator_user_ref.user_id": ["username"], + "process_ref.creator_user_ref.account_login": ["username"], + "process_ref.parent_ref.name": ["parent_name"], + "process_ref.parent_ref.pid": ["parent_pid"], + "process_ref.parent_ref.command_line": ["cmdline"], + "process_ref.parent_ref.image_ref.hashes.MD5": ["parent_md5"], + "process_ref.parent_ref.image_ref.hashes.'SHA-256'": ["parent_sha256"], + "process_ref.process_ref.creator_user_ref.user_id": ["username"], + "process_ref.process_ref.creator_user_ref.account_login": ["username"], + "parent_process_ref.name": ["parent_name"], + "parent_process_ref.pid": ["parent_pid"], + "parent_process_ref.command_line": ["cmdline"], + "parent_process_ref.image_ref.hashes.MD5": ["parent_md5"], + "parent_process_ref.image_ref.hashes.'SHA-256'": ["parent_sha256"], + "parent_process_ref.creator_user_ref.user_id": ["username"], + "parent_process_ref.creator_user_ref.account_login": ["username"], + "domain_ref.value": ["domain", "hostname"], + "file_ref.name": ["process_name", "childproc_name"], + "host_ref.hostname": ["hostname"], + "host_ref.name": ["hostname"], + "registry_ref.key": ["regmod"] + } + }, + "x-oca-asset": { + "fields": { + "domain": ["domain"], + "hostname": ["hostname"], + "ip": ["ipaddr", "ipv6addr"], + "name": ["hostname"], + "type": ["host_type"], + "os.name": ["os_type"], + "os.platform": ["os_type"] + } + }, + "x-cb-response": { + "fields": { + "hostname": ["hostname"], + "host_count": ["host_count"], + "host_type": ["host_type"], + "group": ["group"], + "os_type": ["os_type"], + "crossproc_type": ["crossproc_type"], + "crossproc_count": ["crossproc_count"], + "crossproc_name": ["crossproc_name"], + "tampered": ["tampered"], + "block_status": ["block_status"], + "digsig_result": ["digsig_result"], + "digsig_publisher": ["digsig_publisher"], + "digsig_issuer": ["digsig_issuer"], + "digsig_prog_name": ["digsig_prog_name"], + "digsig_sign_time": ["digsig_sign_time"], + "digsig_subject": ["digsig_subject"], + "has_emet_event": ["has_emet_event"], + "has_emet_config": ["has_emet_config"], + "file_desc": ["file_desc"], + "file_version": ["file_version"], + "filemod_count": ["filemod_count"], + "filemod": ["filemod"], + "regmod_count": ["regmod_count"], + "regmod": ["regmod"], + "blocked_status": ["blocked_status"], + "childproc_count": ["childproc_count"], + "childproc_name": ["childproc_name"], + "company_name": ["company_name"], + "copied_mod_len": ["copied_mod_len"], + "internal_name": ["internal_name"], + "is_64bit": ["is_64bit"], + "is_executable_image": ["is_executable_image"], + "last_server_update": ["last_server_update"], + "last_update": ["last_update"], + "legal_copyright": ["legal_copyright"], + "legal_trademark": ["legal_trademark"], + "modload": ["modload"], + "modload_count": ["modload_count"], + "netconn_count": ["netconn_count"], + "observed_filename": ["observed_filename"], + "orig_mod_len": ["orig_mod_len"], + "original_filename": ["original_filename"], + "parent_id": ["parent_id"], + "parent_name": ["parent_name"], + "private_build": ["private_build"], + "process_id": ["process_id"], + "product_desc": ["product_desc"], + "product_name": ["product_name"], + "product_version": ["product_version"], + "sensor_id": ["sensor_id"], + "special_build": ["special_build"] + } + } +} diff --git a/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..9cd75ce50 --- /dev/null +++ b/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,251 @@ +{ + "username": [ + { + "key": "user-account.user_id", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "start": [ + { + "key": "process.created_time", + "object": "process" + }, + { + "key": "first_observed", + "cybox": false + } + ], + "process_name": [ + { + "key": "file.name", + "object": "process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "process_pid": { + "key": "process.pid", + "object": "process" + }, + "id": { + "key": "process.x_id", + "object": "process" + }, + "unique_id": { + "key": "process.x_unique_id", + "object": "process" + }, + "parent_name": [ + { + "key": "file.name", + "object": "parent_file" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "parent_pid": [ + { + "key": "process.pid", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "parent_id": { + "key": "process.x_id", + "object": "parent_process" + }, + "parent_unique_id": { + "key": "process.x_unique_id", + "object": "parent_process" + }, + "process_md5": { + "key": "file.hashes.MD5", + "object": "process_file" + }, + "process_sha256": { + "key": "file.hashes.SHA-256", + "object": "process_file" + }, + "cmdline": { + "key": "process.command_line", + "object": "process" + }, + "interface_ip": [ + { + "key": "ipv4-addr.value", + "object": "ipv4-addr", + "transformer": "ToIPv4" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "ipv4-addr" + ], + "group": true + } + ], + "path": [ + { + "key": "directory.path", + "object": "dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "dir" + } + ], + "hostname": [ + { + "key": "x-cb-response.host_name", + "object": "cb" + }, + { + "key": "x-oca-asset.hostname", + "object": "host" + } + ], + "host_type": [ + { + "key": "x-cb-response.host_type", + "object": "cb" + }, + { + "key": "x-oca-asset.host_type", + "object": "host" + } + ], + "comms_ip": { + "key": "x-cb-response.comms_ip", + "object": "cb", + "transformer": "ToIPv4" + }, + "os_type": [ + { + "key": "x-cb-response.os_type", + "object": "cb" + }, + { + "key": "x-oca-asset.os_name", + "object": "host" + } + ], + "sensor_id": { + "key": "x-cb-response.sensor_id", + "object": "cb" + }, + "group": { + "key": "x-cb-response.group", + "object": "cb" + }, + "segment_id": { + "key": "x-cb-response.segment_id", + "object": "cb" + }, + "terminated": { + "key": "x-cb-response.terminated", + "object": "cb" + }, + "regmod_count": { + "key": "x-cb-response.regmod_count", + "object": "cb" + }, + "netconn_count": { + "key": "x-cb-response.netconn_count", + "object": "cb" + }, + "filemod_count": { + "key": "x-cb-response.filemod_count", + "object": "cb" + }, + "modload_count": { + "key": "x-cb-response.modload_count", + "object": "cb" + }, + "childproc_count": { + "key": "x-cb-response.childproc_count", + "object": "cb" + }, + "crossproc_count": { + "key": "x-cb-response.crossproc_count", + "object": "cb" + }, + "emet_count": { + "key": "x-cb-response.emet_count", + "object": "cb" + }, + "emet_config": { + "key": "x-cb-response.emet_config", + "object": "cb" + }, + "processblock_count": { + "key": "x-cb-response.processblock_count", + "object": "cb" + }, + "filtering_known_dlls": { + "key": "x-cb-response.filtering_known_dlls", + "object": "cb" + }, + "last_update": { + "key": "last_observed", + "cybox": false + }, + "last_server_update": { + "key": "x-cb-response.last_server_update", + "object": "cb" + }, + "logon_type": { + "key": "x-cb-response.logon_type", + "object": "cb" + }, + "alliance_score_srstrust": { + "key": "x-cb-response.alliance_score_srstrust", + "object": "cb" + }, + "alliance_link_srstrust": { + "key": "x-cb-response.alliance_link_srstrust", + "object": "cb" + }, + "alliance_data_srstrust": { + "key": "x-cb-response.alliance_data_srstrust", + "object": "cb" + }, + "alliance_updated_srstrust": { + "key": "x-cb-response.alliance_updated_srstrust", + "object": "cb" + }, + "server_added_timestamp": + { + "key": "file.ctime", + "object": "binary_file" + }, + "original_filename": { + "key": "file.name", + "object": "binary_file" + }, + "orig_mod_length": { + "key": "file.size", + "object": "binary_file" + }, + "md5": { + "key": "file.hashes.MD5", + "object": "binary_file" + } +} \ No newline at end of file diff --git a/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/to_stix_map_events.json b/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/to_stix_map_events.json new file mode 100644 index 000000000..1a494abaf --- /dev/null +++ b/stix_shifter_modules/carbonblack/stix_translation/json/stix_2_1/to_stix_map_events.json @@ -0,0 +1,391 @@ +{ + "event_timestamp": [ + { + "key": "first_observed", + "cybox": false + }, + { + "key": "last_observed", + "cybox": false + }, + { + "key": "x-oca-event.created", + "object": "event" + } + ], + "event_type": [ + { + "key": "x-oca-event.category", + "object": "event" + } + ], + "process_name": [ + { + "key": "file.name", + "object": "file", + "transformer": "ToFileName" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "process_pid": [ + { + "key": "process.pid", + "object": "process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "filemod_action": { + "key": "x-oca-event.action", + "object": "event" + }, + "filemod_name": [ + { + "key": "file.name", + "object": "file", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "filemod_md5": [ + { + "key": "file.hashes.MD5", + "object": "file" + } + ], + "modload_name": [ + { + "key": "file.name", + "object": "service_file", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "service_directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "service_file", + "references": "service_directory" + }, + { + "key": "process.extensions.windows-service-ext.service_dll_refs", + "object": "process", + "group": true, + "references": ["service_file"] + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "modload_md5": [ + { + "key": "file.hashes.MD5", + "object": "service_file" + } + ], + "domain": [ + { + "key": "domain-name.value", + "object": "domain", + "transformer": "ToDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain" + } + ], + "netconn_local_ipv4": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-oca-event.nt_ref", + "object": "event", + "references": "nt" + } + ], + "netconn_local_port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "netconn_remote_ipv4": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.nt_ref", + "object": "event", + "references": "nt" + } + ], + "netconn_remote_port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "childproc_name": [ + { + "key": "file.name", + "object": "child_file", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "child_directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.image_ref", + "object": "child_process", + "references": "child_file" + }, + { + "key": "file.parent_directory_ref", + "object": "child_file", + "references": "child_directory" + } + ], + "childproc_md5": { + "key": "file.hashes.MD5", + "object": "child_file" + }, + "childproc_sha256": { + "key": "file.hashes.SHA-256", + "object": "child_file" + }, + "childproc_pid": [ + { + "key": "process.pid", + "object": "child_process" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "childproc_username": { + "key": "user-account.user_id", + "object": "child_username" + }, + "childproc_cmdline": { + "key": "process.command_line", + "object": "child_process" + }, + "childproc_action": { + "key": "x-oca-event.action", + "object": "event" + }, + "regmod_name": [ + { + "key": "windows-registry-key.key", + "object": "win_registry" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "win_registry" + } + ], + "crossproc_action": { + "key": "x-oca-event.action", + "object": "event" + }, + "crossproc_name": [ + { + "key": "file.name", + "object": "other_file", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "other_directory", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "other_file", + "references": "other_directory" + }, + { + "key": "process.image_ref", + "object": "other_process", + "references": "other_file" + }, + { + "key": "x-oca-event.cross_process_target_ref", + "object": "event", + "references": "other_process" + } + ], + "crossproc_md5": { + "key": "file.hashes.MD5", + "object": "other_file" + }, + "crossproc_pid": { + "key": "process.pid", + "object": "other_process" + }, + "device_name": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "interface_ip": [ + { + "key": "ipv4-addr.value", + "object": "ipv4-addr", + "transformer": "ToIPv4" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "ipv4-addr" + ], + "group": true + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "device_external_ip": { + "key": "x-cb-response.comms_ip", + "object": "cb", + "transformer": "ToIPv4" + }, + "host_type": [ + { + "key": "x-oca-asset.host_type", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "device_os": [ + { + "key": "x-oca-asset.os_name", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event" + }, + "parent_pid": [ + { + "key": "process.pid", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "parent_name": [ + { + "key": "file.name", + "object": "parent_file", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "process_cmdline": [ + { + "key": "process.command_line", + "object": "process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ] +} diff --git a/stix_shifter_modules/cbcloud/stix_translation/json/from_stix_map.json b/stix_shifter_modules/cbcloud/stix_translation/json/from_stix_map.json index 543fb3680..64acee231 100644 --- a/stix_shifter_modules/cbcloud/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/cbcloud/stix_translation/json/from_stix_map.json @@ -26,13 +26,13 @@ "name": ["process_name"], "binary_ref.name": ["process_name"], "binary_ref.hashes.MD5": ["process_hash"], - "binary_ref.hashes.SHA-256": ["process_hash", "process_sha256"], + "binary_ref.hashes.'SHA-256'": ["process_hash", "process_sha256"], "parent_ref.command_line": ["parent_cmdline"], "parent_ref.pid": ["parent_pid"], "parent_ref.name": ["parent_name"], "parent_ref.binary_ref.name": ["parent_name"], "parent_ref.binary_ref.hashes.MD5": ["parent_hash"], - "parent_ref.binary_ref.hashes.SHA-256": ["parent_hash"] + "parent_ref.binary_ref.hashes.'SHA-256'": ["parent_hash"] } }, "software": { diff --git a/stix_shifter_modules/cbcloud/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/cbcloud/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..308866263 --- /dev/null +++ b/stix_shifter_modules/cbcloud/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,134 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["netconn_ipv4", "netconn_local_ipv4"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["netconn_ipv6", "netconn_local_ipv6"] + } + }, + "file": { + "fields": { + "name": ["process_name"], + "hashes.MD5": ["process_hash"], + "hashes.'SHA-256'": ["process_hash", "process_sha256"] + } + }, + "process": { + "fields": { + "created_time": ["process_start_time"], + "creator_user_ref.account_login": ["process_username"], + "service_name":["process_service_name"], + "command_line": ["process_cmdline"], + "pid": ["process_pid"], + "image_ref.name": ["process_name"], + "image_ref.hashes.MD5": ["process_hash"], + "image_ref.hashes.'SHA-256'": ["process_hash", "process_sha256"], + "parent_ref.command_line": ["parent_cmdline"], + "parent_ref.pid": ["parent_pid"], + "parent_ref.name": ["parent_name"], + "parent_ref.image_ref.name": ["parent_name"], + "parent_ref.image_ref.hashes.MD5": ["parent_hash"], + "parent_ref.image_ref.hashes.'SHA-256'": ["parent_hash"] + } + }, + "software": { + "fields": { + "name": ["process_file_description", "process_internal_name"], + "vendor": ["process_company_name", "process_publisher"], + "version": ["process_product_version"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["netconn_port"], + "dst_port": ["netconn_port"], + "protocols[*]": ["netconn_protocol"], + "src_ref.value": ["netconn_local_ipv4", "netconn_local_ipv6"], + "dst_ref.value": ["netconn_ipv4", "netconn_ipv6"] + } + }, + "user-account": { + "fields": { + "user_id": ["process_username"] + } + }, + "url": { + "fields": { + "value": ["netconn_domain"] + } + }, + "domain-name": { + "fields": { + "value": ["netconn_domain"] + } + }, + "x-cbcloud" : { + "fields": { + "device_name": ["device_name"], + "device_internal_ip": ["device_internal_ip"], + "device_external_ip": ["device_external_ip"], + "device_os": ["device_os"], + "device_os_version": ["device_os_version"], + "device_id": ["device_id"], + "device_group": ["device_group"], + "device_policy": ["device_policy"], + "device_timestamp": ["device_timestamp"], + "process_terminated": ["process_terminated"], + "regmod_count": ["regmod_count"], + "netconn_count": ["netconn_count"], + "filemod_count": ["filemod_count"], + "modload_count": ["modload_count"], + "childproc_count": ["childproc_count"], + "crossproc_count": ["crossproc_count"], + "scriptload_count": ["scriptload_count"], + "parent_cmdline_length": ["parent_cmdline_length"], + "process_cmdline_length": ["process_cmdline_length"], + "parent_publisher_state": ["parent_publisher_state"], + "process_publisher_state": ["process_publisher_state"], + "parent_reputation": ["parent_reputation"], + "process_reputation": ["process_reputation"], + "parent_effective_reputation": ["parent_effective_reputation"], + "process_effective_reputation": ["process_effective_reputation"], + "netconn_location": ["netconn_location"], + "netconn_inbound": ["netconn_inbound"], + "netconn_failed": ["netconn_failed"], + "regmod_name": ["regmod_name"], + "scriptload_name": ["scriptload_name"], + "scriptload_hash": ["scriptload_hash"], + "scriptload_publisher_state": ["scriptload_publisher_state"], + "fileless_scriptload_hash": ["fileless_scriptload_hash"], + "fileless_scriptload_cmdline": ["fileless_scriptload_cmdline"], + "fileless_scriptload_cmdline_length": ["fileless_scriptload_cmdline_length"], + "modload_name": ["modload_name"], + "modload_hash": ["modload_hash"], + "modload_publisher_state": ["modload_publisher_state"], + "filemod_name": ["filemod_name"], + "filemod_hash": ["filemod_hash"], + "filemod_publisher_state": ["filemod_publisher_state"], + "crossproc_action": ["crossproc_action"], + "crossproc_api": ["crossproc_api"], + "crossproc_hash": ["crossproc_hash"], + "crossproc_name": ["crossproc_name"], + "crossproc_target": ["crossproc_target"], + "childproc_cmdline": ["childproc_cmdline"], + "childproc_cmdline_length": ["childproc_cmdline_length"], + "childproc_effective_reputation": ["childproc_effective_reputation"], + "childproc_guid": ["childproc_guid"], + "childproc_hash": ["childproc_hash"], + "childproc_name": ["childproc_name"], + "childproc_publisher_state": ["childproc_publisher_state"], + "childproc_reputation": ["childproc_reputation"], + "hash": ["hash"], + "process_original_filename": ["process_original_filename"], + "process_product_name": ["process_product_name"], + "backend_timestamp": ["backend_timestamp"], + "process_duration": ["process_duration"], + "process_elevated": ["process_elevated"], + "process_integrity_level": ["process_integrity_level"], + "process_privileges": ["process_privileges"] + } + } +} diff --git a/stix_shifter_modules/cbcloud/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/cbcloud/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..763f46709 --- /dev/null +++ b/stix_shifter_modules/cbcloud/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,182 @@ +{ + "process_username": [ + { + "key": "user-account.user_id", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "process_start_time": [ + { + "key": "process.created_time", + "object": "process" + } + ], + "process_name": [ + { + "key": "file.name", + "object": "process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "process_pid": { + "key": "process.pid", + "object": "process" + }, + "process_guid": { + "key": "process.x_unique_id", + "object": "process" + }, + "process_md5": { + "key": "file.hashes.MD5", + "object": "process_file" + }, + "process_sha256": { + "key": "file.hashes.SHA-256", + "object": "process_file" + }, + "process_cmdline": { + "key": "process.command_line", + "object": "process" + }, + "process_path": [ + { + "key": "directory.path", + "object": "process_dir" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_dir" + } + ], + "parent_name": [ + { + "key": "file.name", + "object": "parent_file" + }, + { + "key": "process.image_ref", + "object": "parent", + "references": "parent_file" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent" + } + ], + "parent_pid": { + "key": "process.pid", + "object": "parent" + }, + "parent_guid": { + "key": "process.x_unique_id", + "object": "parent" + }, + "parent_md5": { + "key": "file.hashes.MD5", + "object": "parent_file" + }, + "parent_sha256": { + "key": "file.hashes.SHA-256", + "object": "parent_file" + }, + "parent_cmdline": { + "key": "process.command_line", + "object": "parent" + }, + "parent_path": [ + { + "key": "directory.path", + "object": "parent_dir" + }, + { + "key": "file.parent_directory_ref", + "object": "parent_file", + "references": "parent_dir" + } + ], + "device_name": { + "key": "x-cbcloud.device_name", + "object": "x_cbcloud" + }, + "device_internal_ip": { + "key": "x-cbcloud.device_internal_ip", + "object": "x_cbcloud" + }, + "device_external_ip": { + "key": "x-cbcloud.device_external_ip", + "object": "x_cbcloud" + }, + "device_os": { + "key": "x-cbcloud.device_os", + "object": "x_cbcloud" + }, + "device_id": { + "key": "x-cbcloud.device_id", + "object": "x_cbcloud" + }, + "device_timestamp": [ + { + "key": "first_observed", + "cybox": false + }, + { + "key": "last_observed", + "cybox": false + }, + { + "key": "x-cbcloud.device_timestamp", + "object": "x_cbcloud" + } + ], + "org_id": { + "key": "x-cbcloud.org_id", + "object": "x_cbcloud" + }, + "device_group_id": { + "key": "x-cbcloud.device_group_id", + "object": "x_cbcloud" + }, + "process_terminated": { + "key": "x-cbcloud.process_terminated", + "object": "x_cbcloud" + }, + "regmod_count": { + "key": "x-cbcloud.regmod_count", + "object": "x_cbcloud" + }, + "netconn_count": { + "key": "x-cbcloud.netconn_count", + "object": "x_cbcloud" + }, + "filemod_count": { + "key": "x-cbcloud.filemod_count", + "object": "x_cbcloud" + }, + "modload_count": { + "key": "x-cbcloud.modload_count", + "object": "x_cbcloud" + }, + "childproc_count": { + "key": "x-cbcloud.childproc_count", + "object": "x_cbcloud" + }, + "crossproc_count": { + "key": "x-cbcloud.crossproc_count", + "object": "x_cbcloud" + }, + "scriptload_count": { + "key": "x-cbcloud.scriptload_count", + "object": "x_cbcloud" + } +} \ No newline at end of file diff --git a/stix_shifter_modules/crowdstrike/stix_translation/json/from_stix_map.json b/stix_shifter_modules/crowdstrike/stix_translation/json/from_stix_map.json index a7bcc3c57..62c1c9907 100644 --- a/stix_shifter_modules/crowdstrike/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/crowdstrike/stix_translation/json/from_stix_map.json @@ -34,11 +34,11 @@ "creator_user_ref.user_id": ["behaviors.user_id"], "creator_user_ref.account_login": ["behaviors.user_name"], "binary_ref.hashes.MD5": ["behaviors.md5"], - "binary_ref.hashes.SHA-256": ["behaviors.sha256"], + "binary_ref.hashes.'SHA-256'": ["behaviors.sha256"], "parent_ref.name": ["behaviors.filename"], "parent_ref.command_line": ["behaviors.parent_cmdline"], "parent_ref.binary_ref.hashes.MD5": ["behaviors.parent_details.parent_md5"], - "parent_ref.binary_ref.hashes.SHA-256": ["behaviors.parent_details.parent_sha256"] + "parent_ref.binary_ref.hashes.'SHA-256'": ["behaviors.parent_details.parent_sha256"] } }, "url": { @@ -72,11 +72,11 @@ "process_ref.parent_ref.name": ["behaviors.filename"], "process_ref.parent_ref.command_line": ["behaviors.parent_details.parent_cmdline"], "process_ref.parent_ref.binary_ref.hashes.MD5": ["behaviors.parent_details.parent_md5"], - "process_ref.parent_ref.binary_ref.hashes.SHA-256": ["behaviors.parent_details.parent_sha256"], + "process_ref.parent_ref.binary_ref.hashes.'SHA-256'": ["behaviors.parent_details.parent_sha256"], "parent_process_ref.name": ["behaviors.filename"], "parent_process_ref.command_line": ["behaviors.parent_details.parent_cmdline"], "parent_process_ref.binary_ref.hashes.MD5": ["behaviors.parent_details.parent_md5"], - "parent_process_ref.binary_ref.hashes.SHA-256": ["behaviors.parent_details.parent_sha256"], + "parent_process_ref.binary_ref.hashes.'SHA-256'": ["behaviors.parent_details.parent_sha256"], "parent_process_ref.creator_user_ref.user_id": ["behaviors.user_id"], "parent_process_ref.creator_user_ref.account_login": ["behaviors.user_name"], "domain_ref.value": ["device.hostname"], diff --git a/stix_shifter_modules/crowdstrike/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/crowdstrike/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..e4fdcfa1e --- /dev/null +++ b/stix_shifter_modules/crowdstrike/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,125 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["device.external_ip", "device.local_ip"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["device.external_ip", "device.local_ip"] + } + }, + "mac-addr": { + "fields": { + "value": ["device.mac_address"] + } + }, + "file": { + "fields": { + "name": ["behaviors.filename"], + "hashes.MD5": ["behaviors.md5", "behaviors.parent_details.parent_md5"], + "hashes.SHA-256": ["behaviors.sha256", "behaviors.parent_details.parent_sha256"], + "hashes.parent_MD5": ["behaviors.parent_details.parent_md5"], + "hashes.parent_SHA-256": ["behaviors.parent_details.parent_sha256"], + "hashes.child_MD5": ["behaviors.md5"], + "hashes.child_SHA-256": ["behaviors.sha256"], + "parent_directory_ref.path": ["behaviors.filepath"] + } + }, + "process": { + "fields": { + "command_line": ["behaviors.cmdline"], + "created_time": ["behaviors.timestamp"], + "creator_user_ref.user_id": ["behaviors.user_id"], + "creator_user_ref.account_login": ["behaviors.user_name"], + "image_ref.hashes.MD5": ["behaviors.md5"], + "image_ref.hashes.SHA-256": ["behaviors.sha256"], + "parent_ref.name": ["behaviors.filename"], + "parent_ref.command_line": ["behaviors.parent_cmdline"], + "parent_ref.image_ref.hashes.MD5": ["behaviors.parent_details.parent_md5"], + "parent_ref.image_ref.hashes.SHA-256": ["behaviors.parent_details.parent_sha256"] + } + }, + "url": { + "fields": { + "value": ["device.hostinfo.domain"] + } + }, + "domain-name": { + "fields": { + "value": ["ioc_type.domain"] + } + }, + "user-account": { + "fields": { + "user_id": ["behaviors.user_id"], + "account_login": ["behaviors.user_name"] + } + }, + "directory": { + "fields": { + "path": ["behaviors.filepath"] + } + }, + "x-oca-event": { + "fields": { + "process_ref.name": ["behaviors.filename"], + "process_ref.command_line": ["behaviors.cmdline"], + "process_ref.image_ref.name": ["behaviors.filename"], + "process_ref.creator_user_ref.user_id": ["behaviors.user_id"], + "process_ref.creator_user_ref.account_login": ["behaviors.user_name"], + "process_ref.parent_ref.name": ["behaviors.filename"], + "process_ref.parent_ref.command_line": ["behaviors.parent_details.parent_cmdline"], + "process_ref.parent_ref.image_ref.hashes.MD5": ["behaviors.parent_details.parent_md5"], + "process_ref.parent_ref.image_ref.hashes.SHA-256": ["behaviors.parent_details.parent_sha256"], + "parent_process_ref.name": ["behaviors.filename"], + "parent_process_ref.command_line": ["behaviors.parent_details.parent_cmdline"], + "parent_process_ref.image_ref.hashes.MD5": ["behaviors.parent_details.parent_md5"], + "parent_process_ref.image_ref.hashes.SHA-256": ["behaviors.parent_details.parent_sha256"], + "parent_process_ref.creator_user_ref.user_id": ["behaviors.user_id"], + "parent_process_ref.creator_user_ref.account_login": ["behaviors.user_name"], + "domain_ref.value": ["device.hostname"], + "file_ref.name": ["behaviors.filename"], + "host_ref.hostname": ["device.hostname"], + "host_ref.name": ["device.machine_domain"] + } + }, + "x-oca-asset": { + "fields": { + "name": ["device.machine_domain"], + "hostname": ["device.hostname"], + "ip_refs": ["device.external_ip", "device.local_ip"], + "os_name": ["device.platform_name"], + "os_platform": ["device.platform_name"] + } + }, + "x-crowdstrike": { + "fields": { + "device_id": ["device.device_id"], + "detection_id": ["device.detection_id"], + "scenario": ["behaviors.scenario"], + "tactic_id": ["device.tactic_id"], + "severity": ["device.severity"], + "tactic": ["behaviors.tactic"], + "technique": ["behaviors.technique"], + "technique_id": ["device.technique_id"], + "agent_local_time": ["device.agent_local_time"], + "agent_version": ["device.agent_version"], + "first_seen": ["device.first_seen"], + "last_seen": ["device.last_seen"], + "confidence": ["device.confidence"], + "bios_manufacturer": ["device.bios_manufacturer"], + "bios_version": ["device.bios_version"], + "config_id_base": ["device.config_id_base"], + "config_id_build": ["device.config_id_build"], + "config_id_platform": ["device.config_id_platform"], + "platform_id": ["device.platform_id"], + "product_type": ["device.product_type"], + "product_type_desc": ["device.product_type_desc"], + "site_name": ["device.site_name"], + "system_manufacturer": ["device.system_manufacturer"], + "system_product_name": ["device.system_product_name"], + "modified_timestamp": ["device.modified_timestamp"] + } + } +} diff --git a/stix_shifter_modules/crowdstrike/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/crowdstrike/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..bfff02290 --- /dev/null +++ b/stix_shifter_modules/crowdstrike/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,389 @@ +{ + "timestamp": [ + { + "key": "first_observed", + "cybox": false + }, + { + "key": "last_observed", + "cybox": false + }, + { + "key": "x-oca-event.created", + "object": "event" + } + ], + "filename": [ + { + "key": "file.name", + "object": "process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "machine_domain": { + "key": "x-crowdstrike.machine_domain", + "object": "x-crowdstrike" + }, + "filepath": [ + { + "key": "directory.path", + "object": "process_file_dir" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "cmdline": { + "key": "process.command_line", + "object": "process" + }, + "display_name": { + "key": "x-oca-event.action", + "object": "event" + }, + "description": { + "key": "x-oca-event.outcome", + "object": "event" + }, + "user_name": [ + { + "key": "user-account.account_login", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "user_id": [ + { + "key": "user-account.user_id", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "sha256": { + "key": "file.hashes.SHA-256", + "object": "process_file" + }, + "md5": { + "key": "file.hashes.MD5", + "object": "process_file" + }, + "parent_sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "parent_process_file" + }, + { + "key": "process.image_ref", + "object": "parent_process", + "references": "parent_process_file" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "parent_process_graph_id": [ + { + "key": "process.pid", + "object": "parent_process", + "transformer": "GraphIDToPID" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "triggering_process_graph_id": { + "key": "process.pid", + "object": "process", + "transformer": "GraphIDToPID" + }, + "registry_key": [ + { + "key": "windows-registry-key.key", + "object": "registry_key" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "registry_key" + } + ], + "domain_ioc": [ + { + "key": "domain-name.value", + "object": "domain-name" + }, + { + "key": "network-traffic.dst_ref", + "object": "network-traffic", + "references": "domain-name" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "network-traffic" + } + ], + "sha256_ioc": [ + { + "key": "file.hashes.SHA-256", + "object": "ioc_file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "ioc_file" + } + ], + "quarantined_file_sha256": [ + { + "key": "file.hashes.SHA-256", + "object": "quarantined_file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "quarantined_file" + } + ], + "md5_ioc": [ + { + "key": "file.hashes.MD5", + "object": "ioc_file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "ioc_file" + } + ], + "parent_md5": [ + { + "key": "file.hashes.MD5", + "object": "parent_process_file" + }, + { + "key": "process.image_ref", + "object": "parent_process", + "references": "parent_process_file" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "parent_cmdline": [ + { + "key": "process.command_line", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "external_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst-ipv4-addr" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "dst-ipv4-addr" + ], + "group": true + } + ], + "hostname": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "local_ip": [ + { + "key": "ipv4-addr.value", + "object": "local-ipv4-addr" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "local-ipv4-addr" + ], + "group": true + } + ], + "mac_address": [ + { + "key": "mac-addr.value", + "object": "mac-addr", + "transformer": "CrowdStrikeFormatMac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac-addr" + ], + "group": true + } + ], + "os_version": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "platform_name": { + "key": "x-oca-asset.os_platform", + "object": "host" + }, + "provider": { + "key": "x-oca-event.provider", + "object": "event" + }, + "device_id": { + "key": "x-crowdstrike.device_id", + "object": "x-crowdstrike" + }, + "detection_id": { + "key": "x-crowdstrike.detection_id", + "object": "x-crowdstrike" + }, + "scenario": { + "key": "x-crowdstrike.scenario", + "object": "x-crowdstrike" + }, + "technique": { + "key": "x-crowdstrike.technique", + "object": "x-crowdstrike" + }, + "tactic": { + "key": "x-crowdstrike.tactic", + "object": "x-crowdstrike" + }, + "tactic_id": { + "key": "x-crowdstrike.tactic_id", + "object": "x-crowdstrike" + }, + "severity": { + "key": "x-oca-event.severity", + "object": "event" + }, + "technique_id": { + "key": "x-crowdstrike.technique_id", + "object": "x-crowdstrike" + }, + "agent_local_time": { + "key": "x-crowdstrike.agent_local_time", + "object": "x-crowdstrike" + }, + "agent_version": { + "key": "x-crowdstrike.agent_version", + "object": "x-crowdstrike" + }, + "first_seen": { + "key": "x-crowdstrike.first_seen", + "object": "x-crowdstrike" + }, + "last_seen": { + "key": "x-crowdstrike.last_seen", + "object": "x-crowdstrike" + }, + "platform_id": { + "key": "x-crowdstrike.platform_id", + "object": "x-crowdstrike" + }, + "confidence": { + "key": "x-crowdstrike.confidence", + "object": "x-crowdstrike" + }, + "ioc_type": { + "key": "x-crowdstrike.ioc_type", + "object": "x-crowdstrike" + }, + "ioc_value": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "bios_manufacturer": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "bios_version": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "config_id_base": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "config_id_build": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "config_id_platform": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "product_type": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "product_type_desc": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "site_name": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "system_product_name": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + }, + "modified_timestamp": { + "key": "x-crowdstrike.ioc_value", + "object": "x-crowdstrike" + } +} diff --git a/stix_shifter_modules/datadog/test/stix_translation/test_datadog_stix_to_query.py b/stix_shifter_modules/datadog/test/stix_translation/test_datadog_stix_to_query.py index 0f3b539e8..0692ad0b2 100644 --- a/stix_shifter_modules/datadog/test/stix_translation/test_datadog_stix_to_query.py +++ b/stix_shifter_modules/datadog/test/stix_translation/test_datadog_stix_to_query.py @@ -9,7 +9,7 @@ def _test_query_assertions(query, queries): - assert query['queries'] == queries + assert set(query['queries']) == set(queries) class TestStixToQuery(unittest.TestCase, object): @@ -23,7 +23,8 @@ def test_ipv4_query(self, mock_time): '{"query": {"host": "abc.com", "start": 12345378, "end": 12345678}, "source": "processes"}', '{"query": {"id": 12345678, "start": 12345378, "end": 12345678}, "source": "processes"}', '{"query": {"id_str": "12345678", "start": 12345378, "end": 12345678}, "source": "processes"}'] - assert query['queries'] == queries + + _test_query_assertions(query, queries) @patch('time.time', return_value=12345678) def test_domain_name_query(self, mock_time): @@ -167,7 +168,8 @@ def test_query_from_multiple_observation_expressions_joined_by_AND(self, mock_ti '{"query": {"tags": "tags", "start": 12345378, "end": 12345678}, "source": "events"}', '{"query": {"alert_type": "alert_type", "start": 12345378, "end": 12345678}, "source": "processes"}', '{"query": {"tags": "tags", "start": 12345378, "end": 12345678}, "source": "processes"}'] - assert query['queries'][0] == queries[0] + + _test_query_assertions(query, queries) def test_query_comparator_operator_AND_with_same_field(self): stix_pattern = "[domain-name:value = 'abc.com'] START t'2021-01-28T12:24:01.009Z' STOP t'2021-07-25T12:54:01.009Z'" diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index 480f60ee0..871fc24b5 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -49,9 +49,9 @@ "created": ["file.created"], "size": ["file.size"], "hashes.MD5": ["file.hash.md5"], - "hashes.SHA-1": ["file.hash.sha1"], + "hashes.'SHA-1'": ["file.hash.sha1"], "hashes.'SHA-256'": ["file.hash.sha256"], - "hashes.SHA-512": ["file.hash.sha512"], + "hashes.'SHA-512'": ["file.hash.sha512"], "parent_directory_ref.path": ["file.directory"] } }, diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..b3c5d3aa9 --- /dev/null +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,545 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["source.ip", "destination.ip", "client.ip", "server.ip", "host.ip", "dns.resolved_ip"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["source.ip", "destination.ip", "client.ip", "server.ip", "host.ip", "dns.resolved_ip"] + } + }, + "mac-addr": { + "fields": { + "value": ["source.mac", "destination.mac", "client.mac", "server.mac", "host.mac"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["source.port", "client.port"], + "dst_port": ["destination.port", "server.port"], + "protocols[*]": ["network.transport", "network.type", "network.protocol"], + "src_ref.value": ["source.ip", "client.ip"], + "dst_ref.value": ["destination.ip", "server.ip"], + "src_byte_count": ["source.bytes", "client.bytes"], + "dst_byte_count": ["destination.bytes", "server.bytes"], + "src_packets": ["source.packets", "client.packets"], + "dst_packets": ["destination.packets", "server.packets"] + } + }, + "x-ecs-network": { + "fields": { + "inner.vlan.id": ["network.inner.vlan.id"], + "inner.vlan.name": ["network.inner.vlan.name"], + "name": ["network.name"], + "application": ["network.application"], + "direction": ["network.direction"], + "forwarded_ip": ["network.forwarded_ip"], + "community_id": ["network.community_id"] + } + }, + "artifact": { + "fields": { + "payload_bin": ["event.original"] + } + }, + "file": { + "fields": { + "name": ["file.name", "file.path", "process.name", "process.executable", "process.parent.name", "process.parent.executable"], + "ctime": ["file.created"], + "size": ["file.size"], + "hashes.MD5": ["file.hash.md5"], + "hashes.'SHA-1'": ["file.hash.sha1"], + "hashes.'SHA-256'": ["file.hash.sha256"], + "hashes.'SHA-512'": ["file.hash.sha512"], + "parent_directory_ref.path": ["file.directory"] + } + }, + "x-ecs-file": { + "fields": { + "accessed": ["file.accessed"], + "attributes": ["file.attributes"], + "ctime": ["file.ctime"], + "device": ["file.device"], + "drive_letter": ["file.drive_letter"], + "extension": ["file.extension"], + "gid": ["file.gid"], + "group": ["file.group"], + "inode": ["file.inode"], + "mime_type": ["file.mime_type"], + "mode": ["file.mode"], + "mtime": ["file.mtime"], + "owner": ["file.owner"], + "path": ["file.path"], + "target_path": ["file.target_path"], + "type": ["file.type"], + "uid": ["file.uid"], + "pe.company": ["file.pe.company"], + "pe.description": ["file.pe.description"], + "pe.file_version": ["file.pe.file_version"], + "pe.original_file_name": ["file.pe.original_file_name"], + "pe.product": ["file.pe.product"], + "code_signature.exists": ["file.code_signature.exists"], + "code_signature.status": ["file.code_signature.status"], + "code_signature.subject_name": ["file.code_signature.subject_name"], + "code_signature.trusted": ["file.code_signature.trusted"], + "code_signature.valid": ["file.code_signature.valid"] + } + }, + "directory": { + "fields": { + "path": ["file.directory", "file.path"] + } + }, + "user-account":{ + "fields": { + "user_id": ["user.name", "user.id"], + "account_login": ["user.name"] + } + }, + "x-ecs-user": { + "fields": { + "domain": ["user.domain"], + "full_name": ["user.full_name"], + "hash": ["user.hash"], + "id": ["user.id"], + "group_domain": ["user.group.domain"], + "group_id": ["user.group.id"], + "group_name": ["user.group.name"] + } + }, + "process" : { + "fields": { + "command_line": ["process.command_line", "powershell.command.value"], + "created_time": ["process.start"], + "pid": ["process.pid", "process.ppid", "process.parent.pid", "process.parent.ppid"], + "creator_user_ref.user_id": ["user.name"], + "parent_ref.pid": ["process.ppid", "process.parent.ppid"], + "parent_ref.name": ["process.parent.name"], + "image_ref.name": ["process.executable", "process.parent.executable"], + "x_ttp_tags": ["tags"] + } + }, + "x-ecs-process": { + "fields": { + "args": ["process.args"], + "args_count": ["process.args_count"], + "executable": ["process.executable"], + "entity_id": ["process.entity_id"], + "exit_code": ["process.exit_code"], + "thread.id": ["process.thread.id"], + "thread.name": ["process.thread.name"], + "title": ["process.title"], + "uptime": ["process.uptime"], + "working_directory": ["process.working_directory"], + "parent.args": ["process.parent.args"], + "parent.args_count": ["process.parent.args_count"], + "parent.entity_id": ["process.parent.entity_id"], + "parent.exit_code": ["process.parent.exit_code"], + "parent.pgid": ["process.parent.pgid"], + "parent.thread.id": ["process.parent.thread.id"], + "parent.thread.name": ["process.parent.thread.name"], + "parent.title": ["process.parent.title"], + "parent.uptime": ["process.parent.uptime"], + "parent.working_directory": ["process.parent.working_directory"], + "pe.company": ["process.pe.company"], + "pe.description": ["process.pe.description"], + "pe.file_version": ["process.pe.file_version"], + "pe.original_file_name": ["process.pe.original_file_name"], + "pe.product": ["process.pe.product"], + "code_signature.exists": ["process.code_signature.exists"], + "code_signature.status": ["process.code_signature.status"], + "code_signature.subject_name": ["process.code_signature.subject_name"], + "code_signature.trusted": ["process.code_signature.trusted"], + "code_signature.valid": ["process.code_signature.valid"], + "parent.code_signature.exists": ["process.parent.code_signature.exists"], + "parent.code_signature.status": ["process.parent.code_signature.status"], + "parent.code_signature.subject_name": ["process.parent.code_signature.subject_name"], + "parent.code_signature.trusted": ["process.parent.code_signature.trusted"], + "parent.code_signature.valid": ["process.parent.code_signature.valid"] + } + }, + "url": { + "fields": { + "value": ["url.original"] + } + }, + "domain-name": { + "fields": { + "value": ["url.domain", "dns.question.name", "dns.question.registered_domain", "host.hostname"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["registry.key"] + } + }, + "software": { + "fields": { + "name": ["agent.name"] + } + }, + "autonomous-system": { + "fields": { + "value": ["client.as.organization.name", "server.as.organization.name", "source.as.organization.name", "destination.as.organization.name"], + "number": ["client.as.number", "server.as.number", "source.as.number", "destination.as.number"] + } + }, + "email-addr": { + "fields": { + "name": ["user.email"] + } + }, + "x-oca-event": { + "fields": { + "action": ["event.action"], + "id": ["event.id"], + "category": ["event.category"], + "code": ["event.code"], + "created": ["event.created"], + "dataset": ["event.dataset"], + "duration": ["event.duration"], + "end": ["event.end"], + "hash": ["event.hash"], + "ingested": ["event.ingested"], + "kind": ["event.kind"], + "module": ["event.module"], + "outcome": ["event.outcome"], + "provider": ["event.provider"], + "risk_score": ["event.risk_score"], + "risk_score_norm": ["event.risk_score_norm"], + "sequence": ["event.sequence"], + "severity": ["event.severity"], + "start": ["event.start"], + "timezone": ["event.timezone"], + "type": ["event.type"], + "url": ["event.url"], + "original": ["message", "powershell.file.script_block_text"], + "process_ref.pid": ["process.pid"], + "process_ref.name": ["process.name"], + "process_ref.command_line": ["process.command_line", "powershell.command.value"], + "process_ref.image_ref.name": ["file.name", "process.executable"], + "process_ref.parent_ref.pid": ["process.ppid", "process.parent.ppid"], + "process_ref.parent_ref.command_line": ["process.parent.command_line"], + "process_ref.creator_user_ref.user_id": ["user.name"], + "parent_process_ref.pid": ["process.ppid", "process.parent.ppid"], + "parent_process_ref.command_line": ["process.parent.command_line"], + "domain_ref.value": ["url.domain", "dns.question.name", "dns.question.registered_domain", "host.hostname"], + "file_ref.name": ["file.name"], + "host_ref.hostname": ["host.hostname"], + "host_ref.name": ["host.name"], + "registry_ref.key": ["registry.key", "registry.path"] + } + }, + "x-ecs-cloud": { + "fields": { + "account.id": ["cloud.account.id"], + "availability_zone": ["cloud.availability_zone"], + "instance.id": ["cloud.instance.id"], + "instance.name": ["cloud.instance.name"], + "machine.type": ["cloud.machine.type"], + "provider": ["cloud.provider"], + "region": ["cloud.region"] + } + }, + "x-ecs-container": { + "fields": { + "id": ["container.id"], + "image.name": ["container.image.name"], + "image.tag": ["container.image.tag"], + "labels": ["container.labels"], + "name": ["container.name"], + "runtime": ["container.runtime"] + } + }, + "x-ecs-dll": { + "fields": { + "name": ["dll.name"], + "path": ["dll.path"], + "pe.company": ["dll.pe.company"], + "pe.description": ["dll.pe.description"], + "pe.file_version": ["dll.pe.file_version"], + "pe.original_file_name": ["dll.pe.original_file_name"], + "pe.product": ["dll.pe.product"], + "code_signature.exists": ["dll.code_signature.exists"], + "code_signature.status": ["dll.code_signature.status"], + "code_signature.subject_name": ["dll.code_signature.subject_name"], + "code_signature.trusted": ["dll.code_signature.trusted"], + "code_signature.valid": ["dll.code_signature.valid"] + } + }, + "x-ecs-dns": { + "fields": { + "answers_class": ["dns.answers.class"], + "answers_data": ["dns.answers.data"], + "answers_name": ["dns.answers.name"], + "answers_ttl": ["dns.answers.ttl"], + "answers_type": ["dns.answers.type"], + "header_flags": ["dns.header_flags"], + "id": ["dns.id"], + "op_code": ["dns.op_code"], + "question_class": ["dns.question.class"], + "question_name": ["dns.question.name"], + "question_registered_domain": ["dns.question.registered_domain"], + "question_subdomain": ["dns.question.subdomain"], + "question_top_level_domain": ["dns.question.top_level_domain"], + "question_type": ["dns.question.type"], + "resolved_ip": ["dns.resolved_ip"], + "response_code": ["dns.response_code"], + "type": ["dns.type"] + } + }, + "x-ecs": { + "fields": { + "version": ["ecs.version"] + } + }, + "x-ecs-error": { + "fields": { + "code": ["error.code"], + "id": ["error.id"], + "message": ["error.message"], + "stack_trace": ["error.stack_trace"], + "type": ["error.type"] + } + }, + "x-ecs-group": { + "fields": { + "domain": ["group.domain"], + "id": ["group.id"], + "name": ["group.name"] + } + }, + "x-oca-asset": { + "fields": { + "architecture": ["host.architecture"], + "domain": ["host.domain"], + "hostname": ["host.hostname"], + "id": ["host.id"], + "ip": ["host.ip"], + "mac": ["host.mac"], + "name": ["host.name"], + "type": ["host.type"], + "uptime": ["host.uptime"], + "os.name": ["host.os.name"], + "os.platform": ["host.os.platform"], + "os.version": ["host.os.version"] + } + }, + "x-ecs-http": { + "fields": { + "request_body_bytes": ["http.request.body.bytes"], + "request_body_content": ["http.request.body.content"], + "request_bytes": ["http.request.bytes"], + "request_method": ["http.request.method"], + "request_referrer": ["http.request.referrer"], + "response_body_bytes": ["http.response.body.bytes"], + "response_body_content": ["http.response.body.content"], + "response_bytes": ["http.response.bytes"], + "response_status_code": ["http.response.method"], + "version": ["http.version"] + } + }, + "x-ecs-log": { + "fields": { + "level": ["log.level"], + "logger": ["log.logger"], + "origin_file_line": ["log.origin.file.line"], + "origin_file_name": ["log.origin.file.name"], + "origin_function": ["log.origin.function"], + "original": ["log.original"], + "syslog_facility_code": ["log.syslog.facility.code"], + "syslog_facility_name": ["log.syslog.facility.name"], + "syslog_priority": ["log.syslog.priority"], + "severity_syslog_code": ["log.syslog.severity.code"], + "severity_syslog_name": ["log.syslog.severity.name"] + } + }, + "x-ecs-observer": { + "fields": { + "egress.zone": ["observer.egress.zone"], + "egress.interface.alias": ["observer.egress.interface.alias"], + "egress.interface.id": ["observer.egress.interface.id"], + "egress.interface.name": ["observer.egress.interface.name"], + "hostname": ["observer.hostname"], + "ingress.zone": ["observer.ingress.zone"], + "ingress.interface.alias": ["observer.ingress.interface.alias"], + "ingress.interface.id": ["observer.ingress.interface.id"], + "ingress.interface.name": ["observer.ingress.interface.name"], + "ip": ["observer.ip"], + "mac": ["observer.mac"], + "name": ["observer.name"], + "product": ["observer.product"], + "serial_number": ["observer.serial_number"], + "type": ["observer.type"], + "vendor": ["observer.vendor"], + "version": ["observer.version"], + "os.name": ["observer.os.name"], + "os.platform": ["observer.os.platform"], + "os.version": ["observer.os.version"] + } + }, + "x-ecs-organization": { + "fields": { + "id": ["organization.id"], + "name": ["organization.name"] + } + }, + "x-ecs-pe": { + "fields": { + "company": ["dll.pe.company", "process.pe.company", "file.pe.company"], + "description": ["dll.pe.description", "process.pe.description", "file.pe.description"], + "file_version": ["dll.pe.file_version", "process.pe.file_version", "file.pe.file_version"], + "original_file_name": ["dll.pe.original_file_name", "process.pe.original_file_name", "file.pe.original_file_name"], + "product": ["dll.pe.product", "process.pe.product", "file.pe.product"] + } + }, + "x-ecs-related": { + "fields": { + "hash": ["related.hash"], + "ip": ["related.ip"], + "user": ["related.user"] + } + }, + "x-ecs-rule": { + "fields": { + "author": ["rule.author"], + "category": ["rule.category"], + "description": ["rule.description"], + "id": ["rule.id"], + "license": ["rule.license"], + "name": ["rule.name"], + "reference": ["rule.reference"], + "ruleset": ["rule.ruleset"], + "uuid": ["rule.uuid"], + "version": ["rule.version"] + } + }, + "x-ecs-service": { + "fields": { + "id": ["service.id"], + "name": ["service.name"], + "state": ["service.state"], + "type": ["service.type"], + "version": ["service.version"] + } + }, + "x-ecs-threat": { + "fields": { + "framework": ["threat.framework"], + "tactic_id": ["threat.tactic.id"], + "tactic_name": ["threat.tactic.name"], + "tactic_reference": ["threat.tactic.reference"], + "technique_id": ["threat.technique.id"], + "technique_name": ["threat.technique.name"], + "technique_reference": ["threat.technique.reference"] + } + }, + "x-ecs-trace": { + "fields": { + "id": ["trace.id"] + } + }, + "x-ecs-transaction": { + "fields": { + "id": ["transaction.id"] + } + }, + "x-ecs-user-agent": { + "fields": { + "name": ["user_agent.name"], + "original": ["user_agent.original"], + "version": ["user_agent.version"], + "device_name": ["user_agent.device.name"] + } + }, + "x-ecs-vulnerability": { + "fields": { + "category": ["vulnerability.category"], + "classification": ["vulnerability.classification"], + "description": ["vulnerability.description"], + "enumeration": ["vulnerability.enumeration"], + "id": ["vulnerability.id"], + "reference": ["vulnerability.reference"], + "report_id": ["vulnerability.report_id"], + "severity": ["vulnerability.severity"], + "scanner_vendor": ["vulnerability.scanner.vendor"], + "score_base": ["vulnerability.score.base"], + "score_environmental": ["vulnerability.score.environmental"], + "score_temporal": ["vulnerability.score.temporal"], + "score_version": ["vulnerability.score.version"] + } + }, + "x-ecs-source": { + "fields": { + "address": ["source.address"], + "domain": ["source.domain"], + "nat.ip": ["source.nat.ip"], + "nat.port": ["source.nat.port"], + "registered_domain": ["source.registered_domain"], + "top_level_domain": ["source.top_level_domain"], + "geo.city_name": ["source.geo.city_name"], + "geo.continent_name": ["source.geo.continent_name"], + "geo.country_iso_code": ["source.geo.country_iso_code"], + "geo.country_name": ["source.geo.country_name"], + "geo.location": ["source.geo.location"], + "geo.name": ["source.geo.name"], + "geo.region_iso_code": ["source.geo.region_iso_code"], + "geo.region_name": ["source.geo.region_name"] + } + }, + "x-ecs-destination": { + "fields": { + "address": ["destination.address"], + "domain": ["destination.domain"], + "nat.ip": ["destination.nat.ip"], + "nat.port": ["destination.nat.port"], + "registered_domain": ["destination.registered_domain"], + "top_level_domain": ["destination.top_level_domain"], + "geo.city_name": ["destination.geo.city_name"], + "geo.continent_name": ["destination.geo.continent_name"], + "geo.country_iso_code": ["destination.geo.country_iso_code"], + "geo.country_name": ["destination.geo.country_name"], + "geo.location": ["destination.geo.location"], + "geo.name": ["destination.geo.name"], + "geo.region_iso_code": ["destination.geo.region_iso_code"], + "geo.region_name": ["destination.geo.region_name"] + } + }, + "x-ecs-client": { + "fields": { + "address": ["client.address"], + "domain": ["client.domain"], + "nat.ip": ["client.nat.ip"], + "nat.port": ["client.nat.port"], + "registered_domain": ["client.registered_domain"], + "top_level_domain": ["client.top_level_domain"], + "geo.city_name": ["client.geo.city_name"], + "geo.continent_name": ["client.geo.continent_name"], + "geo.country_iso_code": ["client.geo.country_iso_code"], + "geo.country_name": ["client.geo.country_name"], + "geo.location": ["client.geo.location"], + "geo.name": ["client.geo.name"], + "geo.region_iso_code": ["client.geo.region_iso_code"], + "geo.region_name": ["client.geo.region_name"] + } + }, + "x-ecs-server": { + "fields": { + "address": ["server.address"], + "domain": ["server.domain"], + "nat.ip": ["server.nat.ip"], + "nat.port": ["server.nat.port"], + "registered_domain": ["server.registered_domain"], + "top_level_domain": ["server.top_level_domain"], + "geo.city_name": ["server.geo.city_name"], + "geo.continent_name": ["server.geo.continent_name"], + "geo.country_iso_code": ["server.geo.country_iso_code"], + "geo.country_name": ["server.geo.country_name"], + "geo.location": ["server.geo.location"], + "geo.name": ["server.geo.name"], + "geo.region_iso_code": ["server.geo.region_iso_code"], + "geo.region_name": ["server.geo.region_name"] + } + } +} diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..6d9b22b48 --- /dev/null +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,2331 @@ +{ + "@timestamp": [ + { + "key": "first_observed", + "cybox": false + }, + { + "key": "last_observed", + "cybox": false + } + ], + "source": { + "ip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], + "port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "mac": [ + { + "key": "mac-addr.value", + "object": "src_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + } + ], + "bytes": { + "key": "network-traffic.src_byte_count", + "object": "nt" + }, + "packets": { + "key": "network-traffic.src_packets", + "object": "nt" + }, + "as": { + "number": [ + { + "key": "autonomous-system.number", + "object": "autonomous-system" + }, + { + "key": "ipv4-addr.belongs_to_refs", + "object": "src_ip", + "references": ["autonomous-system"] + } + ], + "organization": { + "name" : { + "key": "autonomous-system.name" + } + } + }, + "address": { + "key": "x-ecs-source.address", + "object": "source" + }, + "domain": { + "key": "x-ecs-source.domain", + "object": "source" + }, + "nat": { + "ip": { + "key": "x-ecs-source.nat_ip", + "object": "source" + }, + "port": { + "key": "x-ecs-source.nat_port", + "object": "source" + } + }, + "registered_domain": { + "key": "x-ecs-source.registered_domain", + "object": "source" + }, + "top_level_domain": { + "key": "x-ecs-source.top_level_domain", + "object": "source" + }, + "geo": { + "city_name": { + "key": "x-ecs-source.geo_city_name", + "object": "source" + }, + "continent_name": { + "key": "x-ecs-source.geo_continent_name", + "object": "source" + }, + "country_iso_code": { + "key": "x-ecs-source.geo_country_iso_code", + "object": "source" + }, + "country_name": { + "key": "x-ecs-source.geo_country_name", + "object": "source" + }, + "location": { + "key": "x-ecs-source.geo_location", + "object": "source" + }, + "name": { + "key": "x-ecs-source.geo_name", + "object": "source" + }, + "region_iso_code": { + "key": "x-ecs-source.geo_region_iso_code", + "object": "source" + }, + "region_name": { + "key": "x-ecs-source.geo_region_name", + "object": "source" + } + } + }, + "client": { + "ip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], + "port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "bytes": { + "key": "network-traffic.src_byte_count", + "object": "nt" + }, + "packets": { + "key": "network-traffic.src_packets", + "object": "nt" + }, + "mac": [ + { + "key": "mac-addr.value", + "object": "src_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + } + ], + "as": { + "number": [ + { + "key": "autonomous-system.number", + "object": "autonomous-system" + }, + { + "key": "ipv4-addr.belongs_to_refs", + "object": "src_ip", + "references": ["autonomous-system"] + } + ], + "organization": { + "name" : { + "key": "autonomous-system.name", + "object": "autonomous-system" + } + } + }, + "address": { + "key": "x-ecs-client.address", + "object": "client" + }, + "domain": { + "key": "x-ecs-client.domain", + "object": "client" + }, + "nat": { + "ip": { + "key": "x-ecs-client.nat_ip", + "object": "client" + }, + "port": { + "key": "x-ecs-client.nat_port", + "object": "client" + } + }, + "registered_domain": { + "key": "x-ecs-client.registered_domain", + "object": "client" + }, + "top_level_domain": { + "key": "x-ecs-client.top_level_domain", + "object": "client" + }, + "geo": { + "city_name": { + "key": "x-ecs-client.geo_city_name", + "object": "client" + }, + "continent_name": { + "key": "x-ecs-client.geo_continent_name", + "object": "client" + }, + "country_iso_code": { + "key": "x-ecs-client.geo_country_iso_code", + "object": "client" + }, + "country_name": { + "key": "x-ecs-client.geo_country_name", + "object": "client" + }, + "location": { + "key": "x-ecs-client.geo_location", + "object": "client" + }, + "name": { + "key": "x-ecs-client.geo_name", + "object": "client" + }, + "region_iso_code": { + "key": "x-ecs-client.geo_region_iso_code", + "object": "client" + }, + "region_name": { + "key": "x-ecs-client.geo_region_name", + "object": "client" + } + } + }, + "destination": { + "ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "mac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + } + ], + "bytes": { + "key": "network-traffic.dst_byte_count", + "object": "nt" + }, + "packets": { + "key": "network-traffic.dst_packets", + "object": "nt" + }, + "as": { + "number": [ + { + "key": "autonomous-system.number", + "object": "autonomous-system" + }, + { + "key": "ipv4-addr.belongs_to_refs", + "object": "src_ip", + "references": ["autonomous-system"] + } + ], + "organization": { + "name" : { + "key": "autonomous-system.name" + } + } + }, + "address": { + "key": "x-ecs-destination.address", + "object": "destination" + }, + "domain": { + "key": "x-ecs-destination.domain", + "object": "destination" + }, + "nat": { + "ip": { + "key": "x-ecs-destination.nat_ip", + "object": "destination" + }, + "port": { + "key": "x-ecs-destination.nat_port", + "object": "destination" + } + }, + "registered_domain": { + "key": "x-ecs-destination.registered_domain", + "object": "destination" + }, + "top_level_domain": { + "key": "x-ecs-destination.top_level_domain", + "object": "destination" + }, + "geo": { + "city_name": { + "key": "x-ecs-destination.geo_city_name", + "object": "destination" + }, + "continent_name": { + "key": "x-ecs-destination.geo_continent_name", + "object": "destination" + }, + "country_iso_code": { + "key": "x-ecs-destination.geo_country_iso_code", + "object": "destination" + }, + "country_name": { + "key": "x-ecs-destination.geo_country_name", + "object": "destination" + }, + "location": { + "key": "x-ecs-destination.geo_location", + "object": "destination" + }, + "name": { + "key": "x-ecs-destination.geo_name", + "object": "destination" + }, + "region_iso_code": { + "key": "x-ecs-destination.geo_region_iso_code", + "object": "destination" + }, + "region_name": { + "key": "x-ecs-destination.geo_region_name", + "object": "destination" + } + } + }, + "server": { + "ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "bytes": { + "key": "network-traffic.src_byte_count", + "object": "nt" + }, + "packets": { + "key": "network-traffic.src_packets", + "object": "nt" + }, + "mac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + } + ], + "as": { + "number": [ + { + "key": "autonomous-system.number", + "object": "autonomous-system" + }, + { + "key": "ipv4-addr.belongs_to_refs", + "object": "dst_ip", + "references": ["autonomous-system"] + } + ], + "organization": { + "name" : { + "key": "autonomous-system.name" + } + } + }, + "address": { + "key": "x-ecs-server.address", + "object": "server" + }, + "domain": { + "key": "x-ecs-server.domain", + "object": "server" + }, + "nat": { + "ip": { + "key": "x-ecs-server.nat_ip", + "object": "server" + }, + "port": { + "key": "x-ecs-server.nat_port", + "object": "server" + } + }, + "registered_domain": { + "key": "x-ecs-server.registered_domain", + "object": "server" + }, + "top_level_domain": { + "key": "x-ecs-server.top_level_domain", + "object": "server" + }, + "geo": { + "city_name": { + "key": "x-ecs-server.geo_city_name", + "object": "server" + }, + "continent_name": { + "key": "x-ecs-server.geo_continent_name", + "object": "server" + }, + "country_iso_code": { + "key": "x-ecs-server.geo_country_iso_code", + "object": "server" + }, + "country_name": { + "key": "x-ecs-server.geo_country_name", + "object": "server" + }, + "location": { + "key": "x-ecs-server.geo_location", + "object": "server" + }, + "name": { + "key": "x-ecs-server.geo_name", + "object": "server" + }, + "region_iso_code": { + "key": "x-ecs-server.geo_region_iso_code", + "object": "server" + }, + "region_name": { + "key": "x-ecs-server.geo_region_name", + "object": "server" + } + } + }, + "network": { + "transport": [ + { + "key": "network-traffic.protocols", + "object": "nt", + "group": "True", + "transformer": "ToLowercaseArray" + }, + { + "key": "process.opened_connection_refs", + "object": "process", + "references": [ + "nt" + ] + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "type": [ + { + "key": "network-traffic.protocols", + "object": "nt", + "group": "True", + "transformer": "ToLowercaseArray" + }, + { + "key": "process.opened_connection_refs", + "object": "process", + "references": [ + "nt" + ] + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "protocol": [ + { + "key": "network-traffic.protocols", + "object": "nt", + "group": "True", + "transformer": "ToLowercaseArray" + }, + { + "key": "process.opened_connection_refs", + "object": "process", + "references": [ + "nt" + ] + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "vlan": { + "id": { + "key": "x-ecs-network.vlan_id", + "object": "network" + }, + "name": { + "key": "x-ecs-network.vlan_name", + "object": "network" + } + }, + "inner": { + "vlan": { + "id": { + "key": "x-ecs-network.inner_vlan_id", + "object": "network" + }, + "name": { + "key": "x-ecs-network.inner_vlan_name", + "object": "network" + } + } + }, + "name": { + "key": "x-ecs-network.name", + "object": "network" + }, + "application": { + "key": "x-ecs-network.application", + "object": "network" + }, + "direction": { + "key": "x-ecs-network.direction", + "object": "network" + }, + "forwarded_ip": { + "key": "x-ecs-network.forwarded_ip", + "object": "network" + }, + "community_id": { + "key": "x-ecs-network.community_id", + "object": "network" + } + }, + "event": { + "original": [ + { + "key": "artifact.payload_bin", + "transformer": "ToBase64", + "object": "artifact" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "artifact" + } + ], + "mime_type_event": { + "key": "artifact.mime_type", + "object": "artifact" + }, + "action": { + "key": "x-oca-event.action", + "object": "event" + }, + "id": { + "key": "x-oca-event.id", + "object": "event" + }, + "category": { + "key": "x-oca-event.category", + "object": "event" + }, + "code": { + "key": "x-oca-event.code", + "object": "event" + }, + "created": { + "key": "x-oca-event.created", + "object": "event" + }, + "dataset": { + "key": "x-oca-event.dataset", + "object": "event" + }, + "duration": { + "key": "x-oca-event.duration", + "object": "event" + }, + "end": { + "key": "x-oca-event.end", + "object": "event" + }, + "hash": { + "key": "x-oca-event.hash", + "object": "event" + }, + "ingested": { + "key": "x-oca-event.ingested", + "object": "event" + }, + "kind": { + "key": "x-oca-event.kind", + "object": "event" + }, + "module": { + "key": "x-oca-event.module", + "object": "event" + }, + "outcome": { + "key": "x-oca-event.outcome", + "object": "event" + }, + "provider": { + "key": "x-oca-event.provider", + "object": "event" + }, + "reference": { + "key": "x-oca-event.reference", + "object": "event" + }, + "risk_score": { + "key": "x-oca-event.risk_score", + "object": "event" + }, + "risk_score_norm": { + "key": "x-oca-event.risk_score_norm", + "object": "event" + }, + "sequence": { + "key": "x-oca-event.sequence", + "object": "event" + }, + "severity": { + "key": "x-oca-event.severity", + "object": "event" + }, + "start": { + "key": "x-oca-event.start", + "object": "event" + }, + "timezone": { + "key": "x-oca-event.timezone", + "object": "event" + }, + "type": { + "key": "x-oca-event.event_type", + "object": "event" + }, + "url": [ + { + "key": "x-oca-event.url", + "object": "event" + }, { + "key": "domain-name.value", + "object": "event_domain", + "transformer": "ToDomainName" + }, { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "event_domain" + } + ] + }, + "url": { + "original": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "domain": [ + { + "key": "domain-name.value", + "object": "domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain" + } + ] + }, + "process": { + "start": { + "key": "process.created_time", + "object": "process" + }, + "pid": [ + { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "code_signature": { + "exists": { + "key": "x-ecs-process.code_signature_exists", + "object": "x_process" + }, + "subject_name": { + "key": "x-ecs-process.code_signature_subject_name", + "object": "x_process" + } + }, + "ppid": [ + { + "key": "process.pid", + "object": "process_parent", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "process_parent" + } + ], + "pe": { + "company": { + "key": "x-ecs-process.pe_company", + "object": "x_process" + }, + "description": { + "key": "x-ecs-process.pe_description", + "object": "x_process" + }, + "file_version": { + "key": "x-ecs-process.pe_file_version", + "object": "x_process" + }, + "original_file_name": { + "key": "x-ecs-process.pe_original_file_name", + "object": "x_process" + }, + "product": { + "key": "x-ecs-process.pe_product", + "object": "x_process" + } + }, + "args": { + "key": "x-ecs-process.args", + "object": "x_process" + }, + "args_count": { + "key": "x-ecs-process.args_count", + "object": "x_process" + }, + "command_line": { + "key": "process.command_line", + "object": "process" + }, + "executable": [ + { + "key": "file.name", + "object": "process_executable", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "process_executable_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_executable" + }, + { + "key": "file.parent_directory_ref", + "object": "process_executable", + "references": "process_executable_dir" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "entity_id": { + "key": "x-ecs-process.entity_id", + "object": "x_process" + }, + "exit_code": { + "key": "x-ecs-process.exit_code", + "object": "x_process" + }, + "parent": { + "args": { + "key": "x-ecs-process.parent_args", + "object": "x_process" + }, + "args_count": { + "key": "x-ecs-process.parent_args_count", + "object": "x_process" + }, + "command_line": { + "key": "process.command_line", + "object": "process_parent" + }, + "entity_id": { + "key": "x-ecs-process.parent_entity_id", + "object": "x_process" + }, + "exit_code": { + "key": "x-ecs-process.parent_exit_code", + "object": "x_process" + }, + "pgid": { + "key": "x-ecs-process.parent_pgid", + "object": "x_process" + }, + "pid": [ + { + "key": "process.pid", + "object": "process_parent", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "process_parent" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_parent" + } + ], + "ppid": { + "key": "x-ecs-process.parent_ppid", + "object": "x_process", + "transformer": "ToInteger" + }, + "start": { + "key": "process.created_time", + "object": "process_parent" + }, + "thread": { + "id": { + "key": "x-ecs-process.parent_thread_id", + "object": "x_process" + }, + "name": { + "key": "x-ecs-process.parent_thread_name", + "object": "x_process" + } + }, + "title": { + "key": "x-ecs-process.parent_title", + "object": "x_process" + }, + "uptime": { + "key": "x-ecs-process.parent_uptime", + "object": "x_process" + }, + "working_directory": { + "key": "x-ecs-process.parent_working_directory", + "object": "x_process" + }, + "executable": [ + { + "key": "file.name", + "object": "process_parent_executable", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "process_parent_executable_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.image_ref", + "object": "process_parent", + "references": "process_parent_executable" + }, + { + "key": "file.parent_directory_ref", + "object": "process_parent_executable", + "references": "process_parent_executable_dir" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "process_parent" + } + ] + }, + "pgid": { + "key": "x-ecs-process.exit_code", + "object": "x_process" + }, + "thread": { + "id": { + "key": "x-ecs-process.thread_id", + "object": "x_process" + }, + "name": { + "key": "x-ecs-process.thread_name", + "object": "x_process" + } + }, + "title": { + "key": "x-ecs-process.title", + "object": "x_process" + }, + "uptime": { + "key": "x-ecs-process.uptime", + "object": "x_process" + }, + "working_directory": { + "key": "x-ecs-process.working_directory", + "object": "x_process" + } + }, + "user": { + "name": [ + { + "key": "user-account.user_id", + "object": "user" + }, + { + "key": "user-account.account_login", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "user" + } + ], + "email": [ + { + "key": "email-addr.value", + "object": "email" + }, + { + "key": "email-addr.belongs_to_ref", + "object": "email", + "references": "user" + } + ], + "domain": { + "key": "x-ecs-user.domain", + "object": "x_user" + }, + "full_name": { + "key": "x-ecs-user.full_name", + "object": "x_user" + }, + "hash": { + "key": "x-ecs-user.hash", + "object": "x_user" + }, + "id": [ + { + "key": "x-ecs-user.id", + "object": "x_user" + }, + { + "key": "user-account.user_id", + "object": "user" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "user" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "user" + } + ], + "group": { + "domain": { + "key": "x-ecs-user.group_domain", + "object": "x_user" + }, + "id": { + "key": "x-ecs-user.group_id", + "object": "x_user" + }, + "name": { + "key": "x-ecs-user.group_name", + "object": "x_user" + } + } + }, + "agent": { + "name": [ + { + "key": "software.name" + }, + { + "key": "x-oca-event.agent", + "object": "event" + } + ], + "type": { + "key": "software.vendor" + }, + "version": { + "key": "software.version" + } + }, + "cloud": { + "account": { + "id": { + "key": "x-ecs-cloud.account_id", + "object": "cloud" + } + }, + "availability_zone": { + "key": "x-ecs-cloud.availability_zone", + "object": "cloud" + }, + "instance": { + "id": { + "key": "x-ecs-cloud.instance_id", + "object": "cloud" + }, + "name": { + "key": "x-ecs-cloud.instance_name", + "object": "cloud" + } + }, + "machine": { + "type": { + "key": "x-ecs-cloud.machine_type", + "object": "cloud" + } + }, + "provider": { + "key": "x-ecs-cloud.provider", + "object": "cloud" + }, + "region": { + "key": "x-ecs-cloud.region", + "object": "cloud" + } + }, + "container": { + "id": { + "key": "x-ecs-container.id", + "object": "container" + }, + "image": { + "name": { + "key": "x-ecs-container.image_name", + "object": "container" + }, + "tag": { + "key": "x-ecs-container.image_tag", + "object": "container" + } + }, + "labels": { + "key": "x-ecs-container.labels", + "object": "container" + }, + "name": { + "key": "x-ecs-container.name", + "object": "container" + }, + "runtime": { + "key": "x-ecs-container.runtime", + "object": "container" + } + }, + "dll":{ + "name": { + "key": "x-ecs-dll.name", + "object": "dll" + }, + "path": { + "key": "x-ecs-dll.path", + "object": "dll" + }, + "pe": { + "company": { + "key": "x-ecs-dll.pe_company", + "object": "dll" + }, + "description": { + "key": "x-ecs-dll.pe_description", + "object": "dll" + }, + "file_version": { + "key": "x-ecs-dll.pe_file_version", + "object": "dll" + }, + "original_file_name": { + "key": "x-ecs-dll.pe_original_file_name", + "object": "dll" + }, + "product": { + "key": "x-ecs-dll.pe_product", + "object": "dll" + } + }, + "code_signature": { + "exists": { + "key": "x-ecs-dll.code_signature_exists", + "object": "dll" + }, + "subject_name": { + "key": "x-ecs-dll.code_signature_subject_name", + "object": "dll" + } + }, + "hash": { + "sha256": { + "key": "x-ecs-dll.hashes.SHA-256", + "object": "dll" + }, + "sha1": { + "key": "x-ecs-dll.hashes.SHA-1", + "object": "dll" + }, + "md5": { + "key": "x-ecs-dll.hashes.MD5", + "object": "dll" + }, + "sha512": { + "key": "x-ecs-dll.hashes.SHA-512", + "object": "dll" + } + } + }, + "dns": { + "answers": + { + "key": "network-traffic.extensions.dns-ext.answers", + "object": "nt" + }, + "header_flags": { + "key": "network-traffic.extensions.dns-ext.header_flags", + "object": "nt" + }, + "id": { + "key": "network-traffic.extensions.dns-ext.dns_id", + "object": "nt" + }, + "op_code": { + "key": "network-traffic.extensions.dns-ext.op_code", + "object": "nt" + }, + "question": { + "class": { + "key": "network-traffic.extensions.dns-ext.question.class", + "object": "nt" + }, + "name": [ + { + "key": "domain-name.value", + "object": "dns_question_domain" + }, + { + "key": "network-traffic.extensions.dns-ext.question.domain_ref", + "object": "nt", + "references": "dns_question_domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "dns_question_domain" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "registered_domain": [ + { + "key": "domain-name.value", + "object": "dns_registered_domain" + }, + { + "key": "network-traffic.extensions.dns-ext.question.registered_domain_ref", + "object": "nt", + "references": "dns_registered_domain" + } + ], + "subdomain": { + "key": "network-traffic.extensions.dns-ext.question.subdomain", + "object": "nt" + }, + "top_level_domain": { + "key": "network-traffic.extensions.dns-ext.question.top_level_domain", + "object": "dns" + }, + "type": { + "key": "network-traffic.extensions.dns-ext.question.type", + "object": "dns" + } + }, + "resolved_ip": [ + { + "key": "ipv4-addr.value", + "object": "resolved_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "resolved_ipv6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-event.ip_refs", + "object": "event", + "references": ["resolved_ip", "resolved_ipv6"], + "unwrap": true + }, + { + "key": "network-traffic.extensions.dns-ext.resolved_ip_refs", + "object": "nt", + "references": ["resolved_ip", "resolved_ipv6"], + "unwrap": true + } + ], + "response_code": { + "key": "network-traffic.extensions.dns-ext.response_code", + "object": "nt" + }, + "type": { + "key": "network-traffic.extensions.dns-ext.type", + "object": "nt" + } + }, + "ecs": { + "version": { + "key": "x-ecs.version", + "object": "ecs" + } + }, + "error": { + "code": { + "key": "x-ecs-error.code", + "object": "error" + }, + "id": { + "key": "x-ecs-error.id", + "object": "error" + }, + "message": { + "key": "x-ecs-error.message", + "object": "error" + }, + "stack_trace": { + "key": "x-ecs-error.stack_trace", + "object": "error" + }, + "type": { + "key": "x-ecs-error.type", + "object": "error" + } + }, + "file": { + "name": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + } + ], + "created": { + "key": "file.ctime", + "object": "file" + }, + "directory": [ + { + "key": "directory.path", + "object": "directory" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + } + ], + "size": { + "key" : "file.size", + "object": "file", + "transformer": "ToInteger" + }, + "hash": { + "sha256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "sha1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "md5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "sha512": { + "key": "file.hashes.SHA-512", + "object": "file" + } + }, + "pe": { + "company": { + "key": "x-ecs-file.pe_company", + "object": "x_file" + }, + "description": { + "key": "x-ecs-file.pe_description", + "object": "x_file" + }, + "file_version": { + "key": "x-ecs-file.pe_file_version", + "object": "x_file" + }, + "original_file_name": { + "key": "x-ecs-file.pe_original_file_name", + "object": "x_file" + }, + "product": { + "key": "x-ecs-file.pe_product", + "object": "x_file" + } + }, + "code_signature": { + "exists": { + "key": "x-ecs-file.code_signature_exists", + "object": "x_file" + }, + "subject_name": { + "key": "x-ecs-file.code_signature_subject_name", + "object": "x_file" + } + }, + "accessed": { + "key": "x-ecs-file.accessed", + "object": "x_file" + }, + "attributes": { + "key": "x-ecs-file.attributes", + "object": "x_file" + }, + "ctime": { + "key": "x-ecs-file.ctime", + "object": "x_file" + }, + "device": { + "key": "x-ecs-file.device", + "object": "x_file" + }, + "drive_letter": { + "key": "x-ecs-file.drive_letter", + "object": "x_file" + }, + "extension": { + "key": "x-ecs-file.extension", + "object": "x_file" + }, + "gid": { + "key": "x-ecs-file.gid", + "object": "x_file" + }, + "group": { + "key": "x-ecs-file.group", + "object": "x_file" + }, + "inode": { + "key": "x-ecs-file.inode", + "object": "x_file" + }, + "mime_type": { + "key": "x-ecs-file.mime_type", + "object": "x_file" + }, + "mode": { + "key": "x-ecs-file.mode", + "object": "x_file" + }, + "mtime": { + "key": "x-ecs-file.mtime", + "object": "x_file" + }, + "owner": { + "key": "x-ecs-file.owner", + "object": "x_file" + }, + "path": { + "key": "x-ecs-file.path", + "object": "x_file" + }, + "target_path": { + "key": "x-ecs-file.target_path", + "object": "x_file" + }, + "type": { + "key": "x-ecs-file.type", + "object": "x_file" + }, + "uid": { + "key": "x-ecs-file.uid", + "object": "x_file" + } + }, + "group": { + "domain": { + "key": "x-ecs-group.domain", + "object": "group" + }, + "id": { + "key": "x-ecs-group.id", + "object": "group" + }, + "name": { + "key": "x-ecs-group.name", + "object": "group" + } + }, + "host": { + "architecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "domain": { + "key": "x-oca-asset.domain", + "object": "host" + }, + "hostname": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "id": { + "key": "x-oca-asset.id", + "object": "host" + }, + "ip": [ + { + "key": "ipv4-addr.value", + "object": "host_ip", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ipv6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": ["host_ip", "host_ipv6"], + "unwrap": true + } + ], + "mac": [ + { + "key": "mac-addr.value", + "object": "host_mac", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": "host_mac", + "unwrap": true + } + ], + "name": [ + { + "key": "x-oca-asset.name", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "type": { + "key": "x-oca-asset.host_type", + "object": "host" + }, + "uptime": { + "key": "x-oca-asset.uptime", + "object": "host" + }, + "geo": { + "city_name": { + "key": "x-oca-asset.geo_city_name", + "object": "host" + }, + "continent_name": { + "key": "x-oca-asset.geo_continent_name", + "object": "host" + }, + "country_iso_code": { + "key": "x-oca-asset.geo_country_iso_code", + "object": "host" + }, + "country_name": { + "key": "x-oca-asset.geo_country_name", + "object": "host" + }, + "location": { + "key": "x-oca-asset.geo_location", + "object": "host" + }, + "name": { + "key": "x-oca-asset.geo_name", + "object": "host" + }, + "region_iso_code": { + "key": "x-oca-asset.geo_region_iso_code", + "object": "host" + }, + "region_name": { + "key": "x-oca-asset.geo_region_name", + "object": "host" + } + }, + "os": { + "name": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "platform": { + "key": "x-oca-asset.os_platform", + "object": "host" + }, + "version": { + "key": "x-oca-asset.os_version", + "object": "host" + } + }, + "user": { + "domain": { + "key": "x-oca-asset.user_domain", + "object": "host" + }, + "email": { + "key": "x-oca-asset.user_email", + "object": "host" + }, + "full_name": { + "key": "x-oca-asset.user_full_name", + "object": "host" + }, + "hash": { + "key": "x-oca-asset.user_hash", + "object": "host" + }, + "id": { + "key": "x-oca-asset.user_id", + "object": "host" + }, + "name": { + "key": "x-oca-asset.user_name", + "object": "host" + }, + "group": { + "domain": { + "key": "x-oca-asset.user_group_domain", + "object": "host" + }, + "id": { + "key": "x-oca-asset.user_group_id", + "object": "host" + }, + "name": { + "key": "x-oca-asset.user_group_name", + "object": "host" + } + } + } + }, + "http": { + "request": { + "body": { + "bytes": { + "key": "x-ecs-http.request_body_bytes", + "object": "http" + }, + "content": { + "key": "x-ecs-http.request_body_content", + "object": "http" + } + }, + "bytes": { + "key": "x-ecs-http.request_bytes", + "object": "http" + }, + "method": { + "key": "x-ecs-http.request_method", + "object": "http" + }, + "referrer": { + "key": "x-ecs-http.request_referrer", + "object": "http" + } + }, + "response": { + "body": { + "bytes": { + "key": "x-ecs-http.response_body_bytes", + "object": "http" + }, + "content": { + "key": "x-ecs-http.response_body_content", + "object": "http" + } + }, + "bytes": { + "key": "x-ecs-http.response_bytes", + "object": "http" + }, + "status_code": { + "key": "x-ecs-http.response_status_code", + "object": "http" + } + }, + "version": { + "key": "x-ecs-http.version", + "object": "http" + } + }, + "log": { + "level": { + "key": "x-ecs-log.level", + "object": "log" + }, + "logger": { + "key": "x-ecs-log.logger", + "object": "log" + }, + "origin": { + "file": { + "line": { + "key": "x-ecs-log.origin_file_line", + "object": "log" + }, + "name": { + "key": "x-ecs-log.origin_file_name", + "object": "log" + } + }, + "function": { + "key": "x-ecs-log.origin_function", + "object": "log" + } + }, + "original": { + "key": "x-ecs-log.original", + "object": "log" + }, + "syslog": { + "facility": { + "code": { + "key": "x-ecs-log.syslog_facility_code", + "object": "log" + }, + "name": { + "key": "x-ecs-log.syslog_facility_name", + "object": "log" + } + }, + "priority": { + "key": "x-ecs-log.syslog_priority", + "object": "log" + }, + "severity": { + "code": { + "key": "x-ecs-log.severity_syslog_code", + "object": "log" + }, + "name": { + "key": "x-ecs-log.severity_syslog_name", + "object": "log" + } + } + } + }, + "observer": { + "egress": { + "zone": { + "key": "x-ecs-observer.egress_zone", + "object": "observer" + }, + "interface": { + "alias": { + "key": "x-ecs-observer.egress_interface_alias", + "object": "observer" + }, + "id": { + "key": "x-ecs-observer.egress_interface_id", + "object": "observer" + }, + "name": { + "key": "x-ecs-observer.egress_interface_name", + "object": "observer" + } + }, + "vlan": { + "id": { + "key": "x-ecs-observer.egress_vlan_id", + "object": "observer" + }, + "name": { + "key": "x-ecs-observer.egress_vlan_name", + "object": "observer" + } + } + }, + "hostname": { + "key": "x-ecs-observer.hostname", + "object": "observer" + }, + "ingress": { + "zone": { + "key": "x-ecs-observer.ingress_zone", + "object": "observer" + }, + "interface": { + "alias": { + "key": "x-ecs-observer.ingress_interface_alias", + "object": "observer" + }, + "id": { + "key": "x-ecs-observer.ingress_interface_id", + "object": "observer" + }, + "name": { + "key": "x-ecs-observer.ingress_interface_name", + "object": "observer" + } + }, + "vlan": { + "id": { + "key": "x-ecs-observer.ingress_vlan_id", + "object": "observer" + }, + "name": { + "key": "x-ecs-observer.ingress_vlan_name", + "object": "observer" + } + } + }, + "ip": { + "key": "x-ecs-observer.ip", + "object": "observer" + }, + "mac": { + "key": "x-ecs-observer.mac", + "object": "observer" + }, + "name": { + "key": "x-ecs-observer.name", + "object": "observer" + }, + "product": { + "key": "x-ecs-observer.product", + "object": "observer" + }, + "serial_number": { + "key": "x-ecs-observer.serial_number", + "object": "observer" + }, + "type": { + "key": "x-ecs-observer.type", + "object": "observer" + }, + "vendor": { + "key": "x-ecs-observer.vendor", + "object": "observer" + }, + "version": { + "key": "x-ecs-observer.version", + "object": "observer" + }, + "os": { + "name": { + "key": "x-ecs-observer.os_name", + "object": "observer" + }, + "platform": { + "key": "x-ecs-observer.os_platform", + "object": "observer" + }, + "version": { + "key": "x-ecs-observer.os_version", + "object": "observer" + } + }, + "geo": { + "city_name": { + "key": "x-ecs-observer.geo_city_name", + "object": "observer" + }, + "continent_name": { + "key": "x-ecs-observer.geo_continent_name", + "object": "observer" + }, + "country_iso_code": { + "key": "x-ecs-observer.geo_country_iso_code", + "object": "observer" + }, + "country_name": { + "key": "x-ecs-observer.geo_country_name", + "object": "observer" + }, + "location": { + "key": "x-ecs-observer.geo_location", + "object": "observer" + }, + "name": { + "key": "x-ecs-observer.geo_name", + "object": "observer" + }, + "region_iso_code": { + "key": "x-ecs-observer.geo_region_iso_code", + "object": "observer" + }, + "region_name": { + "key": "x-ecs-observer.geo_region_name", + "object": "observer" + } + } + }, + "organization": { + "id": { + "key": "x-ecs-organization.id", + "object": "organization" + }, + "name": { + "key": "x-ecs-organization.name", + "object": "organization" + } + }, + "package": { + "name": { + "key": "software.name" + }, + "version": { + "key": "software.version" + } + }, + "registry": { + "key": { + "key": "x-ecs-registry.key", + "object": "registry" + }, + "data": { + "bytes": { + "key": "x-ecs-registry.data_bytes", + "object": "registry" + }, + "strings": { + "key": "x-ecs-registry.data_strings", + "object": "registry" + }, + "type": { + "key": "x-ecs-registry.data_type", + "object": "registry" + } + }, + "hive": { + "key": "x-ecs-registry.hive", + "object": "registry" + }, + "path": [ + { + "key": "x-ecs-registry.path", + "object": "registry" + }, + { + "key": "windows-registry-key.key", + "object": "winregistry", + "transformer": "PathToStixRegistryKey" + }, + { + "key": "windows-registry-key.values", + "object": "winregistry", + "transformer": "PathToStixRegistryValue" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "winregistry" + } + ], + "value": { + "key": "x-ecs-registry.value", + "object": "registry" + } + }, + "related": { + "hash": { + "key": "x-ecs-related.hash", + "object": "related" + }, + "ip": { + "key": "x-ecs-related.ip", + "object": "related" + }, + "user": { + "key": "x-ecs-related.user", + "object": "related" + } + }, + "rule": { + "author": { + "key": "x-ecs-rule.author", + "object": "rule" + }, + "category": { + "key": "x-ecs-rule.category", + "object": "rule" + }, + "description": { + "key": "x-ecs-rule.description", + "object": "rule" + }, + "id": { + "key": "x-ecs-rule.id", + "object": "rule" + }, + "license": { + "key": "x-ecs-rule.license", + "object": "rule" + }, + "name": { + "key": "x-ecs-rule.name", + "object": "rule" + }, + "reference": { + "key": "x-ecs-rule.reference", + "object": "rule" + }, + "ruleset": { + "key": "x-ecs-rule.ruleset", + "object": "rule" + }, + "uuid": { + "key": "x-ecs-rule.uuid", + "object": "rule" + }, + "version": { + "key": "x-ecs-rule.version", + "object": "rule" + } + }, + "service": { + "id": { + "key": "x-ecs-service.id", + "object": "service" + }, + "name": { + "key": "x-ecs-service.name", + "object": "service" + }, + "state": { + "key": "x-ecs-service.state", + "object": "service" + }, + "type": { + "key": "x-ecs-service.type", + "object": "service" + }, + "version": { + "key": "x-ecs-service.version", + "object": "service" + }, + "ephemeral_id": { + "key": "x-ecs-service.ephemeral_id", + "object": "service" + }, + "node": { + "name": { + "key": "x-ecs-service.node_name", + "object": "service" + } + } + }, + "threat": { + "framework": { + "key": "x-ecs-threat.framework", + "object": "threat" + }, + "tactic": { + "id": { + "key": "x-ecs-threat.tactic_id", + "object": "threat" + }, + "name": { + "key": "x-ecs-threat.tactic_name", + "object": "threat" + }, + "reference": { + "key": "x-ecs-threat.tactic_reference", + "object": "threat" + } + }, + "technique": { + "id": { + "key": "x-ecs-threat.technique_id", + "object": "threat" + }, + "name": { + "key": "x-ecs-threat.technique_name", + "object": "threat" + }, + "reference": { + "key": "x-ecs-threat.technique_reference", + "object": "threat" + } + } + }, + "tls": { + "client": { + "issuer": { + "key": "x509-certificate.issuer", + "object": "client" + }, + "hash": { + "sha256": { + "key": "x509-certificate.hashes.SHA-256", + "object": "client" + }, + "sha1": { + "key": "x509-certificate.hashes.SHA-1", + "object": "client" + }, + "md5": { + "key": "x509-certificate.hashes.MD5", + "object": "client" + } + }, + "not_after": { + "key": "x509-certificate.validity_not_after", + "object": "client" + }, + "not_before": { + "key": "x509-certificate.validity_not_before", + "object": "client" + }, + "subject": { + "key": "x509-certificate.subject", + "object": "client" + }, + "certificate": { + "key": "x-ecs-tls.client_certificate", + "object": "tls" + }, + "certificate_chain": { + "key": "x-ecs-tls.client_certificate_chain", + "object": "tls" + }, + "ja3": { + "key": "x-ecs-tls.client_ja3", + "object": "tls" + }, + "supported_ciphers": { + "key": "x-ecs-tls.client_supported_ciphers", + "object": "tls" + } + }, + "server": { + "issuer": { + "key": "x509-certificate.issuer", + "object": "server" + }, + "hash": { + "sha256": { + "key": "x509-certificate.hashes.SHA-256", + "object": "server" + }, + "sha1": { + "key": "x509-certificate.hashes.SHA-1", + "object": "server" + }, + "md5": { + "key": "x509-certificate.hashes.MD5", + "object": "server" + } + }, + "not_after": { + "key": "x509-certificate.validity_not_after", + "object": "server" + }, + "not_before": { + "key": "x509-certificate.validity_not_before", + "object": "server" + }, + "subject": { + "key": "x509-certificate.subject", + "object": "server" + }, + "certificate": { + "key": "x-ecs-tls.server_certificate", + "object": "tls" + }, + "certificate_chain": { + "key": "x-ecs-tls.server_certificate_chain", + "object": "tls" + }, + "ja3s": { + "key": "x-ecs-tls.server_ja3s", + "object": "tls" + } + }, + "cipher": { + "key": "x-ecs-tls.cipher", + "object": "tls" + }, + "curve": { + "key": "x-ecs-tls.curve", + "object": "tls" + }, + "established": { + "key": "x-ecs-tls.established", + "object": "tls" + }, + "next_protocol": { + "key": "x-ecs-tls.next_protocol", + "object": "tls" + }, + "resumed": { + "key": "x-ecs-tls.resumed", + "object": "tls" + }, + "version": { + "key": "x-ecs-tls.version", + "object": "tls" + }, + "version_protocol": { + "key": "x-ecs-tls.version_protocol", + "object": "tls" + } + }, + "trace": { + "id": { + "key": "x-ecs-trace.id", + "object": "trace" + } + }, + "transaction": { + "id": { + "key": "x-ecs-transaction.id", + "object": "transaction" + } + }, + "user_agent": { + "name": { + "key": "x-ecs-user_agent.name", + "object": "user_agent" + }, + "original": { + "key": "x-ecs-user_agent.original", + "object": "user_agent" + }, + "version": { + "key": "x-ecs-user_agent.version", + "object": "user_agent" + }, + "device": { + "name": { + "key": "x-ecs-user_agent.device_name", + "object": "user_agent" + } + } + }, + "vulnerability": { + "category": { + "key": "x-ecs-vulnerability.category", + "object": "vulnerability" + }, + "classification": { + "key": "x-ecs-vulnerability.classification", + "object": "vulnerability" + }, + "description": { + "key": "x-ecs-vulnerability.description", + "object": "vulnerability" + }, + "enumeration": { + "key": "x-ecs-vulnerability.enumeration", + "object": "vulnerability" + }, + "id": { + "key": "x-ecs-vulnerability.id", + "object": "vulnerability" + }, + "reference": { + "key": "x-ecs-vulnerability.reference", + "object": "vulnerability" + }, + "report_id": { + "key": "x-ecs-vulnerability.report_id", + "object": "vulnerability" + }, + "severity": { + "key": "x-ecs-vulnerability.severity", + "object": "vulnerability" + }, + "scanner": { + "vendor": { + "key": "x-ecs-vulnerability.scanner_vendor", + "object": "vulnerability" + } + }, + "score": { + "base": { + "key": "x-ecs-vulnerability.score_base", + "object": "vulnerability" + }, + "environmental": { + "key": "x-ecs-vulnerability.score_environmental", + "object": "vulnerability" + }, + "temporal": { + "key": "x-ecs-vulnerability.score_temporal", + "object": "vulnerability" + }, + "version": { + "key": "x-ecs-vulnerability.score_version", + "object": "vulnerability" + } + } + }, + "tags": { + "key": "process.x_ttp_tags", + "object": "process" + } +} diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 2edaf1664..066f5e2e0 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -637,7 +637,8 @@ }, "category": { "key": "x-oca-event.category", - "object": "event" + "object": "event", + "transformer": "ValueToList" }, "code": { "key": "x-oca-event.code", diff --git a/stix_shifter_modules/lang_en.json b/stix_shifter_modules/lang_en.json index 1c7f117cf..5b9f0ff99 100644 --- a/stix_shifter_modules/lang_en.json +++ b/stix_shifter_modules/lang_en.json @@ -11,6 +11,10 @@ "label": "Data source description", "description": "Assign a description to indicate the purpose of the connection" }, + "apphostId": { + "label": "Edge Gateway", + "description": "Specify an edge gateway host if the connection is to run on an edge gateway." + }, "options": { "result_limit": { "label": "Result Size Limit", diff --git a/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json index 085273d7b..39e5bf500 100644 --- a/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json @@ -72,7 +72,7 @@ "DeviceRegistryEvents.InitiatingProcessFileName", "DeviceRegistryEvents.InitiatingProcessParentFileName" ], - "hashes.SHA-1": [ + "hashes.'SHA-1'": [ "DeviceFileEvents.SHA1", "DeviceFileEvents.InitiatingProcessSHA1", "DeviceProcessEvents.SHA1", @@ -167,7 +167,7 @@ ], "child_refs.creator_user_ref.account_login": ["DeviceProcessEvents.AccountName"], "creator_user_ref.account_login": ["DeviceProcessEvents.InitiatingProcessAccountName"], - "binary_ref.hashes.SHA-1": [ + "binary_ref.hashes.'SHA-1'": [ "DeviceFileEvents.SHA1", "DeviceFileEvents.InitiatingProcessSHA1", "DeviceProcessEvents.SHA1", @@ -175,7 +175,7 @@ "DeviceNetworkEvents.InitiatingProcessSHA1", "DeviceRegistryEvents.InitiatingProcessSHA1" ], - "binary_ref.hashes.SHA-256": [ + "binary_ref.hashes.'SHA-256'": [ "DeviceFileEvents.SHA256", "DeviceFileEvents.InitiatingProcessSHA256", "DeviceProcessEvents.SHA256", @@ -186,10 +186,10 @@ "child_refs.binary_ref.hashes.MD5": [ "DeviceProcessEvents.MD5" ], - "child_refs.binary_ref.hashes.SHA-256": [ + "child_refs.binary_ref.hashes.'SHA-256'": [ "DeviceProcessEvents.SHA256" ], - "child_refs.binary_ref.hashes.SHA-1": [ + "child_refs.binary_ref.hashes.'SHA-1'": [ "DeviceProcessEvents.SHA1" ], "binary_ref.hashes.MD5": [ @@ -380,7 +380,7 @@ "DeviceRegistryEvents.InitiatingProcessCommandLine", "DeviceFileEvents.InitiatingProcessCommandLine" ], - "process_ref.binary_ref.hashes.SHA-256": [ + "process_ref.binary_ref.hashes.'SHA-256'": [ "DeviceProcessEvents.InitiatingProcessSHA256", "DeviceNetworkEvents.InitiatingProcessSHA256", "DeviceRegistryEvents.InitiatingProcessSHA256", @@ -407,7 +407,7 @@ "DeviceRegistryEvents.InitiatingProcessMD5", "DeviceFileEvents.InitiatingProcessMD5" ], - "process_ref.binary_ref.hashes.SHA-256": [ + "process_ref.binary_ref.hashes.'SHA-256'": [ "DeviceProcessEvents.InitiatingProcessSHA256", "DeviceNetworkEvents.InitiatingProcessSHA256", "DeviceRegistryEvents.InitiatingProcessSHA256", diff --git a/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..b8919e0ae --- /dev/null +++ b/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,453 @@ +{ + "ipv4-addr": { + "fields": { + "value": [ + "DeviceNetworkEvents.LocalIP", + "DeviceNetworkEvents.RemoteIP" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "DeviceNetworkEvents.LocalIP", + "DeviceNetworkEvents.RemoteIP" + ] + } + }, + "network-traffic": { + "fields": { + "src_port": [ + "DeviceNetworkEvents.LocalPort" + ], + "dst_port": [ + "DeviceNetworkEvents.RemotePort" + ], + "protocols[*]": [ + "DeviceNetworkEvents.Protocol" + ], + "src_ref.value": [ + "DeviceNetworkEvents.LocalIP", + "DeviceNetworkInfo.MacAddress" + ], + "dst_ref.value": [ + "DeviceNetworkEvents.RemoteIP" + ] + } + }, + "url": { + "fields": { + "value": [ + "DeviceNetworkEvents.RemoteUrl" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "DeviceFileEvents.DeviceName", + "DeviceProcessEvents.DeviceName", + "DeviceNetworkEvents.DeviceName", + "DeviceRegistryEvents.DeviceName", + "DeviceEvents.DeviceName", + "DeviceInfo.DeviceName", + "DeviceAlertEvents.DeviceName", + "DeviceImageLoadEvents.DeviceName", + "DeviceLogonEvents.DeviceName", + "DeviceNetworkInfo.DeviceName" + ] + } + }, + "file": { + "fields": { + "name": [ + "DeviceFileEvents.FileName", + "DeviceFileEvents.InitiatingProcessFileName", + "DeviceFileEvents.InitiatingProcessParentFileName", + "DeviceProcessEvents.FileName", + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceNetworkEvents.InitiatingProcessFileName", + "DeviceNetworkEvents.InitiatingProcessParentFileName", + "DeviceRegistryEvents.InitiatingProcessFileName", + "DeviceRegistryEvents.InitiatingProcessParentFileName" + ], + "hashes.'SHA-1'": [ + "DeviceFileEvents.'SHA1'", + "DeviceFileEvents.InitiatingProcessSHA1", + "DeviceProcessEvents.SHA1", + "DeviceProcessEvents.InitiatingProcessSHA1", + "DeviceNetworkEvents.InitiatingProcessSHA1", + "DeviceRegistryEvents.InitiatingProcessSHA1" + ], + "hashes.'SHA-256'": [ + "DeviceFileEvents.SHA256", + "DeviceFileEvents.InitiatingProcessSHA256", + "DeviceProcessEvents.SHA256", + "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceNetworkEvents.InitiatingProcessSHA256", + "DeviceRegistryEvents.InitiatingProcessSHA256" + ], + "hashes.MD5": [ + "DeviceFileEvents.MD5", + "DeviceFileEvents.InitiatingProcessMD5", + "DeviceProcessEvents.MD5", + "DeviceProcessEvents.InitiatingProcessMD5", + "DeviceNetworkEvents.InitiatingProcessMD5", + "DeviceRegistryEvents.InitiatingProcessMD5" + ], + "parent_directory_ref.path": [ + "DeviceFileEvents.FolderPath", + "DeviceFileEvents.InitiatingProcessFolderPath", + "DeviceProcessEvents.FolderPath", + "DeviceProcessEvents.InitiatingProcessFolderPath", + "DeviceNetworkEvents.InitiatingProcessFolderPath", + "DeviceRegistryEvents.InitiatingProcessFolderPath" + ] + } + }, + "process": { + "fields": { + "parent_ref.name": [ + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceNetworkEvents.InitiatingProcessFileName", + "DeviceRegistryEvents.InitiatingProcessFileName", + "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceNetworkEvents.InitiatingProcessParentFileName", + "DeviceRegistryEvents.InitiatingProcessParentFileName" + ], + "command_line": [ + "DeviceProcessEvents.ProcessCommandLine", + "DeviceProcessEvents.InitiatingProcessCommandLine", + "DeviceNetworkEvents.InitiatingProcessCommandLine", + "DeviceRegistryEvents.InitiatingProcessCommandLine" + ], + "pid": [ + "DeviceProcessEvents.ProcessId", + "DeviceProcessEvents.InitiatingProcessId", + "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceNetworkEvents.InitiatingProcessId", + "DeviceNetworkEvents.InitiatingProcessParentId", + "DeviceRegistryEvents.InitiatingProcessId", + "DeviceRegistryEvents.InitiatingProcessParentId", + "DeviceFileEvents.InitiatingProcessId", + "DeviceFileEvents.InitiatingProcessParentId" + ], + "parent_ref.pid": [ + "DeviceProcessEvents.InitiatingProcessId", + "DeviceNetworkEvents.InitiatingProcessId", + "DeviceRegistryEvents.InitiatingProcessId", + "DeviceFileEvents.InitiatingProcessId", + "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceNetworkEvents.InitiatingProcessParentId", + "DeviceRegistryEvents.InitiatingProcessParentId", + "DeviceFileEvents.InitiatingProcessParentId" + ], + "child_refs.pid": [ + "DeviceProcessEvents.ProcessId", + "DeviceProcessEvents.InitiatingProcessId", + "DeviceNetworkEvents.InitiatingProcessId", + "DeviceRegistryEvents.InitiatingProcessId", + "DeviceFileEvents.InitiatingProcessId" + ], + "creator_user_ref.user_id": [ + "DeviceProcessEvents.AccountSid", + "DeviceProcessEvents.InitiatingProcessAccountSid" + ], + "child_refs.creator_user_ref.account_login": ["DeviceProcessEvents.AccountName"], + "creator_user_ref.account_login": ["DeviceProcessEvents.InitiatingProcessAccountName"], + "image_ref.hashes.'SHA-1'": [ + "DeviceFileEvents.SHA1", + "DeviceFileEvents.InitiatingProcessSHA1", + "DeviceProcessEvents.SHA1", + "DeviceProcessEvents.InitiatingProcessSHA1", + "DeviceNetworkEvents.InitiatingProcessSHA1", + "DeviceRegistryEvents.InitiatingProcessSHA1" + ], + "image_ref.hashes.'SHA-256'": [ + "DeviceFileEvents.SHA256", + "DeviceFileEvents.InitiatingProcessSHA256", + "DeviceProcessEvents.SHA256", + "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceNetworkEvents.InitiatingProcessSHA256", + "DeviceRegistryEvents.InitiatingProcessSHA256" + ], + "child_refs.image_ref.hashes.MD5": [ + "DeviceProcessEvents.MD5" + ], + "child_refs.image_ref.hashes.'SHA-256'": [ + "DeviceProcessEvents.SHA256" + ], + "child_refs.image_ref.hashes.'SHA-1'": [ + "DeviceProcessEvents.SHA1" + ], + "image_ref.hashes.MD5": [ + "DeviceFileEvents.MD5", + "DeviceFileEvents.InitiatingProcessMD5", + "DeviceProcessEvents.MD5", + "DeviceProcessEvents.InitiatingProcessMD5", + "DeviceNetworkEvents.InitiatingProcessMD5", + "DeviceRegistryEvents.InitiatingProcessMD5" + ], + "created_time": [ + "DeviceProcessEvents.ProcessCreationTime", + "DeviceProcessEvents.InitiatingProcessParentCreationTime", + "DeviceNetworkEvents.InitiatingProcessCreationTime", + "DeviceNetworkEvents.InitiatingProcessParentCreationTime", + "DeviceRegistryEvents.InitiatingProcessCreationTime", + "DeviceRegistryEvents.InitiatingProcessParentCreationTime", + "DeviceFileEvents.InitiatingProcessCreationTime", + "DeviceFileEvents.InitiatingProcessParentCreationTime" + ], + "parent_ref.created_time": [ + "DeviceProcessEvents.InitiatingProcessParentCreationTime", + "DeviceNetworkEvents.InitiatingProcessParentCreationTime", + "DeviceRegistryEvents.InitiatingProcessParentCreationTime", + "DeviceFileEvents.InitiatingProcessParentCreationTime" + ], + "image_ref.parent_directory_ref.path": [ + "DeviceProcessEvents.InitiatingProcessFolderPath", + "DeviceNetworkEvents.InitiatingProcessFolderPath", + "DeviceRegistryEvents.InitiatingProcessFolderPath", + "DeviceFileEvents.InitiatingProcessFolderPath" + ], + "child_refs.image_ref.parent_directory_ref.path": [ + "DeviceProcessEvents.FolderPath" + ] + } + }, + "user-account": { + "fields": { + "user_id": [ + "DeviceProcessEvents.AccountSid", + "DeviceNetworkEvents.InitiatingProcessAccountSid", + "DeviceRegistryEvents.InitiatingProcessAccountSid" + ], + "account_login": [ + "DeviceProcessEvents.AccountName", + "DeviceNetworkEvents.InitiatingProcessAccountName", + "DeviceRegistryEvents.InitiatingProcessAccountName" + ] + } + }, + "windows-registry-key": { + "fields": { + "key": [ + "DeviceRegistryEvents.RegistryKey" + ], + "values[*]": [ + "DeviceRegistryEvents.RegistryValueName" + ] + } + }, + "mac-addr": { + "fields": { + "value": [ + "DeviceNetworkInfo.MacAddress" + ] + } + }, + "x-msatp": { + "fields": { + "computer_name": [ + "DeviceProcessEvents.DeviceName", + "DeviceNetworkEvents.DeviceName", + "DeviceRegistryEvents.DeviceName", + "DeviceFileEvents.DeviceName" + ], + "machine_id": [ + "DeviceProcessEvents.DeviceId", + "DeviceNetworkEvents.DeviceId", + "DeviceRegistryEvents.DeviceId", + "DeviceFileEvents.DeviceId" + ] + } + }, + "directory": { + "fields": { + "path": [ + "DeviceFileEvents.FolderPath", + "DeviceFileEvents.InitiatingProcessFolderPath", + "DeviceProcessEvents.FolderPath", + "DeviceProcessEvents.InitiatingProcessFolderPath", + "DeviceNetworkEvents.InitiatingProcessFolderPath", + "DeviceRegistryEvents.InitiatingProcessFolderPath" + ] + } + }, + "x-oca-asset": { + "fields": { + "domain": [ + "DeviceFileEvents.DeviceName", + "DeviceProcessEvents.DeviceName", + "DeviceNetworkEvents.DeviceName", + "DeviceRegistryEvents.DeviceName", + "DeviceEvents.DeviceName", + "DeviceInfo.DeviceName", + "DeviceAlertEvents.DeviceName", + "DeviceImageLoadEvents.DeviceName", + "DeviceLogonEvents.DeviceName", + "DeviceNetworkInfo.DeviceName" + ], + "hostname": [ + "DeviceFileEvents.DeviceName", + "DeviceProcessEvents.DeviceName", + "DeviceNetworkEvents.DeviceName", + "DeviceRegistryEvents.DeviceName", + "DeviceEvents.DeviceName", + "DeviceInfo.DeviceName", + "DeviceAlertEvents.DeviceName", + "DeviceImageLoadEvents.DeviceName", + "DeviceLogonEvents.DeviceName", + "DeviceNetworkInfo.DeviceName" + ], + "name": [ + "DeviceFileEvents.DeviceName", + "DeviceProcessEvents.DeviceName", + "DeviceNetworkEvents.DeviceName", + "DeviceRegistryEvents.DeviceName", + "DeviceEvents.DeviceName", + "DeviceInfo.DeviceName", + "DeviceAlertEvents.DeviceName", + "DeviceImageLoadEvents.DeviceName", + "DeviceLogonEvents.DeviceName", + "DeviceNetworkInfo.DeviceName" + ], + "ip": ["DeviceNetworkEvents.LocalIP"], + "os.name": [ + "DeviceInfo.OSPlatform" + ], + "os.platform": [ + "DeviceInfo.OSPlatform" + ] + } + }, + "x-oca-event": { + "fields": { + "process_ref.pid": [ + "DeviceProcessEvents.InitiatingProcessId", + "DeviceNetworkEvents.InitiatingProcessId", + "DeviceRegistryEvents.InitiatingProcessId", + "DeviceFileEvents.InitiatingProcessId" + ], + "process_ref.child_refs.pid": [ + "DeviceProcessEvents.ProcessId" + ], + "process_ref.name": [ + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceNetworkEvents.InitiatingProcessFileName", + "DeviceRegistryEvents.InitiatingProcessFileName", + "DeviceFileEvents.InitiatingProcessFileName" + ], + "process_ref.child_refs.name": [ + "DeviceProcessEvents.FileName" + ], + "process_ref.image_ref.name": [ + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceNetworkEvents.InitiatingProcessFileName", + "DeviceRegistryEvents.InitiatingProcessFileName", + "DeviceFileEvents.InitiatingProcessFileName" + ], + "process_ref.creator_user_ref.account_login": [ + "DeviceProcessEvents.AccountName" + ], + "process_ref.parent_ref.name": [ + "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceNetworkEvents.InitiatingProcessParentFileName", + "DeviceRegistryEvents.InitiatingProcessParentFileName", + "DeviceFileEvents.InitiatingProcessParentFileName" + ], + "process_ref.parent_ref.pid": [ + "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceNetworkEvents.InitiatingProcessParentId", + "DeviceRegistryEvents.InitiatingProcessParentId", + "DeviceFileEvents.InitiatingProcessParentId" + ], + "process_ref.command_line": [ + "DeviceProcessEvents.InitiatingProcessCommandLine", + "DeviceNetworkEvents.InitiatingProcessCommandLine", + "DeviceRegistryEvents.InitiatingProcessCommandLine", + "DeviceFileEvents.InitiatingProcessCommandLine" + ], + "process_ref.image_ref.hashes.'SHA-256'": [ + "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceNetworkEvents.InitiatingProcessSHA256", + "DeviceRegistryEvents.InitiatingProcessSHA256", + "DeviceFileEvents.InitiatingProcessSHA256" + ], + "process_ref.process_ref.creator_user_ref.account_login": [ + "DeviceProcessEvents.AccountName" + ], + "parent_process_ref.name": [ + "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceNetworkEvents.InitiatingProcessParentFileName", + "DeviceRegistryEvents.InitiatingProcessParentFileName", + "DeviceFileEvents.InitiatingProcessParentFileName" + ], + "parent_process_ref.pid": [ + "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceNetworkEvents.InitiatingProcessParentId", + "DeviceRegistryEvents.InitiatingProcessParentId", + "DeviceFileEvents.InitiatingProcessParentId" + ], + "process_ref.image_ref.hashes.MD5": [ + "DeviceProcessEvents.InitiatingProcessMD5", + "DeviceNetworkEvents.InitiatingProcessMD5", + "DeviceRegistryEvents.InitiatingProcessMD5", + "DeviceFileEvents.InitiatingProcessMD5" + ], + "process_ref.image_ref.hashes.'SHA-256'": [ + "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceNetworkEvents.InitiatingProcessSHA256", + "DeviceRegistryEvents.InitiatingProcessSHA256", + "DeviceFileEvents.InitiatingProcessSHA256" + ], + "process_ref.creator_user_ref.user_id": [ + "DeviceProcessEvents.InitiatingProcessAccountSid", + "DeviceNetworkEvents.InitiatingProcessAccountSid", + "DeviceRegistryEvents.InitiatingProcessAccountSid", + "DeviceFileEvents.InitiatingProcessAccountSid" + ], + "domain_ref.value": [ + "DeviceProcessEvents.DeviceName", + "DeviceNetworkEvents.DeviceName", + "DeviceRegistryEvents.DeviceName", + "DeviceFileEvents.DeviceName" + ], + "file_ref.name": [ + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceNetworkEvents.InitiatingProcessFileName", + "DeviceRegistryEvents.InitiatingProcessFileName" + ], + "registry_ref.key": [ + "DeviceRegistryEvents.RegistryKey" + ], + "host_ref.hostname": [ + "DeviceFileEvents.DeviceName", + "DeviceProcessEvents.DeviceName", + "DeviceNetworkEvents.DeviceName", + "DeviceRegistryEvents.DeviceName", + "DeviceEvents.DeviceName", + "DeviceInfo.DeviceName", + "DeviceAlertEvents.DeviceName", + "DeviceImageLoadEvents.DeviceName", + "DeviceLogonEvents.DeviceName", + "DeviceNetworkInfo.DeviceName" + ], + "host_ref.name": [ + "DeviceFileEvents.DeviceName", + "DeviceProcessEvents.DeviceName", + "DeviceNetworkEvents.DeviceName", + "DeviceRegistryEvents.DeviceName", + "DeviceEvents.DeviceName", + "DeviceInfo.DeviceName", + "DeviceAlertEvents.DeviceName", + "DeviceImageLoadEvents.DeviceName", + "DeviceLogonEvents.DeviceName", + "DeviceNetworkInfo.DeviceName" + ] + } + } +} diff --git a/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..7a64ab440 --- /dev/null +++ b/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,996 @@ +{ + "DeviceNetworkEvents": { + "Timestamp": [ + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "LocalIP": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], + "RemoteIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "LocalPort": [ + { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + } + ], + "RemotePort": [ + { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + } + ], + "Protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "FormatTCPProtocol" + }, + "RemoteUrl": { + "key": "url.value" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "nc_process_file" + }, + { + "key": "process.image_ref", + "object": "nc_process", + "references": "nc_process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "nc_process_file" + }, + { + "key": "process.image_ref", + "object": "nc_process", + "references": "nc_process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "nc_process_file" + }, + { + "key": "process.image_ref", + "object": "nc_process", + "references": "nc_process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "nc_process_file" + }, + { + "key": "process.image_ref", + "object": "nc_process", + "references": "nc_process_file" + } + ], + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "nc_parent_process_file" + }, + { + "key": "process.parent_ref", + "object": "nc_process", + "references": "nc_parent_process" + }, + { + "key": "process.image_ref", + "object": "nc_parent_process", + "references": "nc_parent_process_file" + } + ], + "InitiatingProcessId": { + "key": "process.pid", + "object": "nc_process", + "transformer": "ToInteger" + }, + "InitiatingProcessParentId": [ + { + "key": "process.pid", + "object": "nc_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "nc_process", + "references": "nc_parent_process" + } + ], + "InitiatingProcessCommandLine": { + "key": "process.command_line", + "object": "nc_process" + }, + "InitiatingProcessCreationTime": { + "key": "process.created_time", + "object": "nc_process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessParentCreationTime": [ + { + "key": "process.created_time", + "object": "nc_parent_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.parent_ref", + "object": "nc_process", + "references": "nc_parent_process" + } + ], + "InitiatingProcessAccountSid": [ + { + "key": "user-account.user_id", + "object": "nc_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "nc_process", + "references": "nc_process_creator" + } + ], + "InitiatingProcessAccountName": [ + { + "key": "user-account.account_login", + "object": "nc_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "nc_process", + "references": "nc_process_creator" + } + ], + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "nc_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "nc_process_file", + "references": "nc_process_file_dir" + }, + { + "key": "process.image_ref", + "object": "nc_process", + "references": "nc_process_file" + } + ], + "MacAddress": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "src_ip", + "references": ["mac"] + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "DeviceName": { + "key": "x-msatp.computer_name" + }, + "DeviceId": { + "key": "x-msatp.machine_id" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + } + }, + "DeviceProcessEvents": { + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "FileName": [ + { + "key": "file.name", + "object": "child_file" + }, + { + "key": "process.image_ref", + "object": "child_process", + "references": "child_file" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "FolderPath": [ + { + "key": "directory.path", + "object": "child_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "child_file", + "references": "child_file_dir" + } + ], + "SHA1": { + "key": "file.hashes.SHA-1", + "object": "child_file" + }, + "SHA256": { + "key": "file.hashes.SHA-256", + "object": "child_file" + }, + "MD5": { + "key": "file.hashes.MD5", + "object": "child_file" + }, + "ProcessId": [ + { + "key": "process.pid", + "object": "child_process", + "transformer": "ToInteger" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "ProcessCommandLine": [ + { + "key": "process.command_line", + "object": "child_process" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + } + ], + "ProcessCreationTime": [ + { + "key": "process.created_time", + "object": "child_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + } + ], + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "process_file" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "parent_process_file" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "process.image_ref", + "object": "parent_process", + "references": "parent_process_file" + } + ], + "InitiatingProcessId": { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + "InitiatingProcessParentId": [ + { + "key": "process.pid", + "object": "parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "InitiatingProcessCommandLine": { + "key": "process.command_line", + "object": "process" + }, + "InitiatingProcessCreationTime": { + "key": "process.created_time", + "object": "process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessParentCreationTime": [ + { + "key": "process.created_time", + "object": "parent_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "AccountSid": [ + { + "key": "user-account.user_id", + "object": "child_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "child_process", + "references": "child_process_creator" + } + ], + "InitiatingProcessAccountSid": [ + { + "key": "user-account.user_id", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "AccountName": [ + { + "key": "user-account.account_login", + "object": "child_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "child_process", + "references": "child_process_creator" + } + ], + "InitiatingProcessAccountName": [ + { + "key": "user-account.account_login", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "process_file" + } + ], + "DeviceName": { + "key": "x-msatp.computer_name" + }, + "DeviceId": { + "key": "x-msatp.machine_id" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + } + }, + "DeviceFileEvents": { + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "FileName": [ + { + "key": "file.name", + "object": "file" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + } + ], + "SHA1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "SHA256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "MD5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "fc_process_file" + }, + { + "key": "process.image_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "fc_process_file" + }, + { + "key": "process.image_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "fc_process_file" + }, + { + "key": "process.image_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "fc_process_file" + }, + { + "key": "process.image_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "fc_parent_process_file" + }, + { + "key": "process.parent_ref", + "object": "fc_process", + "references": "fc_parent_process" + }, + { + "key": "process.image_ref", + "object": "fc_parent_process", + "references": "fc_parent_process_file" + } + ], + "InitiatingProcessId": [ + { + "key": "process.pid", + "object": "fc_process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "fc_process" + } + ], + "InitiatingProcessParentId": [ + { + "key": "process.pid", + "object": "fc_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "fc_process", + "references": "fc_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "fc_parent_process" + } + ], + "InitiatingProcessCommandLine": { + "key": "process.command_line", + "object": "fc_process" + }, + "InitiatingProcessCreationTime": { + "key": "process.created_time", + "object": "fc_process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessParentCreationTime": [ + { + "key": "process.created_time", + "object": "fc_parent_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.parent_ref", + "object": "fc_process", + "references": "fc_parent_process" + } + ], + "InitiatingProcessAccountSid": [ + { + "key": "user-account.user_id", + "object": "fc_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "fc_process", + "references": "fc_process_creator" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "fc_process_creator" + } + ], + "InitiatingProcessAccountName": [ + { + "key": "user-account.account_login", + "object": "fc_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "fc_process", + "references": "fc_process_creator" + } + ], + "FolderPath": [ + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" + } + ], + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "fc_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_process_file", + "references": "fc_process_file_dir" + }, + { + "key": "process.image_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "DeviceId": { + "key": "x-oca-asset.device_id", + "object": "host" + }, + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + } + }, + "DeviceRegistryEvents": { + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "RegistryKey": { + "key": "windows-registry-key.key", + "object": "re_key" + }, + "RegistryValues": { + "key": "windows-registry-key.values", + "object": "re_key", + "transformer": "MsatpToRegistryValue" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "re_process_file" + }, + { + "key": "process.image_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "re_process_file" + }, + { + "key": "process.image_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "re_process_file" + }, + { + "key": "process.image_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "re_process_file" + }, + { + "key": "process.image_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "re_parent_process_file" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + }, + { + "key": "process.image_ref", + "object": "re_parent_process", + "references": "re_parent_process_file" + } + ], + "InitiatingProcessId": [ + { + "key": "process.pid", + "object": "re_process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "re_process" + } + ], + "InitiatingProcessParentId": [ + { + "key": "process.pid", + "object": "re_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "re_parent_process" + } + ], + "InitiatingProcessCommandLine": { + "key": "process.command_line", + "object": "re_process" + }, + "InitiatingProcessCreationTime": { + "key": "process.created_time", + "object": "re_process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessParentCreationTime": [ + { + "key": "process.created_time", + "object": "re_parent_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + } + ], + "InitiatingProcessAccountSid": [ + { + "key": "user-account.user_id", + "object": "re_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "re_process", + "references": "re_process_creator" + } + ], + "InitiatingProcessAccountName": [ + { + "key": "user-account.account_login", + "object": "re_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "re_process", + "references": "re_process_creator" + } + ], + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "re_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "re_process_file", + "references": "re_process_file_dir" + }, + { + "key": "process.image_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "DeviceId": { + "key": "x-oca-asset.device_id", + "object": "host" + }, + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + } + }, + "DeviceNetworkInfo": { + "IpAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip", + "unwrap": true, + "transformer": "MsatpIpTransformer" + }, + { + "key": "ipv6-addr.value", + "object": "host_ipv6", + "unwrap": true, + "transformer": "MsatpIpTransformer" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip", + "host_ipv6" + ], + "unwrap": true + } + ], + "MacAddress": [ + { + "key": "mac-addr.value", + "object": "host_mac" + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": "host_mac" + } + ] + } +} \ No newline at end of file diff --git a/stix_shifter_modules/mysql/stix_translation/json/from_stix_map.json b/stix_shifter_modules/mysql/stix_translation/json/from_stix_map.json index 1bea12713..e0c411a17 100644 --- a/stix_shifter_modules/mysql/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/mysql/stix_translation/json/from_stix_map.json @@ -19,7 +19,10 @@ "name": ["filename"], "hashes.'SHA-256'": ["sha256hash"], "hashes.MD5": ["md5hash"], - "parent_directory_ref.path": ["file_path"] + "parent_directory_ref.path": ["file_path"], + "created": ["file_created_time"], + "modified": ["file_modified_time"], + "accessed": ["file_accessed_time"] } }, "user-account": { @@ -29,7 +32,10 @@ }, "directory": { "fields": { - "path": ["file_path"] + "path": ["file_path"], + "created": ["directory_created_time"], + "modified": ["directory_modified_time"], + "accessed": ["directory_accessed_time"] } }, "network-traffic": { @@ -43,6 +49,15 @@ "end": ["entry_time"] } }, + "process": { + "fields": { + "pid": ["process_id"], + "name": ["process_name"], + "arguments": ["process_arguments"], + "created": ["process_created_time"], + "binary_ref.name": ["filename"] + } + }, "x-mysql": { "fields": { "system_name": ["system_name"], diff --git a/stix_shifter_modules/mysql/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/mysql/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..dec038912 --- /dev/null +++ b/stix_shifter_modules/mysql/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,65 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["source_ipaddr", "dest_ipaddr"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["source_ipaddr", "dest_ipaddr"] + } + }, + "url": { + "fields": { + "value": ["url"] + } + }, + "file": { + "fields": { + "name": ["filename"], + "hashes.'SHA-256'": ["sha256hash"], + "hashes.MD5": ["md5hash"], + "parent_directory_ref.path": ["file_path"], + "ctime": ["file_created_time"], + "mtime": ["file_modified_time"], + "atime": ["file_accessed_time"] + } + }, + "user-account": { + "fields": { + "user_id": ["username"] + } + }, + "directory": { + "fields": { + "path": ["file_path"], + "ctime": ["directory_created_time"], + "mtime": ["directory_modified_time"], + "atime": ["directory_accessed_time"] + } + }, + "network-traffic": { + "fields": { + "src_ref.value": ["source_ipaddr"], + "dst_ref.value": ["dest_ipaddr"], + "src_port": ["source_port"], + "dst_port": ["dest_port"], + "protocols[*]": ["protocol"], + "start": ["entry_time"], + "end": ["entry_time"] + } + }, + "process": { + "fields": { + "pid": ["process_id"], + "created_time": ["process_created_time"], + "image_ref.name": ["filename"] + } + }, + "x-mysql": { + "fields": { + "system_name": ["system_name"], + "severity": ["severity"] + } + } +} diff --git a/stix_shifter_modules/mysql/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/mysql/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..a17f5f146 --- /dev/null +++ b/stix_shifter_modules/mysql/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,153 @@ +{ + "source_ipaddr": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + } + ], + "dest_ipaddr": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + } + ], + "entry_time": [ + { + "key": "first_observed", + "transformer": "EpochToTimestamp", + "cybox": false + }, + { + "key": "last_observed", + "transformer": "EpochToTimestamp", + "cybox": false + } + ], + "url": { + "key": "url.value" + }, + "source_port": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "dest_port": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "username": [ + { + "key": "user-account.user_id", + "object": "useraccount" + } + ], + "filename": [ + { + "key": "file.name", + "object": "fl" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "fl" + } + ], + "sha256hash": { + "key": "file.hashes.SHA-256", + "object": "fl" + }, + "md5hash": { + "key": "file.hashes.MD5", + "object": "fl" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + "file_path": [ + { + "key": "directory.path", + "object": "file_directory" + }, + { + "key": "file.parent_directory_ref", + "object": "fl", + "references": "file_directory" + } + ], + "file_created_time": [ + { + "key": "file.ctime", + "object": "fl", + "transformer": "EpochToTimestamp" + } + ], + "file_modified_time": [ + { + "key": "file.mtime", + "object": "fl", + "transformer": "EpochToTimestamp" + } + ], + "file_accessed_time": [ + { + "key": "file.atime", + "object": "fl", + "transformer": "EpochToTimestamp" + } + ], + "directory_created_time": [ + { + "key": "directory.ctime", + "object": "file_directory", + "transformer": "EpochToTimestamp" + } + ], + "directory_modified_time": [ + { + "key": "directory.mtime", + "object": "file_directory", + "transformer": "EpochToTimestamp" + } + ], + "directory_accessed_time": [ + { + "key": "directory.atime", + "object": "file_directory", + "transformer": "EpochToTimestamp" + } + ], + "process_id": [ + { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + } + ], + "process_arguments": [ + { + "key": "process.arguments", + "object": "process" + } + ], + "process_created_time": [ + { + "key": "process.created_time", + "object": "process", + "transformer": "EpochToTimestamp" + } + ] + +} \ No newline at end of file diff --git a/stix_shifter_modules/mysql/stix_translation/json/to_stix_map.json b/stix_shifter_modules/mysql/stix_translation/json/to_stix_map.json index 217f54fcd..23e757a50 100644 --- a/stix_shifter_modules/mysql/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/mysql/stix_translation/json/to_stix_map.json @@ -56,6 +56,11 @@ { "key": "file.name", "object": "fl" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "fl" } ], "sha256hash": { @@ -81,5 +86,74 @@ "object": "fl", "references": "file_directory" } + ], + "file_created_time": [ + { + "key": "file.created", + "object": "fl", + "transformer": "EpochToTimestamp" + } + ], + "file_modified_time": [ + { + "key": "file.modified", + "object": "fl", + "transformer": "EpochToTimestamp" + } + ], + "file_accessed_time": [ + { + "key": "file.accessed", + "object": "fl", + "transformer": "EpochToTimestamp" + } + ], + "directory_created_time": [ + { + "key": "directory.created", + "object": "file_directory", + "transformer": "EpochToTimestamp" + } + ], + "directory_modified_time": [ + { + "key": "directory.modified", + "object": "file_directory", + "transformer": "EpochToTimestamp" + } + ], + "directory_accessed_time": [ + { + "key": "directory.accessed", + "object": "file_directory", + "transformer": "EpochToTimestamp" + } + ], + "process_id": [ + { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + } + ], + "process_name": [ + { + "key": "process.name", + "object": "process" + } + ], + "process_arguments": [ + { + "key": "process.arguments", + "object": "process" + } + ], + "process_created_time": [ + { + "key": "process.created", + "object": "process", + "transformer": "EpochToTimestamp" + } ] + } \ No newline at end of file diff --git a/stix_shifter_modules/mysql/stix_translation/query_constructor.py b/stix_shifter_modules/mysql/stix_translation/query_constructor.py index 27ecf588f..350f139a6 100644 --- a/stix_shifter_modules/mysql/stix_translation/query_constructor.py +++ b/stix_shifter_modules/mysql/stix_translation/query_constructor.py @@ -12,6 +12,8 @@ "dest_ipaddr": ["ipv4", "ipv4_cidr"], } +TIMESTAMP_STIX_PROPERTIES = ["created", "modified", "accessed", "ctime", "mtime", "atime", "created_time", "modifed_time"] + logger = logging.getLogger(__name__) @@ -82,6 +84,12 @@ def _format_start_stop_qualifier(self, expression, qualifier) -> str: qualified_query = "%s AND (entry_time >= %s OR entry_time <= %s)" % (expression, start, stop) return qualified_query + @classmethod + def _format_timestamp(self, value): + transformer = TimestampToMilliseconds() + value = re.sub("'", "", value) + return transformer.transform(value) + @staticmethod def _escape_value(value, comparator=None) -> str: if isinstance(value, str): @@ -111,6 +119,8 @@ def _parse_reference(self, stix_field, value_type, mapped_field, value, comparat @staticmethod def _parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array): + if stix_field in TIMESTAMP_STIX_PROPERTIES: + value = self._format_timestamp(value) comparison_string = "" is_reference_value = self._is_reference_value(stix_field) # Need to use expression.value to match against regex since the passed-in value has already been formated. diff --git a/stix_shifter_modules/mysql/tests/stix_translation/test_mysql_stix_to_query.py b/stix_shifter_modules/mysql/tests/stix_translation/test_mysql_stix_to_query.py index 110cdf732..44f4eb2de 100644 --- a/stix_shifter_modules/mysql/tests/stix_translation/test_mysql_stix_to_query.py +++ b/stix_shifter_modules/mysql/tests/stix_translation/test_mysql_stix_to_query.py @@ -4,8 +4,12 @@ import re -FROM_STIX_MAPPING_FILE = open('stix_shifter_modules/mysql/stix_translation/json/from_stix_map.json').read() -FROM_STIX_MAPPINGS = json.loads(FROM_STIX_MAPPING_FILE) +FROM_STIX_MAPPING_FILE_2_0 = open('stix_shifter_modules/mysql/stix_translation/json/from_stix_map.json').read() +FROM_STIX_MAPPINGS_2_0 = json.loads(FROM_STIX_MAPPING_FILE_2_0) +FROM_STIX_MAPPING_FILE_2_1 = open('stix_shifter_modules/mysql/stix_translation/json/stix_2_1/from_stix_map.json').read() +FROM_STIX_MAPPINGS_2_1 = json.loads(FROM_STIX_MAPPING_FILE_2_1) +EPOCH = 1634657528000 +TIMESTAMP = "'2021-10-19T15:32:08.000Z'" TEST_VALUES = { "source_ipaddr": "'0.0.0.0'", @@ -15,13 +19,23 @@ "sha256hash": "'sha256filehash'", "md5hash": "'md5filehash'", "file_path": "'C:/directory/'", + "directory_created_time": TIMESTAMP, + "directory_modified_time": TIMESTAMP, + "directory_accessed_time": TIMESTAMP, "username": "'admin'", "source_port": 1234, "dest_port": 5678, "protocol": "'tcp'", - "entry_time": "'2019-01-28T12:24:01.009Z'", + "entry_time": TIMESTAMP, "system_name": "'computer'", - "severity": 5 + "severity": 5, + "file_created_time": TIMESTAMP, + "file_modified_time": TIMESTAMP, + "file_accessed_time": TIMESTAMP, + "process_id": 12345, + "process_name": "'hackingAllTheThings'", + "process_arguments": "'some args'", + "process_created_time": TIMESTAMP } translation = stix_translation.StixTranslation() @@ -31,42 +45,45 @@ def _test_query_assertions(field, queries): for query in queries: assert field in query value = TEST_VALUES[field] - if field == "entry_time": - value = '1548678241009' + if value == TIMESTAMP: + value = EPOCH if not isinstance(value, str): value = str(value) assert value in query -def _translate_query(stix_pattern): - return translation.translate('mysql', 'query', '{}', stix_pattern) +def _translate_query(stix_pattern, options={}): + return translation.translate('mysql', 'query', '{}', stix_pattern, options) -def _add_single_quotes(stix_property): - stix_property = re.sub("SHA-256", "'SHA-256'", stix_property) - stix_property = re.sub("MD5", "'MD5'", stix_property) - return stix_property +def _test_mappings(mappings, stix_spec='2.0'): + for stix_object, value in mappings.items(): + for stix_property, field_list in value["fields"].items(): + field_count = len(field_list) + stix_pattern = "[" + for field in field_list: + test_value = TEST_VALUES.get(field) + if not test_value: + assert False, "'{}' datasource field missing from TEST_VALUES dictionary.".format(field) + stix_pattern += "{}:{} = {}".format(stix_object, stix_property, TEST_VALUES.get(field)) + if field_count > 1: + stix_pattern += " OR " + field_count -= 1 + stix_pattern += "]" + if stix_spec == "2.1": + pattern_translation = _translate_query(stix_pattern, {"stix_2.1": True}) + else: + pattern_translation = _translate_query(stix_pattern) + assert pattern_translation.get("queries"), "failed to translate {}".format(stix_pattern) + for field in field_list: + _test_query_assertions(field, pattern_translation["queries"]) class TestQueryTranslator(unittest.TestCase, object): - def test_all_mappings(self): - for stix_object, value in FROM_STIX_MAPPINGS.items(): - for stix_property, field_list in value["fields"].items(): - # if stix_object == 'file': - # stix_property = _add_single_quotes(stix_property) - field_count = len(field_list) - stix_pattern = "[" - for field in field_list: - test_value = TEST_VALUES.get(field) - if not test_value: - assert False, "'{}' datasource field missing from TEST_VALUES dictionary.".format(field) - stix_pattern += "{}:{} = {}".format(stix_object, stix_property, TEST_VALUES.get(field)) - if field_count > 1: - stix_pattern += " OR " - field_count -= 1 - stix_pattern += "]" - pattern_translation = _translate_query(stix_pattern) - assert pattern_translation.get("queries"), "failed to translate {}".format(stix_pattern) - for field in field_list: - _test_query_assertions(field, pattern_translation["queries"]) + def test_all_mappings_stix_2_0(self): + _test_mappings(FROM_STIX_MAPPINGS_2_0) + + + def test_all_mappings_stix_2_1(self): + _test_mappings(FROM_STIX_MAPPINGS_2_1, "2.1") diff --git a/stix_shifter_modules/qradar/stix_translation/json/aql_events_fields.json b/stix_shifter_modules/qradar/stix_translation/json/aql_events_fields.json index 7eb220db1..28e3a604c 100644 --- a/stix_shifter_modules/qradar/stix_translation/json/aql_events_fields.json +++ b/stix_shifter_modules/qradar/stix_translation/json/aql_events_fields.json @@ -51,13 +51,7 @@ "\"Process CommandLine\" as ProcessCommandLine", "ParentCommandLine", "TargetImage", - "\"Granted Access\" as GrantedAccess", - "\"Call Trace\" as CallTrace", "SourceImage", - "PipeName", - "StartModule", - "StartFunction", - "Signed", "Message", "\"Registry Value Name\" as RegistryValueName", "\"IMP Hash\" as IMPHash", diff --git a/stix_shifter_modules/qradar/stix_translation/json/events_from_stix_map.json b/stix_shifter_modules/qradar/stix_translation/json/events_from_stix_map.json index fa82b7d3e..dda9dd81b 100644 --- a/stix_shifter_modules/qradar/stix_translation/json/events_from_stix_map.json +++ b/stix_shifter_modules/qradar/stix_translation/json/events_from_stix_map.json @@ -95,13 +95,13 @@ "process": { "fields": { "pid": ["ProcessId"], - "name": ["ProcessName", "Image", "ParentImage"], - "binary_ref.name": ["Image"], - "binary_ref.parent_directory_ref.path": ["Image"], + "name": ["ProcessName", "Image", "ParentImage", "TargetImage"], + "binary_ref.name": ["Image", "TargetImage"], + "binary_ref.parent_directory_ref.path": ["Image", "TargetImage"], "parent_ref.binary_ref.name": ["ParentImage"], "command_line": ["ProcessCommandLine", "ParentCommandLine"], "parent_ref.command_line": ["ParentCommandLine"], - "extensions.windows-service-ext.service_dll_refs[*].name": ["ServiceFileName"] + "extensions.'windows-service-ext'.service_dll_refs[*].name": ["ServiceFileName"] } }, "x-oca-event": { @@ -114,7 +114,7 @@ "agent": ["logsourcename"], "provider": ["logsourcetypename"], "process_ref.command_line": ["ProcessCommandLine"], - "process_ref.binary_ref.name": ["Image"], + "process_ref.binary_ref.name": ["Image", "TargetImage"], "process_ref.parent_ref.command_line": ["ParentCommandLine"], "process_ref.creator_user_ref.user_id": ["username"], "process_ref.name": ["ProcessName"], @@ -143,14 +143,5 @@ "key": ["ObjectName", "RegistryKey"], "values[*].name": ["RegistryValueName"] } - }, - "x-ibm-windows": { - "fields": { - "targetimage": ["TargetImage"], - "grantedaccess": ["GrantedAccess"], - "calltrace": ["CallTrace"], - "sourceimage": ["Image"], - "pipename": ["PipeName"] - } } } diff --git a/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/events_from_stix_map.json b/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/events_from_stix_map.json new file mode 100644 index 000000000..c950beda7 --- /dev/null +++ b/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/events_from_stix_map.json @@ -0,0 +1,156 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["sourceip", "destinationip", "identityip"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["sourceip", "destinationip"] + } + }, + "url": { + "fields": { + "value": ["url"] + } + }, + "mac-addr": { + "fields": { + "value": ["sourcemac", "destinationmac"] + } + }, + "file": { + "fields": { + "name": ["filename"], + "hashes.'SHA-256'": ["sha256hash"], + "hashes.MD5": ["md5hash"], + "hashes.'SHA-1'": ["sha1hash"], + "parent_directory_ref": ["filepath"], + "parent_directory_ref.path": ["filepath"] + } + }, + "directory": { + "fields": { + "path": ["filepath"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["sourceport"], + "dst_port": ["destinationport"], + "protocols[*]": ["protocolid"], + "start": ["starttime"], + "end": ["endtime"], + "src_ref.value": ["sourceip", "sourcemac"], + "dst_ref.value": ["destinationip", "destinationmac"] + } + }, + "user-account": { + "fields": { + "user_id": ["username"], + "account_login": ["username"] + } + }, + "artifact": { + "fields": { + "payload_bin": ["eventpayload"] + } + }, + "domain-name": { + "fields": { + "value": ["domainname", "UrlHost"] + } + }, + "x-qradar": { + "fields": { + "qid": ["qid"], + "magnitude": ["magnitude"], + "log_source_id": ["logsourceid"], + "device_type": ["devicetype"], + "category_id": ["categoryid"], + "high_level_category_id": ["high_level_category_id"], + "direction": ["direction"], + "severity": ["eventseverity"], + "credibility": ["credibility"], + "relevance": ["relevance"], + "domain_id": ["domainid"], + "has_offense": ["hasoffense"], + "INOFFENSE": ["INOFFENSE"] + } + }, + "x-ibm-finding": { + "fields": { + "name": ["crename"], + "description": ["credescription"], + "severity": ["eventseverity"], + "start": ["starttime"], + "end": ["endtime"], + "magnitude": ["magnitude"], + "event_count": ["eventcount"], + "src_geolocation": ["sourcegeographic"], + "dst_geolocation": ["destinationgeographic"], + "rule_names[*]": ["rulenames"] + } + }, + "process": { + "fields": { + "pid": ["ProcessId"], + "name": ["ProcessName", "Image", "ParentImage"], + "image_ref.name": ["Image"], + "image_ref.parent_directory_ref.path": ["Image"], + "parent_ref.image_ref.name": ["ParentImage"], + "command_line": ["ProcessCommandLine", "ParentCommandLine"], + "parent_ref.command_line": ["ParentCommandLine"], + "extensions.'windows-service-ext'.service_dll_refs[*].name": ["ServiceFileName"] + } + }, + "x-oca-event": { + "fields": { + "action": ["qidname"], + "code": ["EventID"], + "outcome": ["categoryname"], + "category": ["high_level_category_name"], + "created": ["devicetime"], + "agent": ["logsourcename"], + "provider": ["logsourcetypename"], + "process_ref.command_line": ["ProcessCommandLine"], + "process_ref.image_ref.name": ["Image"], + "process_ref.parent_ref.command_line": ["ParentCommandLine"], + "process_ref.creator_user_ref.user_id": ["username"], + "process_ref.name": ["ProcessName"], + "process_ref.pid": ["ProcessId"], + "parent_process_ref.command_line": ["ParentCommandLine"], + "parent_process_ref.image_ref.name": ["ParentImage"], + "domain_ref.value": ["domainname", "UrlHost"], + "file_ref.name": ["filename"], + "host_ref.hostname": ["identityhostname"], + "host_ref.ip_refs[*].value": ["identityip"], + "registry_ref.key": ["ObjectName", "RegistryKey"], + "user_ref.user_id": ["username"], + "url_ref.value": ["url"], + "original_ref.payload_bin": ["eventpayload", "Message"] + } + }, + "x-oca-asset": { + "fields": { + "hostname": ["identityhostname"], + "ip_refs[*].value": ["identityip", "sourceip"], + "mac_refs[*].value": ["sourcemac"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["ObjectName", "RegistryKey"], + "values[*].name": ["RegistryValueName"] + } + }, + "x-ibm-windows": { + "fields": { + "targetimage": ["TargetImage"], + "grantedaccess": ["GrantedAccess"], + "calltrace": ["CallTrace"], + "sourceimage": ["Image"], + "pipename": ["PipeName"] + } + } +} diff --git a/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/flows_from_stix_map.json b/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/flows_from_stix_map.json new file mode 100644 index 000000000..b8d8eadfa --- /dev/null +++ b/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/flows_from_stix_map.json @@ -0,0 +1,92 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["sourceip", "destinationip"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["sourcev6", "destinationv6"] + } + }, + "file": { + "fields": { + "name": [ "filename" ], + "size": [ "filesize" ], + "hashes.'SHA-256'": [ "sha256hash" ], + "hashes.MD5": [ "md5hash" ], + "hashes.'SHA-1'": [ "sha1hash" ], + "mime-type": [ "contenttype" ] + } + }, + "domain-name": { + "fields": { + "value": [ "dnsdomainname" ] + } + }, + "url": { + "fields": { + "value": [ "dnsdomainname", "tlsservernameindication", "httphost" ] + } + }, + "network-traffic": { + "fields": { + "src_port": ["sourceport"], + "dst_port": ["destinationport"], + "protocols[*]": ["protocolid"], + "start": ["starttime"], + "end": ["endtime"], + "src_ref.value": ["sourceip", "sourcev6"], + "dst_ref.value": ["destinationip", "destinationv6"], + "src_byte_count": ["sourcebytes"], + "dst_byte_count": ["destinationbytes"], + "src_packets": ["sourcepackets"], + "dst_packets": ["destinationpackets"], + "extensions.'http-request-ext'.request_header.Host": [ "httphost" ], + "extensions.'http-request-ext'.request_header.Referer": [ "httpreferrer" ], + "extensions.'http-request-ext'.request_header.Server": [ "httpserver" ], + "extensions.'http-request-ext'.request_header.User-Agent": [ "httpuseragent" ], + "extensions.'http-request-ext'.request_header.Content-Type": [ "contenttype" ], + "extensions.'http-request-ext'.request_version": [ "httpversion" ], + "ipfix.flowId": [ "flowid" ] + } + }, + "software": { + "fields": { + "name": ["applicationname"] + } + }, + "artifact": { + "fields": { + "payload_bin": ["flowsourcepayload", "flowdestinationpayload"] + } + }, + "x-qradar": { + "fields": { + "qid": ["qid"], + "qid_name": ["qidname"], + "flow_source": ["flowsource"], + "flow_interface_id": ["flowinterfaceid"], + "flow_interface": ["flowinterface"], + "geographic": ["geographic"], + "category_name": ["categoryname"], + "credibility": ["credibility"], + "severity": ["flowseverity"], + "direction": ["direction"], + "relevance": ["relevance"], + "first_packet_time": ["firstpackettime"], + "last_packet_time": ["lastpackettime"], + "application_id": ["applicationid"], + "application_name": ["applicationname"], + "flow_type": [ "flowtype" ], + "file_entropy": [ "fileentropy" ], + "http_response_code": [ "httpresponsecode" ], + "tls_server_name_indication": [ "tlsservernameindication" ], + "tls_ja3_hash": [ "tlsja3hash" ], + "tls_ja3s_hash": [ "tlsja3shash" ], + "suspect_content_descriptions": [ "suspectcontentdescriptions" ], + "has_offense": ["hasoffense"], + "INOFFENSE": ["INOFFENSE"] + } + } +} diff --git a/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..eb635486e --- /dev/null +++ b/stix_shifter_modules/qradar/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,885 @@ +{ + "categoryid": { + "key": "x-qradar.category_id", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "username": [ + { + "key": "user-account.user_id", + "object": "useraccount" + }, + { + "key": "x-ibm-finding.src_application_user_ref", + "object": "finding", + "references": "useraccount" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "useraccount" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "useraccount" + } + ], + "categoryname": [ + { + "key": "x-oca-event.outcome", + "object": "event" + } + ], + "high_level_category_name": [ + { + "key": "x-oca-event.category", + "object": "event", + "transformer": "ValueToList" + } + ], + "high_level_category_id": { + "key": "x-qradar.high_level_category_id", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "identityip": [ + { + "key": "ipv4-addr.value", + "object": "host_ip_addr_v4" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip_addr_v6" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": ["host_ip_addr_v4"], + "group": true + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": ["host_ip_addr_v6"], + "group": true + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "identityhostname": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "qidname": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "relevance": { + "key": "x-qradar.relevance", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "devicetime": [ + { + "key": "x-oca-event.created", + "transformer": "EpochToTimestamp", + "object": "event" + } + ], + "destinationip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-ibm-finding.dst_ip_ref", + "object": "finding", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "destinationv6": [ + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "eventcount": [ + { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + { + "key": "x-ibm-finding.event_count", + "transformer": "ToInteger", + "object": "finding" + }, + { + "key": "x-ibm-finding.finding_type", + "object": "finding", + "value": "event" + } + ], + "starttime": [ + { + "key": "first_observed", + "transformer": "EpochToTimestamp", + "cybox": false + }, + { + "key": "x-ibm-finding.start", + "transformer": "EpochToTimestamp", + "object": "finding" + } + ], + "endtime": [ + { + "key": "last_observed", + "transformer": "EpochToTimestamp", + "cybox": false + }, + { + "key": "x-ibm-finding.end", + "transformer": "EpochToTimestamp", + "object": "finding" + } + ], + "logsourceid": { + "key": "x-qradar.log_source_id", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "logsourcename": [ + { + "key": "x-oca-event.agent", + "object": "event" + } + ], + "logsourcetypename": [ + { + "key": "x-oca-event.provider", + "object": "event" + } + ], + "direction": { + "key": "x-qradar.direction", + "object": "x-qradar" + }, + "magnitude": { + "key": "x-ibm-finding.magnitude", + "transformer": "ToInteger", + "object": "finding" + }, + "sourceip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-ibm-finding.src_ip_ref", + "object": "finding", + "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": ["src_ip"], + "group": true + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "sourcev6": [ + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "sourcemac": [ + { + "key": "mac-addr.value", + "object": "src_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": ["src_mac"] + } + ], + "destinationmac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + } + ], + "qid": { + "key": "x-qradar.qid", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "url": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "UrlHost": [ + { + "key": "domain-name.value", + "object": "dns_domain", + "transformer": "ToDomainName" + }, + { + "key": "network-traffic.extensions.dns-ext.question.domain_ref", + "object": "nt", + "references": "dns_domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "dns_domain" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "filename": [ + { + "key": "file.name", + "object": "fl" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "fl" + } + ], + "sha256hash": { + "key": "file.hashes.SHA-256", + "object": "fl" + }, + "sha1hash": { + "key": "file.hashes.SHA-1", + "object": "fl" + }, + "md5hash": { + "key": "file.hashes.MD5", + "object": "fl" + }, + "filehash": { + "key": "file.hashes.UNKNOWN", + "object": "fl" + }, + "filesize": { + "key": "file.size", + "object": "fl" + }, + "filepath": [ + { + "key": "directory.path", + "object": "dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "fl" + }, + { + "key": "file.parent_directory_ref", + "object": "fl", + "references": "dir" + } + ], + "eventpayload": [ + { + "key": "artifact.payload_bin", + "transformer": "ToBase64", + "object": "artifact" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "artifact" + } + ], + "mime_type_eventpayload": { + "key": "artifact.mime_type", + "object": "artifact" + }, + "flowsourcepayload": [ + { + "key": "artifact.payload_bin", + "object": "src_payload", + "transformer": "ToBase64" + }, + { + "key": "network-traffic.src_payload_ref", + "object": "nt", + "references": "src_payload" + } + ], + "mime_type_flowsourcepayload": { + "key": "artifact.mime_type", + "object": "artifact" + }, + "flowdestinationpayload": [ + { + "key": "artifact.payload_bin", + "object": "dst_payload", + "transformer": "ToBase64" + }, + { + "key": "network-traffic.dst_payload_ref", + "object": "nt", + "references": "dst_payload" + } + ], + "mime_type_flowdestinationpayload": { + "key": "artifact.mime_type", + "object": "artifact" + }, + "destinationport": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "sourceport": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "sourcebytes": { + "key": "network-traffic.src_byte_count", + "object": "nt", + "transformer": "ToInteger" + }, + "destinationbytes": { + "key": "network-traffic.dst_byte_count", + "object": "nt", + "transformer": "ToInteger" + }, + "sourcepackets": { + "key": "network-traffic.src_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "destinationpackets": { + "key": "network-traffic.dst_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + "domainname": [ + { + "key": "domain-name.value", + "object": "domain2", + "transformer": "ToDomainName" + }, + { + "key": "x-qradar.domain_name", + "object": "x-qradar" + } + ], + "dnsdomainname": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "domain-name.value", + "object": "domain1", + "transformer": "ToDomainName" + }, + { + "key": "x-qradar.domain_name", + "object": "x-qradar" + } + ], + "flowsource": [ + { + "key": "x-qradar.flow_source", + "object": "x-qradar" + } + ] + , + "flowinterface": { + "key": "x-qradar.flow_interface", + "object": "x-qradar" + }, + "flowinterfaceid": { + "key": "x-qradar.flow_interface_id", + "object": "x-qradar" + }, + "geographic": { + "key": "x-qradar.geographic", + "object": "x-qradar" + }, + "sourcegeographic": { + "key": "x-ibm-finding.src_geolocation", + "object": "finding" + }, + "destinationgeographic": { + "key": "x-ibm-finding.dst_geolocation", + "object": "finding" + }, + "credibility": { + "key": "x-qradar.credibility", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "eventseverity": { + "key": "x-ibm-finding.severity", + "transformer": "ToInteger", + "object": "finding" + }, + "flowseverity": { + "key": "x-qradar.severity", + "transformer": "ToInteger", + "object": "x-qradar" + }, + "firstpackettime": { + "key": "x-qradar.first_packet_time", + "object": "x-qradar" + }, + "lastpackettime": { + "key": "x-qradar.last_packet_time", + "object": "x-qradar" + }, + "applicationname": { + "key": "software.name" + }, + "applicationid": { + "key": "x-qradar.application_id", + "object": "x-qradar" + }, + "creeventlist": { + "key": "x-qradar.cre_event_list", + "object": "x-qradar" + }, + "rulenames": { + "key": "x-ibm-finding.rule_names", + "object": "finding" + }, + "crename": { + "key": "x-ibm-finding.name", + "object": "finding" + }, + "credescription": { + "key": "x-ibm-finding.description", + "object": "finding" + }, + "domainid": { + "key": "x-qradar.domain_id", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "devicetype": { + "key": "x-qradar.device_type", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "flowtype": { + "key": "x-qradar.flow_type", + "object": "x-qradar" + }, + "fileentropy": { + "key": "x-qradar.file_entropy", + "object": "x-qradar" + }, + "httphost": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "network-traffic.extensions.http-request-ext.request_header.Host", + "object": "nt" + } + ], + "httpreferrer": { + "key": "network-traffic.extensions.http-request-ext.request_header.Referer", + "object": "nt" + }, + "httpresponsecode": { + "key": "x-qradar.http_response_code", + "object": "x-qradar" + }, + "httpserver": { + "key": "network-traffic.extensions.http-request-ext.request_header.Server", + "object": "nt" + }, + "httpuseragent": { + "key": "network-traffic.extensions.http-request-ext.request_header.User-Agent", + "object": "nt" + }, + "httpversion": { + "key": "network-traffic.extensions.http-request-ext.request_version", + "object": "nt" + }, + "tlsja3hash": { + "key": "x-qradar.tls_ja3_hash", + "object": "x-qradar" + }, + "tlsja3shash": { + "key": "x-qradar.tls_ja3s_hash", + "object": "x-qradar" + }, + "suspectcontentdescriptions": { + "key": "x-qradar.suspect_content_descriptions", + "object": "x-qradar", + "transformer": "ToLowercaseArray" + }, + "flowid": { + "key": "network-traffic.ipfix.flowId", + "object": "nt", + "transformer": "ToInteger" + }, + "contenttype": [ + { + "key": "network-traffic.extensions.http-request-ext.request_header.Content-Type", + "object": "nt" + }, + { + "key": "file.mime_type", + "object": "fl" + }, + { + "key": "email-message.content_type", + "object": "email" + } + ], + "tlsservernameindication": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-qradar.tls_server_name_indication", + "object": "x-qradar" + } + ], + "Image": [ + { + "key": "file.name", + "object": "file_image", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory_image", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file_image" + }, + { + "key": "file.parent_directory_ref", + "object": "file_image", + "references": "directory_image" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "ParentImage": [ + { + "key": "file.name", + "object": "file_parent_image", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory_parent_image", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_parent_image", + "references": "directory_parent_image" + }, + { + "key": "process.image_ref", + "object": "parent_process", + "references": "file_parent_image" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "ProcessCommandLine": [ + { + "key": "process.command_line", + "object": "process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "ParentCommandLine": [ + { + "key": "process.command_line", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "ProcessId": [ + { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "EventID": { + "key": "x-oca-event.code", + "object": "event", + "transformer": "ToInteger" + }, + "ParentProcessId": [ + { + "key": "process.pid", + "object": "parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "ObjectName": [ + { + "key": "windows-registry-key.key", + "object": "winregistry", + "transformer": "ObjectnameToStixRegistryKey" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "winregistry" + } + ], + "RegistryKey": [ + { + "key": "x-qradar.registry_key", + "object": "x-qradar" + } + ], + "RegistryValueName": [ + { + "key": "windows-registry-key.values", + "object": "winregistry", + "transformer": "RegValueNameToStixRegistryValues" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "winregistry" + } + ], + "TargetImage": { + "key": "x-ibm-windows.targetimage", + "object": "xwin" + }, + "GrantedAccess": { + "key": "x-ibm-windows.granted_access", + "object": "xwin" + }, + "CallTrace": { + "key": "x-ibm-windows.call_trace", + "object": "xwin" + }, + "SourceImage": { + "key": "x-ibm-windows.source_image", + "object": "xwin" + }, + "PipeName": { + "key": "x-ibm-windows.pipe_name", + "object": "xwin" + }, + "StartModule": { + "key": "x-ibm-windows.start_module", + "object": "xwin" + }, + "StartFunction": { + "key": "x-ibm-windows.start_function", + "object": "xwin" + }, + "Signed": { + "key": "x-ibm-windows.signed", + "object": "xwin" + }, + "Message": [ + { + "key": "artifact.payload_bin", + "object": "event_payload", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "event_payload" + }, + { + "key": "x-oca-event.original", + "object": "event" + } + ], + "mime_type_message": { + "key": "artifact.mime_type", + "object": "artifact" + }, + "IMPHash": { + "key": "x-ibm-windows.imphash", + "object": "xwin" + }, + "ServiceFileName": [ + { + "key": "file.name", + "object": "service_file_name", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "service_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "service_file_name", + "references": "service_file_dir" + }, + { + "key": "process.extensions.windows-service-ext.service_dll_refs", + "object": "process", + "group": true, + "references": ["service_file_name"] + } + ], + "hasoffense": { + "key": "x-qradar.has_offense", + "object": "x-qradar" + } +} diff --git a/stix_shifter_modules/qradar/stix_translation/json/to_stix_map.json b/stix_shifter_modules/qradar/stix_translation/json/to_stix_map.json index ea70a5331..d93c46144 100644 --- a/stix_shifter_modules/qradar/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/qradar/stix_translation/json/to_stix_map.json @@ -308,7 +308,7 @@ } ], "UrlHost": [ - { + { "key": "domain-name.value", "object": "dns_domain", "transformer": "ToDomainName" @@ -491,8 +491,7 @@ "key": "x-qradar.flow_source", "object": "x-qradar" } - ] - , + ], "flowinterface": { "key": "x-qradar.flow_interface", "object": "x-qradar" @@ -800,9 +799,9 @@ ], "RegistryValueName": [ { - "key": "windows-registry-key.values", - "object": "winregistry", - "transformer": "RegValueNameToStixRegistryValues" + "key": "windows-registry-key.values", + "object": "winregistry", + "transformer": "RegValueNameToStixRegistryValues" }, { "key": "x-oca-event.registry_ref", @@ -810,38 +809,55 @@ "references": "winregistry" } ], - "TargetImage": { - "key": "x-ibm-windows.targetimage", - "object": "xwin" - }, - "GrantedAccess": { - "key": "x-ibm-windows.granted_access", - "object": "xwin" - }, - "CallTrace": { - "key": "x-ibm-windows.call_trace", - "object": "xwin" - }, - "SourceImage": { - "key": "x-ibm-windows.source_image", - "object": "xwin" - }, - "PipeName": { - "key": "x-ibm-windows.pipe_name", - "object": "xwin" - }, - "StartModule": { - "key": "x-ibm-windows.start_module", - "object": "xwin" - }, - "StartFunction": { - "key": "x-ibm-windows.start_function", - "object": "xwin" - }, - "Signed": { - "key": "x-ibm-windows.signed", - "object": "xwin" - }, + "TargetImage": [ + { + "key": "file.name", + "object": "file_target_image", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory_target_image", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.binary_ref", + "object": "target_process", + "references": "file_target_image" + }, + { + "key": "file.parent_directory_ref", + "object": "file_target_image", + "references": "directory_target_image" + } + ], + "SourceImage": [ + { + "key": "file.name", + "object": "file_image", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory_image", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "file_image" + }, + { + "key": "file.parent_directory_ref", + "object": "file_image", + "references": "directory_image" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], "Message": [ { "key": "artifact.payload_bin", @@ -859,12 +875,8 @@ } ], "mime_type_message": { - "key": "artifact.mime_type", - "object": "artifact" - }, - "IMPHash": { - "key": "x-ibm-windows.imphash", - "object": "xwin" + "key": "artifact.mime_type", + "object": "artifact" }, "ServiceFileName": [ { diff --git a/stix_shifter_modules/qradar/stix_transmission/results_connector.py b/stix_shifter_modules/qradar/stix_transmission/results_connector.py index 362848893..d56896544 100644 --- a/stix_shifter_modules/qradar/stix_transmission/results_connector.py +++ b/stix_shifter_modules/qradar/stix_transmission/results_connector.py @@ -21,14 +21,15 @@ def create_results_connection(self, search_id, offset, length): return_obj = dict() error = None response_text = response.read() + response_dict = dict() try: response_dict = json.loads(response_text) except ValueError as ex: self.logger.debug(response_text) - error = Exception(f'Can not parse response: {ex} : {response_text}') + error = Exception(f'Can not parse response from Qradar server. The response is not a valid json: {response_text} : {ex}') - if 200 <= response_code <= 299: + if 200 <= response_code <= 299 and error is None: return_obj['success'] = True return_obj['data'] = response_dict.get('events', response_dict.get('flows')) else: diff --git a/stix_shifter_modules/qradar_perf_test/stix_translation/json/aql_events_fields.json b/stix_shifter_modules/qradar_perf_test/stix_translation/json/aql_events_fields.json index 7eb220db1..28e3a604c 100644 --- a/stix_shifter_modules/qradar_perf_test/stix_translation/json/aql_events_fields.json +++ b/stix_shifter_modules/qradar_perf_test/stix_translation/json/aql_events_fields.json @@ -51,13 +51,7 @@ "\"Process CommandLine\" as ProcessCommandLine", "ParentCommandLine", "TargetImage", - "\"Granted Access\" as GrantedAccess", - "\"Call Trace\" as CallTrace", "SourceImage", - "PipeName", - "StartModule", - "StartFunction", - "Signed", "Message", "\"Registry Value Name\" as RegistryValueName", "\"IMP Hash\" as IMPHash", diff --git a/stix_shifter_modules/qradar_perf_test/stix_translation/json/events_from_stix_map.json b/stix_shifter_modules/qradar_perf_test/stix_translation/json/events_from_stix_map.json index fa82b7d3e..dda9dd81b 100644 --- a/stix_shifter_modules/qradar_perf_test/stix_translation/json/events_from_stix_map.json +++ b/stix_shifter_modules/qradar_perf_test/stix_translation/json/events_from_stix_map.json @@ -95,13 +95,13 @@ "process": { "fields": { "pid": ["ProcessId"], - "name": ["ProcessName", "Image", "ParentImage"], - "binary_ref.name": ["Image"], - "binary_ref.parent_directory_ref.path": ["Image"], + "name": ["ProcessName", "Image", "ParentImage", "TargetImage"], + "binary_ref.name": ["Image", "TargetImage"], + "binary_ref.parent_directory_ref.path": ["Image", "TargetImage"], "parent_ref.binary_ref.name": ["ParentImage"], "command_line": ["ProcessCommandLine", "ParentCommandLine"], "parent_ref.command_line": ["ParentCommandLine"], - "extensions.windows-service-ext.service_dll_refs[*].name": ["ServiceFileName"] + "extensions.'windows-service-ext'.service_dll_refs[*].name": ["ServiceFileName"] } }, "x-oca-event": { @@ -114,7 +114,7 @@ "agent": ["logsourcename"], "provider": ["logsourcetypename"], "process_ref.command_line": ["ProcessCommandLine"], - "process_ref.binary_ref.name": ["Image"], + "process_ref.binary_ref.name": ["Image", "TargetImage"], "process_ref.parent_ref.command_line": ["ParentCommandLine"], "process_ref.creator_user_ref.user_id": ["username"], "process_ref.name": ["ProcessName"], @@ -143,14 +143,5 @@ "key": ["ObjectName", "RegistryKey"], "values[*].name": ["RegistryValueName"] } - }, - "x-ibm-windows": { - "fields": { - "targetimage": ["TargetImage"], - "grantedaccess": ["GrantedAccess"], - "calltrace": ["CallTrace"], - "sourceimage": ["Image"], - "pipename": ["PipeName"] - } } } diff --git a/stix_shifter_modules/qradar_perf_test/stix_translation/json/stix_2_1/events_from_stix_map.json b/stix_shifter_modules/qradar_perf_test/stix_translation/json/stix_2_1/events_from_stix_map.json new file mode 100644 index 000000000..c950beda7 --- /dev/null +++ b/stix_shifter_modules/qradar_perf_test/stix_translation/json/stix_2_1/events_from_stix_map.json @@ -0,0 +1,156 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["sourceip", "destinationip", "identityip"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["sourceip", "destinationip"] + } + }, + "url": { + "fields": { + "value": ["url"] + } + }, + "mac-addr": { + "fields": { + "value": ["sourcemac", "destinationmac"] + } + }, + "file": { + "fields": { + "name": ["filename"], + "hashes.'SHA-256'": ["sha256hash"], + "hashes.MD5": ["md5hash"], + "hashes.'SHA-1'": ["sha1hash"], + "parent_directory_ref": ["filepath"], + "parent_directory_ref.path": ["filepath"] + } + }, + "directory": { + "fields": { + "path": ["filepath"] + } + }, + "network-traffic": { + "fields": { + "src_port": ["sourceport"], + "dst_port": ["destinationport"], + "protocols[*]": ["protocolid"], + "start": ["starttime"], + "end": ["endtime"], + "src_ref.value": ["sourceip", "sourcemac"], + "dst_ref.value": ["destinationip", "destinationmac"] + } + }, + "user-account": { + "fields": { + "user_id": ["username"], + "account_login": ["username"] + } + }, + "artifact": { + "fields": { + "payload_bin": ["eventpayload"] + } + }, + "domain-name": { + "fields": { + "value": ["domainname", "UrlHost"] + } + }, + "x-qradar": { + "fields": { + "qid": ["qid"], + "magnitude": ["magnitude"], + "log_source_id": ["logsourceid"], + "device_type": ["devicetype"], + "category_id": ["categoryid"], + "high_level_category_id": ["high_level_category_id"], + "direction": ["direction"], + "severity": ["eventseverity"], + "credibility": ["credibility"], + "relevance": ["relevance"], + "domain_id": ["domainid"], + "has_offense": ["hasoffense"], + "INOFFENSE": ["INOFFENSE"] + } + }, + "x-ibm-finding": { + "fields": { + "name": ["crename"], + "description": ["credescription"], + "severity": ["eventseverity"], + "start": ["starttime"], + "end": ["endtime"], + "magnitude": ["magnitude"], + "event_count": ["eventcount"], + "src_geolocation": ["sourcegeographic"], + "dst_geolocation": ["destinationgeographic"], + "rule_names[*]": ["rulenames"] + } + }, + "process": { + "fields": { + "pid": ["ProcessId"], + "name": ["ProcessName", "Image", "ParentImage"], + "image_ref.name": ["Image"], + "image_ref.parent_directory_ref.path": ["Image"], + "parent_ref.image_ref.name": ["ParentImage"], + "command_line": ["ProcessCommandLine", "ParentCommandLine"], + "parent_ref.command_line": ["ParentCommandLine"], + "extensions.'windows-service-ext'.service_dll_refs[*].name": ["ServiceFileName"] + } + }, + "x-oca-event": { + "fields": { + "action": ["qidname"], + "code": ["EventID"], + "outcome": ["categoryname"], + "category": ["high_level_category_name"], + "created": ["devicetime"], + "agent": ["logsourcename"], + "provider": ["logsourcetypename"], + "process_ref.command_line": ["ProcessCommandLine"], + "process_ref.image_ref.name": ["Image"], + "process_ref.parent_ref.command_line": ["ParentCommandLine"], + "process_ref.creator_user_ref.user_id": ["username"], + "process_ref.name": ["ProcessName"], + "process_ref.pid": ["ProcessId"], + "parent_process_ref.command_line": ["ParentCommandLine"], + "parent_process_ref.image_ref.name": ["ParentImage"], + "domain_ref.value": ["domainname", "UrlHost"], + "file_ref.name": ["filename"], + "host_ref.hostname": ["identityhostname"], + "host_ref.ip_refs[*].value": ["identityip"], + "registry_ref.key": ["ObjectName", "RegistryKey"], + "user_ref.user_id": ["username"], + "url_ref.value": ["url"], + "original_ref.payload_bin": ["eventpayload", "Message"] + } + }, + "x-oca-asset": { + "fields": { + "hostname": ["identityhostname"], + "ip_refs[*].value": ["identityip", "sourceip"], + "mac_refs[*].value": ["sourcemac"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["ObjectName", "RegistryKey"], + "values[*].name": ["RegistryValueName"] + } + }, + "x-ibm-windows": { + "fields": { + "targetimage": ["TargetImage"], + "grantedaccess": ["GrantedAccess"], + "calltrace": ["CallTrace"], + "sourceimage": ["Image"], + "pipename": ["PipeName"] + } + } +} diff --git a/stix_shifter_modules/qradar_perf_test/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/qradar_perf_test/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..5e5d1f1c3 --- /dev/null +++ b/stix_shifter_modules/qradar_perf_test/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,869 @@ +{ + "categoryid": { + "key": "x-qradar.category_id", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "username": [ + { + "key": "user-account.user_id", + "object": "useraccount" + }, + { + "key": "x-ibm-finding.src_application_user_ref", + "object": "finding", + "references": "useraccount" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "useraccount" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "useraccount" + } + ], + "categoryname": [ + { + "key": "x-oca-event.outcome", + "object": "event" + } + ], + "high_level_category_name": [ + { + "key": "x-oca-event.category", + "object": "event", + "transformer": "ValueToList" + } + ], + "high_level_category_id": { + "key": "x-qradar.high_level_category_id", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "identityip": [ + { + "key": "ipv4-addr.value", + "object": "host_ip_addr_v4" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip_addr_v6" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": ["host_ip_addr_v4"], + "group": true + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": ["host_ip_addr_v6"], + "group": true + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "identityhostname": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "qidname": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "relevance": { + "key": "x-qradar.relevance", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "devicetime": [ + { + "key": "x-oca-event.created", + "transformer": "EpochToTimestamp", + "object": "event" + } + ], + "destinationip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-ibm-finding.dst_ip_ref", + "object": "finding", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "destinationv6": [ + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "eventcount": [ + { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + { + "key": "x-ibm-finding.event_count", + "transformer": "ToInteger", + "object": "finding" + }, + { + "key": "x-ibm-finding.finding_type", + "object": "finding", + "value": "event" + } + ], + "starttime": [ + { + "key": "first_observed", + "transformer": "EpochToTimestamp", + "cybox": false + }, + { + "key": "x-ibm-finding.start", + "transformer": "EpochToTimestamp", + "object": "finding" + } + ], + "endtime": [ + { + "key": "last_observed", + "transformer": "EpochToTimestamp", + "cybox": false + }, + { + "key": "x-ibm-finding.end", + "transformer": "EpochToTimestamp", + "object": "finding" + } + ], + "logsourceid": { + "key": "x-qradar.log_source_id", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "logsourcename": [ + { + "key": "x-oca-event.agent", + "object": "event" + } + ], + "logsourcetypename": [ + { + "key": "x-oca-event.provider", + "object": "event" + } + ], + "direction": { + "key": "x-qradar.direction", + "object": "x-qradar" + }, + "magnitude": { + "key": "x-ibm-finding.magnitude", + "transformer": "ToInteger", + "object": "finding" + }, + "sourceip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-ibm-finding.src_ip_ref", + "object": "finding", + "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": ["src_ip"], + "group": true + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "sourcev6": [ + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "sourcemac": [ + { + "key": "mac-addr.value", + "object": "src_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": ["src_mac"] + } + ], + "destinationmac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + } + ], + "qid": { + "key": "x-qradar.qid", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "url": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "UrlHost": [ + { + "key": "domain-name.value", + "object": "dns_domain", + "transformer": "ToDomainName" + }, + { + "key": "network-traffic.extensions.dns-ext.question.domain_ref", + "object": "nt", + "references": "dns_domain" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "dns_domain" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "filename": [ + { + "key": "file.name", + "object": "fl" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "fl" + } + ], + "sha256hash": { + "key": "file.hashes.SHA-256", + "object": "fl" + }, + "sha1hash": { + "key": "file.hashes.SHA-1", + "object": "fl" + }, + "md5hash": { + "key": "file.hashes.MD5", + "object": "fl" + }, + "filehash": { + "key": "file.hashes.UNKNOWN", + "object": "fl" + }, + "filesize": { + "key": "file.size", + "object": "fl" + }, + "filepath": [ + { + "key": "directory.path", + "object": "dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "fl" + }, + { + "key": "file.parent_directory_ref", + "object": "fl", + "references": "dir" + } + ], + "eventpayload": [ + { + "key": "artifact.payload_bin", + "transformer": "ToBase64", + "object": "artifact" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "artifact" + } + ], + "flowsourcepayload": [ + { + "key": "artifact.payload_bin", + "object": "src_payload", + "transformer": "ToBase64" + }, + { + "key": "network-traffic.src_payload_ref", + "object": "nt", + "references": "src_payload" + } + ], + "flowdestinationpayload": [ + { + "key": "artifact.payload_bin", + "object": "dst_payload", + "transformer": "ToBase64" + }, + { + "key": "network-traffic.dst_payload_ref", + "object": "nt", + "references": "dst_payload" + } + ], + "destinationport": { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + "sourceport": { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + "sourcebytes": { + "key": "network-traffic.src_byte_count", + "object": "nt", + "transformer": "ToInteger" + }, + "destinationbytes": { + "key": "network-traffic.dst_byte_count", + "object": "nt", + "transformer": "ToInteger" + }, + "sourcepackets": { + "key": "network-traffic.src_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "destinationpackets": { + "key": "network-traffic.dst_packets", + "object": "nt", + "transformer": "ToInteger" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "nt", + "transformer": "ToLowercaseArray" + }, + "domainname": [ + { + "key": "domain-name.value", + "object": "domain2", + "transformer": "ToDomainName" + }, + { + "key": "x-qradar.domain_name", + "object": "x-qradar" + } + ], + "dnsdomainname": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "domain-name.value", + "object": "domain1", + "transformer": "ToDomainName" + }, + { + "key": "x-qradar.domain_name", + "object": "x-qradar" + } + ], + "flowsource": [ + { + "key": "x-qradar.flow_source", + "object": "x-qradar" + } + ] + , + "flowinterface": { + "key": "x-qradar.flow_interface", + "object": "x-qradar" + }, + "flowinterfaceid": { + "key": "x-qradar.flow_interface_id", + "object": "x-qradar" + }, + "geographic": { + "key": "x-qradar.geographic", + "object": "x-qradar" + }, + "sourcegeographic": { + "key": "x-ibm-finding.src_geolocation", + "object": "finding" + }, + "destinationgeographic": { + "key": "x-ibm-finding.dst_geolocation", + "object": "finding" + }, + "credibility": { + "key": "x-qradar.credibility", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "eventseverity": { + "key": "x-ibm-finding.severity", + "transformer": "ToInteger", + "object": "finding" + }, + "flowseverity": { + "key": "x-qradar.severity", + "transformer": "ToInteger", + "object": "x-qradar" + }, + "firstpackettime": { + "key": "x-qradar.first_packet_time", + "object": "x-qradar" + }, + "lastpackettime": { + "key": "x-qradar.last_packet_time", + "object": "x-qradar" + }, + "applicationname": { + "key": "software.name" + }, + "applicationid": { + "key": "x-qradar.application_id", + "object": "x-qradar" + }, + "creeventlist": { + "key": "x-qradar.cre_event_list", + "object": "x-qradar" + }, + "rulenames": { + "key": "x-ibm-finding.rule_names", + "object": "finding" + }, + "crename": { + "key": "x-ibm-finding.name", + "object": "finding" + }, + "credescription": { + "key": "x-ibm-finding.description", + "object": "finding" + }, + "domainid": { + "key": "x-qradar.domain_id", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "devicetype": { + "key": "x-qradar.device_type", + "object": "x-qradar", + "transformer": "ToInteger" + }, + "flowtype": { + "key": "x-qradar.flow_type", + "object": "x-qradar" + }, + "fileentropy": { + "key": "x-qradar.file_entropy", + "object": "x-qradar" + }, + "httphost": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "network-traffic.extensions.http-request-ext.request_header.Host", + "object": "nt" + } + ], + "httpreferrer": { + "key": "network-traffic.extensions.http-request-ext.request_header.Referer", + "object": "nt" + }, + "httpresponsecode": { + "key": "x-qradar.http_response_code", + "object": "x-qradar" + }, + "httpserver": { + "key": "network-traffic.extensions.http-request-ext.request_header.Server", + "object": "nt" + }, + "httpuseragent": { + "key": "network-traffic.extensions.http-request-ext.request_header.User-Agent", + "object": "nt" + }, + "httpversion": { + "key": "network-traffic.extensions.http-request-ext.request_version", + "object": "nt" + }, + "tlsja3hash": { + "key": "x-qradar.tls_ja3_hash", + "object": "x-qradar" + }, + "tlsja3shash": { + "key": "x-qradar.tls_ja3s_hash", + "object": "x-qradar" + }, + "suspectcontentdescriptions": { + "key": "x-qradar.suspect_content_descriptions", + "object": "x-qradar", + "transformer": "ToLowercaseArray" + }, + "flowid": { + "key": "network-traffic.ipfix.flowId", + "object": "nt", + "transformer": "ToInteger" + }, + "contenttype": [ + { + "key": "network-traffic.extensions.http-request-ext.request_header.Content-Type", + "object": "nt" + }, + { + "key": "file.mime_type", + "object": "fl" + }, + { + "key": "email-message.content_type", + "object": "email" + } + ], + "tlsservernameindication": [ + { + "key": "url.value", + "object": "url" + }, + { + "key": "x-qradar.tls_server_name_indication", + "object": "x-qradar" + } + ], + "Image": [ + { + "key": "file.name", + "object": "file_image", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory_image", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file_image" + }, + { + "key": "file.parent_directory_ref", + "object": "file_image", + "references": "directory_image" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "ParentImage": [ + { + "key": "file.name", + "object": "file_parent_image", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory_parent_image", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file_parent_image", + "references": "directory_parent_image" + }, + { + "key": "process.image_ref", + "object": "parent_process", + "references": "file_parent_image" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "ProcessCommandLine": [ + { + "key": "process.command_line", + "object": "process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "ParentCommandLine": [ + { + "key": "process.command_line", + "object": "parent_process" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "ProcessId": [ + { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], + "EventID": { + "key": "x-oca-event.code", + "object": "event", + "transformer": "ToInteger" + }, + "ParentProcessId": [ + { + "key": "process.pid", + "object": "parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "x-oca-event.parent_process_ref", + "object": "event", + "references": "parent_process" + } + ], + "ObjectName": [ + { + "key": "windows-registry-key.key", + "object": "winregistry", + "transformer": "ObjectnameToStixRegistryKey" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "winregistry" + } + ], + "RegistryKey": [ + { + "key": "x-qradar.registry_key", + "object": "x-qradar" + } + ], + "RegistryValueName": [ + { + "key": "windows-registry-key.values", + "object": "winregistry", + "transformer": "RegValueNameToStixRegistryValues" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "winregistry" + } + ], + "TargetImage": { + "key": "x-ibm-windows.targetimage", + "object": "xwin" + }, + "GrantedAccess": { + "key": "x-ibm-windows.granted_access", + "object": "xwin" + }, + "CallTrace": { + "key": "x-ibm-windows.call_trace", + "object": "xwin" + }, + "SourceImage": { + "key": "x-ibm-windows.source_image", + "object": "xwin" + }, + "PipeName": { + "key": "x-ibm-windows.pipe_name", + "object": "xwin" + }, + "StartModule": { + "key": "x-ibm-windows.start_module", + "object": "xwin" + }, + "StartFunction": { + "key": "x-ibm-windows.start_function", + "object": "xwin" + }, + "Signed": { + "key": "x-ibm-windows.signed", + "object": "xwin" + }, + "Message": [ + { + "key": "artifact.payload_bin", + "object": "event_payload", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "event_payload" + }, + { + "key": "x-oca-event.original", + "object": "event" + } + ], + "IMPHash": { + "key": "x-ibm-windows.imphash", + "object": "xwin" + }, + "ServiceFileName": [ + { + "key": "file.name", + "object": "service_file_name", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "service_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "service_file_name", + "references": "service_file_dir" + }, + { + "key": "process.extensions.windows-service-ext.service_dll_refs", + "object": "process", + "group": true, + "references": ["service_file_name"] + } + ], + "hasoffense": { + "key": "x-qradar.has_offense", + "object": "x-qradar" + } +} diff --git a/stix_shifter_modules/qradar_perf_test/stix_translation/json/to_stix_map.json b/stix_shifter_modules/qradar_perf_test/stix_translation/json/to_stix_map.json index c16d9225a..761a25f74 100644 --- a/stix_shifter_modules/qradar_perf_test/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/qradar_perf_test/stix_translation/json/to_stix_map.json @@ -798,38 +798,55 @@ "references": "winregistry" } ], - "TargetImage": { - "key": "x-ibm-windows.targetimage", - "object": "xwin" - }, - "GrantedAccess": { - "key": "x-ibm-windows.granted_access", - "object": "xwin" - }, - "CallTrace": { - "key": "x-ibm-windows.call_trace", - "object": "xwin" - }, - "SourceImage": { - "key": "x-ibm-windows.source_image", - "object": "xwin" - }, - "PipeName": { - "key": "x-ibm-windows.pipe_name", - "object": "xwin" - }, - "StartModule": { - "key": "x-ibm-windows.start_module", - "object": "xwin" - }, - "StartFunction": { - "key": "x-ibm-windows.start_function", - "object": "xwin" - }, - "Signed": { - "key": "x-ibm-windows.signed", - "object": "xwin" - }, + "TargetImage": [ + { + "key": "file.name", + "object": "file_target_image", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory_target_image", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.binary_ref", + "object": "target_process", + "references": "file_target_image" + }, + { + "key": "file.parent_directory_ref", + "object": "file_target_image", + "references": "directory_target_image" + } + ], + "SourceImage": [ + { + "key": "file.name", + "object": "file_image", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "directory_image", + "transformer": "ToDirectoryPath" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "file_image" + }, + { + "key": "file.parent_directory_ref", + "object": "file_image", + "references": "directory_image" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "process" + } + ], "Message": [ { "key": "artifact.payload_bin", @@ -846,10 +863,6 @@ "object": "event" } ], - "IMPHash": { - "key": "x-ibm-windows.imphash", - "object": "xwin" - }, "ServiceFileName": [ { "key": "file.name", diff --git a/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..23b3bf1a4 --- /dev/null +++ b/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,166 @@ +{ + "directory": { + "cim_type": "endpoint", + "fields": { + "path": "file_path", + "ctime": "file_create_time", + "mtime": "file_modify_time" + } + }, + "domain-name": { + "cim_type": "flow", + "fields": { + "value": [ + "host", + "url" + ] + } + }, + "x-readable-payload": { + "cim_type": "", + "fields": { + "value": "_raw" + } + }, + "email-addr": { + "cim_type": "email", + "fields": { + "value": [ + "src_user", + "recipient" + ] + } + }, + "email-message": { + "cim_type": "email", + "fields": { + "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.MD5": "file_hash", + "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.'SHA-1'": "file_hash", + "body_multipart.[*].'mime-part-type'.body_raw_ref.hashes.'SHA-256'": "file_hash", + "body_multipart.[*].'mime-part-type'.body_raw_ref.name": "file_name", + "body_multipart.[*].'mime-part-type'.body_raw_ref.size": "file_size", + "to_refs.[*].value": "recipient", + "cc_refs.[*].value": "recipient", + "bcc_refs.[*].value": "recipient", + "subject": "subject", + "sender_ref.value": "src_user", + "from_ref.value": "src_user" + } + }, + "file": { + "cim_type": "endpoint", + "fields": { + "hashes.MD5": "file_hash", + "hashes.'SHA-1'": "file_hash", + "hashes.'SHA-256'": "file_hash", + "name": ["file_name", "process"], + "ctime": "file_create_time", + "mtime": "file_modify_time", + "parent_directory_ref.path": "file_path", + "size": "file_size" + } + }, + "ipv4-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_ip", + "dest_ip" + ] + } + }, + "ipv6-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_ipv6", + "dest_ipv6" + ] + } + }, + "mac-addr": { + "cim_type": "flow", + "fields": { + "value": [ + "src_mac", + "dest_mac" + ] + } + }, + "network-traffic": { + "cim_type": "network", + "fields": { + "src_ref.value": "src", + "src_port": "src_port", + "dst_ref.value": "src", + "dst_port": "dest_port", + "protocols[*]": "protocol", + "start": "earliest", + "end": "latest" + } + }, + "process": { + "cim_type": "process", + "fields": { + "pid": "pid", + "creator_user_ref.account_login": "user", + "image_ref.parent_directory_ref.path": "file_path", + "image_ref.name": ["file_name", "process"] + } + }, + "url": { + "cim_type": "web", + "fields": { + "value": "url" + } + }, + "user-account": { + "cim_type": "authentication", + "fields": { + "user_id": "user" + } + }, + "windows-registry-key": { + "cim_type": "endpoint", + "fields": { + "key": "object", + "values[*]": "result", + "creator_user_ref.account_login": "user" + } + }, + "x509-certificate": { + "cim_type": "certificate", + "fields": { + "hashes.'SHA-256'": "ssl_hash", + "hashes.'SHA-1'": "ssl_hash", + "version": "ssl_version", + "serial_number": "ssl_serial", + "signature_algorithm": "ssl_signature_algorithm", + "issuer": "ssl_issuer", + "subject": "ssl_subject", + "subject_public_key_algorithm": "ssl_publickey_algorithm" + } + }, + "x-splunk": { + "cim_type": "endpoint", + "fields": { + "log_source": ["source"], + "log_source_type": ["_sourcetype"], + "direction": ["Direction"], + "event_id": ["EventID"], + "event_name": ["EventName"], + "mitre_tactic_id": ["TacticId"], + "mitre_tactic": ["Tactic"], + "mitre_technique_id": ["TechniqueId"], + "mitre_technique": ["Technique"] + } + }, + "x-ibm-finding": { + "cim_type": "endpoint", + "fields": { + "name": ["ss_name"], + "src_device": ["DeviceType"], + "severity": ["severity"] + } + } +} diff --git a/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..0a8236e4c --- /dev/null +++ b/stix_shifter_modules/splunk/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,330 @@ +{ + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "_raw": { + "key": "artifact.payload_bin", + "transformer": "ToBase64", + "object": "artifact" + }, + "mime_type_raw": { + "key": "artifact.mime_type", + "object": "artifact" + }, + "dest_ip": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "network-traffic", + "references": "dst_ip" + } + ], + "src_ip": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "network-traffic", + "references": "src_ip" + } + ], + "dest_port": { + "key": "network-traffic.dst_port", + "object": "network-traffic", + "transformer": "ToInteger" + }, + "src_port": { + "key": "network-traffic.src_port", + "object": "network-traffic", + "transformer": "ToInteger" + }, + "protocol": { + "key": "network-traffic.protocols", + "object": "network-traffic", + "transformer": "ToLowercaseArray" + }, + "_time": [ + { + "key": "first_observed", + "transformer": "SplunkToTimestamp", + "cybox": false + }, + { + "key": "last_observed", + "transformer": "SplunkToTimestamp", + "cybox": false + } + ], + "user": [ + { + "key": "user-account.account_login", + "object": "authentication" + }, + { + "key": "x_splunk_spl.user", + "cybox": false + }, + { + "key": "user-account.user_id", + "object": "authentication" + } + ], + "process_user": [ + { + "key": "windows-registry-key.creator_user_ref", + "object": "windows-registry-key", + "references": "authentication" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "authentication" + } + ], + "bytes": { + "key": "x_splunk_spl.bytes", + "cybox": false + }, + "ssl_hash": [ + { + "key": "x509-certificate.hashes.SHA-256", + "object": "certificate" + } + ], + "ssl_version": { + "key": "x509-certificate.version", + "object": "certificate" + }, + "ssl_serial": { + "key": "x509-certificate.serial_number", + "object": "certificate" + }, + "ssl_signature_algorithm": { + "key": "x509-certificate.signature_algorithm", + "object": "certificate" + }, + "ssl_issuer": { + "key": "x509-certificate.issuer", + "object": "certificate" + }, + "ssl_subject": { + "key": "x509-certificate.subject", + "object": "certificate" + }, + "ssl_publickey_algorithm": { + "key": "x509-certificate.subject_public_key_algorithm", + "object": "certificate" + }, + "process_id": { + "key": "process.pid", + "object": "process" + }, + "file_path": [ + { + "key": "directory.path", + "object": "directory" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "directory" + }, + { + "key": "process.image_ref", + "object": "process", + "references": "file" + } + ], + "file_create_time": [ + { + "key": "directory.ctime", + "object": "directory", + "transformer": "SplunkToTimestamp" + }, + { + "key": "file.ctime", + "object": "file", + "transformer": "SplunkToTimestamp" + } + ], + "file_modify_time": [ + { + "key": "directory.mtime", + "object": "directory", + "transformer": "SplunkToTimestamp" + }, + { + "key": "file.mtime", + "object": "file", + "transformer": "SplunkToTimestamp" + } + ], + "file_hash": { + "key": "file.hashes.UNKNOWN", + "object": "file" + }, + "file_name": { + "key": "file.name", + "object": "file" + }, + "file_size": { + "key": "file.size", + "object": "file" + }, + "src_mac": [ + { + "key": "mac-addr.value", + "object": "src_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "src_ip", + "references": ["src_mac"] + } + ], + "dest_mac": [ + { + "key": "mac-addr.value", + "object": "dst_mac" + }, + { + "key": "ipv4-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + }, + { + "key": "ipv6-addr.resolves_to_refs", + "object": "dst_ip", + "references": ["dst_mac"] + } + ], + "object_path": { + "key": "windows-registry-key.key", + "object": "windows-registry-key" + }, + "url": [ + { + "key": "url.value", + "object": "web" + }, + { + "key": "domain-name.value", + "transformer": "ToDomainName" + } + ], + "src_user": [ + { + "key": "email-addr.value", + "object": "email-addr" + }, + { + "key": "email-message.sender_ref", + "object": "email-message", + "references": "email-addr" + }, + { + "key": "email-message.from_ref", + "object": "email-message", + "references": "email-addr" + } + ], + "subject": { + "key": "email-message.subject", + "object": "email-message" + }, + "is_multipart": { + "key": "email-message.is_multipart", + "object": "email-message", + "transformer": "StringToBool" + }, + "source": [ + { + "key": "x-splunk.log_source", + "object": "x-splunk" + } + ], + "_sourcetype": [ + { + "key": "x-splunk.log_source_type", + "object": "x-splunk" + } + ], + "Direction": [ + { + "key": "x-splunk.direction", + "object": "x-splunk" + } + ], + "EventID": [ + { + "key": "x-splunk.event_id", + "object": "x-splunk" + } + ], + "TacticId": [ + { + "key": "x-splunk.mitre_tactic_id", + "object": "x-splunk" + } + ], + "Tactic": [ + { + "key": "x-splunk.mitre_tactic", + "object": "x-splunk" + } + ], + "TechniqueId": [ + { + "key": "x-splunk.mitre_technique_id", + "object": "x-splunk" + } + ], + "Technique": [ + { + "key": "x-splunk.mitre_technique", + "object": "x-splunk" + } + ], + "EventName": [ + { + "key": "x-splunk.event_name", + "object": "x-splunk" + } + ], + "ss_name": { + "key": "x-ibm-finding.name", + "object": "x-ibm-finding" + }, + "DeviceType": [ + { + "key": "x-ibm-finding.src_device", + "object": "x-ibm-finding" + } + ], + "severity": [ + { + "key": "x-ibm-finding.severity", + "object": "x-ibm-finding" + } + ] +} diff --git a/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/messageActivityData_from_stix_map.json b/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/messageActivityData_from_stix_map.json index 5d2cc3404..e51c95795 100644 --- a/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/messageActivityData_from_stix_map.json +++ b/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/messageActivityData_from_stix_map.json @@ -21,7 +21,7 @@ }, "file": { "fields": { - "hashes.SHA-1": ["file_sha1"], + "hashes.'SHA-1'": ["file_sha1"], "name": ["file_name"] } }, diff --git a/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/endpointActivityData_from_stix_map.json b/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/endpointActivityData_from_stix_map.json new file mode 100644 index 000000000..bf86b525a --- /dev/null +++ b/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/endpointActivityData_from_stix_map.json @@ -0,0 +1,53 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["src", "dst", "objectIp", "objectIps"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["src", "dst", "objectIp", "objectIps"] + } + }, + "url": { + "fields": { + "value": ["request"] + } + }, + "file": { + "fields": { + "hashes.'SHA-1'": ["srcFileHashSha1", "objectFileHashSha1", "parentFileHashSha1", "processFileHashSha1"] + } + }, + "network-traffic": { + "fields": { + "src_ref.value": ["src"], + "dst_ref.value": ["dst", "objectIp"], + "src_port": ["spt"], + "dst_port": ["dpt", "objectPort"] + } + }, + "user-account": { + "fields": { + "account_login": ["logonUser"], + "user_id": ["objectUser"] + } + }, + "process": { + "fields": { + "command_line": ["processCmd", "parentCmd", "objectCmd"] + } + }, + "windows-registry-key": { + "fields": { + "key": ["objectRegistryKeyHandle"], + "values[*].name": ["objectRegistryValue"], + "values[*].data": ["objectRegistryData"] + } + }, + "domain-name": { + "fields": { + "value": ["hostName"] + } + } +} diff --git a/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/messageActivityData_from_stix_map.json b/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/messageActivityData_from_stix_map.json new file mode 100644 index 000000000..e51c95795 --- /dev/null +++ b/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/messageActivityData_from_stix_map.json @@ -0,0 +1,41 @@ +{ + "ipv4-addr": { + "fields": { + "value": ["source_ip"] + } + }, + "ipv6-addr": { + "fields": { + "value": ["source_ip"] + } + }, + "url": { + "fields": { + "value": ["url"] + } + }, + "network-traffic": { + "fields": { + "src_ref.value": ["source_ip"] + } + }, + "file": { + "fields": { + "hashes.'SHA-1'": ["file_sha1"], + "name": ["file_name"] + } + }, + "email-message": { + "fields": { + "sender_ref.value": ["sender", "mailbox"], + "to_refs[*].value": ["recipient", "mailbox"], + "subject": ["subject"], + "message_id": ["message_id"] + } + }, + "domain-name": { + "fields": { + "value": ["source_domain"] + } + } +} diff --git a/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..dd9461884 --- /dev/null +++ b/stix_shifter_modules/trendmicro_vision_one/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,387 @@ +{ + "eventTime": [ + { + "key": "first_observed", + "transformer": "EpochToTimestamp", + "cybox": false + }, + { + "key": "last_observed", + "transformer": "EpochToTimestamp", + "cybox": false + } + ], + "src": [ + { + "key": "ipv4-addr.value", + "object": "src" + }, + { + "key": "ipv6-addr.value", + "object": "src" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src" + }, + { + "key": "network-traffic.protocols", + "object": "nt", + "value": ["tcp"] + } + ], + "spt": [ + { + "key": "network-traffic.src_port", + "object": "nt" + }, + { + "key": "network-traffic.protocols", + "object": "nt", + "value": ["tcp"] + } + ], + "dst": [ + { + "key": "ipv4-addr.value", + "object": "dst" + }, + { + "key": "ipv6-addr.value", + "object": "dst" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst" + }, + { + "key": "network-traffic.protocols", + "object": "nt", + "value": ["tcp"] + } + ], + "dpt": [ + { + "key": "network-traffic.dst_port", + "object": "nt" + }, + { + "key": "network-traffic.protocols", + "object": "nt", + "value": ["tcp"] + } + ], + "endpointIp": [ + { + "key": "ipv4-addr.value", + "unwrap": true, + "transformer": "FilterIPv4List", + "object": "endpointIpV4" + }, + { + "key": "ipv6-addr.value", + "unwrap": true, + "transformer": "FilterIPv6List", + "object": "endpointIpV6" + } + ], + "hostName": { + "key": "domain-name.value" + }, + "objectHostName": { + "key": "domain-name.value" + }, + "logonUser": { + "key": "user-account.account_login", + "object": "userAccount" + }, + "objectUser": { + "key": "user-account.user_id", + "object": "userAccount" + }, + "objectIp": [ + { + "key": "ipv4-addr.value", + "object": "objectIp" + }, + { + "key": "network-traffic.dst_ref", + "object": "object-nt", + "references": "objectIp" + }, + { + "key": "network-traffic.protocols", + "object": "object-nt", + "value": ["tcp"] + } + ], + "objectPort": [ + { + "key": "network-traffic.dst_port", + "object": "object-nt" + }, + { + "key": "network-traffic.protocols", + "object": "object-nt", + "value": ["tcp"] + } + ], + "objectIps": [ + { + "key": "ipv4-addr.value", + "unwrap": true, + "transformer": "FilterIPv4List", + "object": "objectIpsV4" + }, + { + "key": "ipv6-addr.value", + "unwrap": true, + "transformer": "FilterIPv6List", + "object": "objectIpsV6" + } + ], + "objectCmd": { + "key": "process.command_line", + "object": "objectCmd" + }, + "objectFileHashSha1": [ + { + "key": "file.hashes.SHA-1", + "object": "objectFile" + }, + { + "key": "process.image_ref", + "object": "objectCmd", + "references": "objectFile" + } + ], + "objectFilePath": [ + { + "key": "file.name", + "object": "objectFile", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "objectFilePath", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "objectFile", + "references": "objectFilePath" + }, + { + "key": "process.image_ref", + "object": "objectCmd", + "references": "objectFile" + } + ], + "objectRegistryKeyHandle": { + "key": "windows-registry-key.key", + "object": "registry-key", + "transformer": "ConvertWindowsRegistry" + }, + "objectRegistryValueType": { + "key": "windows-registry-key.values", + "object": "registry-key" + }, + "processCmd": { + "key": "process.command_line", + "object": "processCmd" + }, + "processFileHashSha1": [ + { + "key": "file.hashes.SHA-1", + "object": "processFile" + }, + { + "key": "process.image_ref", + "object": "processCmd", + "references": "processFile" + } + ], + "processFilePath": [ + { + "key": "file.name", + "object": "processFile", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "processFilePath", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "processFile", + "references": "processFilePath" + }, + { + "key": "process.image_ref", + "object": "processCmd", + "references": "processFile" + } + ], + "parentCmd": { + "key": "process.command_line", + "object": "parentCmd" + }, + "parentFileHashSha1": [ + { + "key": "file.hashes.SHA-1", + "object": "parentFile" + }, + { + "key": "process.image_ref", + "object": "parentCmd", + "references": "parentFile" + } + ], + "parentFilePath": [ + { + "key": "file.name", + "object": "parentFile", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "parentFilePath", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "parentFile", + "references": "parentFilePath" + }, + { + "key": "process.image_ref", + "object": "parentCmd", + "references": "parentFile" + } + ], + "request": { + "key": "url.value" + }, + "srcFilePath": [ + { + "key": "file.name", + "object": "srcFile", + "transformer": "ToFileName" + }, + { + "key": "directory.path", + "object": "srcFilePath", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "srcFile", + "references": "srcFilePath" + } + ], + "srcFileHashSha1": [ + { + "key": "file.hashes.SHA-1", + "object": "srcFile" + } + ], + "mail_message_sender": [ + { + "key": "email-addr.value", + "object": "sender" + }, + { + "key": "email-message.sender_ref", + "object": "mail", + "references": "sender" + }, + { + "key": "email-message.is_multipart", + "object": "mail", + "value": true + } + ], + "mail_message_recipient": [ + { + "key": "email-addr.value", + "object": "recipient", + "unwrap": true + }, + { + "key": "email-message.to_refs", + "object": "mail", + "references": ["recipient"], + "unwrap": true + }, + { + "key": "email-message.is_multipart", + "object": "mail", + "value": true + } + ], + "mail_message_subject": [ + { + "key": "email-message.subject", + "object": "mail" + }, { + "key": "email-message.is_multipart", + "object": "mail", + "value": true + } + ], + "mail_urls": { + "key": "url.value", + "unwrap": true + }, + "source_domain": { + "key": "domain-name.value" + }, + "source_ip": [ + { + "key": "ipv4-addr.value", + "object": "source_ip" + }, + { + "key": "ipv6-addr.value", + "object": "source_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "message-nt", + "references": "source_ip" + }, + { + "key": "network-traffic.protocols", + "object": "message-nt", + "value": ["tcp"] + } + ], + "mail_message_delivery_time": [ + { + "key": "email-message.date", + "object": "mail" + }, { + "key": "email-message.is_multipart", + "object": "mail", + "value": true + } + ], + "mail_attachments": { + "file_name": { + "key": "file.name", + "object": "attachment" + }, + "file_sha1": { + "key": "file.hashes.SHA-1", + "object": "attachment" + } + }, + "mail_internet_headers": { + "key": "email-message.additional_header_fields", + "object": "mail", + "transformer": "ConvertInternetHeaders" + } +} \ No newline at end of file diff --git a/stix_shifter_utils/modules/base/stix_translation/base_query_translator.py b/stix_shifter_utils/modules/base/stix_translation/base_query_translator.py index d55d9c51d..c8ada0dce 100644 --- a/stix_shifter_utils/modules/base/stix_translation/base_query_translator.py +++ b/stix_shifter_utils/modules/base/stix_translation/base_query_translator.py @@ -31,12 +31,18 @@ def fetch_mapping(self, basepath, dialect, options): :param basepath: path of data source translation module :type basepath: str """ - from_stix_path = path.join(basepath, 'json', f'{dialect}_from_stix_map.json') - if path.isfile(from_stix_path): - return self.read_json(from_stix_path, options) + stix_2_0_mapping_directory_path = path.join(basepath, 'json') + stix_2_1_mapping_directory_path = path.join(basepath, 'json/stix_2_1') + mapping_file = f'{dialect}_from_stix_map.json' + from_stix_path = path.join(basepath, 'json', mapping_file) + if not path.isfile(from_stix_path): + # use default mapping file since 'default_stix_map.json' isn't a real dialect + mapping_file = 'from_stix_map.json' + if options.get("stix_2.1") and path.isdir(stix_2_1_mapping_directory_path): + from_stix_path = path.join(stix_2_1_mapping_directory_path, mapping_file) else: - from_stix_path = path.join(basepath, 'json', 'from_stix_map.json') - return self.read_json(from_stix_path, options) + from_stix_path = path.join(stix_2_0_mapping_directory_path, mapping_file) + return self.read_json(from_stix_path, options) def map_field(self, stix_object_name, stix_property_name): """ diff --git a/stix_shifter_utils/modules/base/stix_translation/base_results_translator.py b/stix_shifter_utils/modules/base/stix_translation/base_results_translator.py index 9d49deba5..827a6a152 100644 --- a/stix_shifter_utils/modules/base/stix_translation/base_results_translator.py +++ b/stix_shifter_utils/modules/base/stix_translation/base_results_translator.py @@ -14,7 +14,12 @@ def __init__(self, options, dialect, base_file_path=None, callback=None): self.module_name = base_file_path.split(os.sep)[-2] self.map_data = {} self.logger = logger.set_logger(__name__) - filepath = os.path.abspath(os.path.join(base_file_path, "json", "to_stix_map.json")) + stix_2_0_mapping_directory_path = os.path.join(base_file_path, 'json') + stix_2_1_mapping_directory_path = os.path.join(base_file_path, 'json/stix_2_1') + if options.get("stix_2.1") and os.path.isdir(stix_2_1_mapping_directory_path): + filepath = os.path.abspath(os.path.join(stix_2_1_mapping_directory_path, "to_stix_map.json")) + else: + filepath = os.path.abspath(os.path.join(stix_2_0_mapping_directory_path, "to_stix_map.json")) self.map_data = self.read_json(filepath, options) self.transformers = get_module_transformers(self.module_name) diff --git a/stix_shifter_utils/stix_translation/src/json_to_stix/id_contributing_properties.json b/stix_shifter_utils/stix_translation/src/json_to_stix/id_contributing_properties.json new file mode 100644 index 000000000..7bd3a99a6 --- /dev/null +++ b/stix_shifter_utils/stix_translation/src/json_to_stix/id_contributing_properties.json @@ -0,0 +1,20 @@ +{ + "artifact": [[ "MD5", "SHA-1", "SHA-256", "SHA-512" ], "payload_bin"], + "autonomous-system": ["number"], + "directory": ["path"], + "domain-name": ["value"], + "email-addr": ["value"], + "email-message": ["from_ref", "subject", "body"], + "file": [[ "MD5", "SHA-1", "SHA-256", "SHA-512" ], "name", "extensions"], + "ipv4-addr": ["value"], + "ipv6-addr": ["value"], + "mac-addr": ["value"], + "mutex": ["name"], + "network-traffic": ["start", "src_ref", "dst_ref", "src_port", "dst_port", "protocols"], + "process": [], + "software": ["name", "cpe", "vendor", "version"], + "url": ["value"], + "user-account": ["account_type", "user_id", "account_login"], + "windows-registry-key": ["key", "values"], + "x509-certificate": [[ "MD5", "SHA-1", "SHA-256", "SHA-512" ], "serial_number"] +} \ No newline at end of file diff --git a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py index 58113ef1f..facf61c23 100644 --- a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py +++ b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py @@ -107,6 +107,7 @@ def _add_property(obj, key, stix_value, group=False): if (isinstance(child_obj[split_key[-1]], list)): child_obj[split_key[-1]].extend(stix_value) # append to existing list + def _handle_cybox_key_def(self, key_to_add, observation, stix_value, obj_name_map, obj_name, group=False): """ Handle the translation of the input property to its STIX CybOX property @@ -117,6 +118,7 @@ def _handle_cybox_key_def(self, key_to_add, observation, stix_value, obj_name_ma :param obj_name: the object name derived from the mapping file """ obj_type, obj_prop = key_to_add.split('.', 1) + objs_dir = observation['objects'] if obj_name in obj_name_map: @@ -350,18 +352,34 @@ def _transform(self, object_map, observation, ds_map, ds_key, obj): # STIX 2.1 helper methods def _generate_and_apply_deterministic_id(self, object_id_map, cybox_objects): # Generates ID based on common namespace and SCO properties (omitting id and spec_version) - # TODO: References may need to be include as part of the ID generation + # TODO: Handle references when part of ID contributing properties + + with open("stix_shifter_utils/stix_translation/src/json_to_stix/id_contributing_properties.json", 'r') as f: + contributing_properties_definitions = json.load(f) + for key, cybox in cybox_objects.items(): - cybox_type = "" - # set id mapping key to original id object_id_map[key] = "" cybox_properties = {} - for property, value in cybox.items(): - if property == "type": - cybox_type = value - if not (property == "id" or re.match(".*_ref$", property)): - cybox_properties[property] = value - unique_id = cybox_type + "--" + str(uuid.uuid5(namespace=uuid.UUID(UUID5_NAMESPACE), name=json.dumps(cybox_properties))) + cybox_type = cybox.get("type") + contributing_properties = contributing_properties_definitions.get(cybox_type) + + if contributing_properties: + for contr_prop in contributing_properties: + if type(contr_prop) is list: # list of hash types + for hashtype in contr_prop: + hash_prop = "hashes.{}".format(hashtype) + if hash_prop in cybox: + cybox_properties[hash_prop] = cybox[hash_prop] + break + elif contr_prop in cybox and not re.match(".*_ref$", contr_prop): # chicken and egg problem with refs + cybox_properties[contr_prop] = cybox[contr_prop] + if cybox_properties: + unique_id = cybox_type + "--" + str(uuid.uuid5(namespace=uuid.UUID(UUID5_NAMESPACE), name=json.dumps(cybox_properties))) + else: + self.logger.error("STIX object '{}' needs at least one of the following properties to generate ID {}".format(cybox_type, contributing_properties)) + else: # STIX process or custom object used UUID4 for identifier + unique_id = "{}--{}".format(cybox_type, str(uuid.uuid4())) + # set id mapping value to new id object_id_map[key] = unique_id # replace old id with new diff --git a/stix_shifter_utils/stix_translation/src/utils/transformers.py b/stix_shifter_utils/stix_translation/src/utils/transformers.py index efca74743..f5e966471 100644 --- a/stix_shifter_utils/stix_translation/src/utils/transformers.py +++ b/stix_shifter_utils/stix_translation/src/utils/transformers.py @@ -28,7 +28,7 @@ def transform(value): class EpochToTimestamp(ValueTransformer): - """A value transformer for the timestamps""" + """A value transformer for the 13-digit timestamps""" @staticmethod def transform(epoch): diff --git a/tests/stix_translation/test_results_translation.py b/tests/stix_translation/test_results_translation.py index e57e26ae2..b630061e6 100644 --- a/tests/stix_translation/test_results_translation.py +++ b/tests/stix_translation/test_results_translation.py @@ -1,15 +1,14 @@ from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator from stix_shifter_modules.mysql.entry_point import EntryPoint from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers +import json MODULE = 'mysql' RESULTS = 'results' TRANSFORMERS = get_module_transformers(MODULE) -epoch_to_timestamp_class = TRANSFORMERS.get('EpochToTimestamp') -EPOCH_START = 1531169112 -EPOCH_END = 1531169254 -entry_point = EntryPoint() -MAP_DATA = entry_point.get_results_translator().map_data +EPOCH = 1634657528000 +TIMESTAMP = "2021-10-19T15:32:08.000Z" + DATA_SOURCE = { "type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", @@ -28,7 +27,27 @@ "filename": "somefile.exe", "sha256hash": "sha256_hash", "md5hash": "md5_hash", - "file_path": "C:/some/path/" + "file_path": "C:/some/path/", + "file_created_time": EPOCH, + "file_modified_time": EPOCH, + "file_accessed_time": EPOCH, + "directory_created_time": EPOCH, + "directory_modified_time": EPOCH, + "directory_accessed_time": EPOCH, + "process_id": 12345, + "process_name": "file executed", + "process_arguments": "some args", + "process_created_time": EPOCH + } + +CYBOX_ID = { + "source-ipv4-addr": "ipv4-addr--0b6a89e3-e345-51b7-a8ee-aaff7ebf2df5", + "dest-ipv4-addr": "ipv4-addr--cb8e152d-60f0-596a-81e4-a22cc4a7f063", + "url": "url--8265905f-c609-52e3-ae52-6681bcd6086d", + "user-account": "user-account--3cd7ffc9-89f7-5b58-948c-117ec9b3e22a", + "network-traffic": "network-traffic--2ec70516-29b5-59f3-9743-3b93e97db6d8", + "file": "file--243f1b5f-0391-501c-bed0-17e9f204f1d2", + "directory": "directory--9ce39e76-d59e-5db2-8f0e-2001f689ea9d" } OPTIONS = {} @@ -68,10 +87,9 @@ def get_object_keys(objects): yield id_val def test_common_prop(self): - DATA = {"entry_time": EPOCH_START, "entry_time": EPOCH_END, "eventcount": 1} - - result_bundle = json_to_stix_translator.convert_to_stix( - DATA_SOURCE, MAP_DATA, [DATA], TRANSFORMERS, {}) + DATA = {"entry_time": EPOCH, "entry_time": EPOCH, "eventcount": 1} + entry_point = EntryPoint() + result_bundle = entry_point.translate_results(json.dumps(DATA_SOURCE), json.dumps([DATA])) assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] @@ -95,9 +113,8 @@ def test_common_prop(self): assert observed_data['last_observed'] def test_STIX_2_0_cybox_observables(self): - - result_bundle = json_to_stix_translator.convert_to_stix( - DATA_SOURCE, MAP_DATA, [DATA], TRANSFORMERS, {}) + entry_point = EntryPoint() + result_bundle = entry_point.translate_results(json.dumps(DATA_SOURCE), json.dumps([DATA])) assert result_bundle['type'] == 'bundle' assert "spec_version" in result_bundle @@ -145,26 +162,51 @@ def test_STIX_2_0_cybox_observables(self): assert "user_id" in stix_object and stix_object['user_id'] == DATA['username'] # file - stix_object = TestTransform.get_first_of_type(objects.values(), 'file') - assert stix_object, 'file object type not found' - assert "name" in stix_object and stix_object['name'] == DATA['filename'] - assert "hashes" in stix_object - hashes = stix_object["hashes"] + file_object = TestTransform.get_first_of_type(objects.values(), 'file') + assert file_object, 'file object type not found' + assert "name" in file_object and file_object['name'] == DATA['filename'] + assert "created" in file_object and file_object['created'] == TIMESTAMP + assert "ctime" not in file_object + assert "modified" in file_object and file_object['modified'] == TIMESTAMP + assert "mtime" not in file_object + assert "accessed" in file_object and file_object['accessed'] == TIMESTAMP + assert "atime" not in file_object + assert "parent_directory_ref" in file_object + assert "hashes" in file_object + hashes = file_object["hashes"] assert "MD5" in hashes and hashes["MD5"] == DATA["md5hash"] - assert "SHA-256" in hashes and hashes["SHA-256"] == DATA["sha256hash"] + assert "SHA-256" in hashes and hashes["SHA-256"] == DATA["sha256hash"] - directory_ref = stix_object['parent_directory_ref'] - assert directory_ref in objects, f"dst_ref with key {stix_object['parent_directory_ref']} not found" + directory_ref = file_object['parent_directory_ref'] + assert directory_ref in objects, f"dst_ref with key {file_object['parent_directory_ref']} not found" # directory - stix_object = TestTransform.get_first_of_type(objects.values(), 'directory') - assert stix_object, 'directory object type not found' - assert "path" in stix_object and stix_object["path"] == DATA["file_path"] + directory_object = TestTransform.get_first_of_type(objects.values(), 'directory') + assert directory_object, 'directory object type not found' + assert "path" in directory_object and directory_object["path"] == DATA["file_path"] + assert "created" in directory_object and directory_object['created'] == TIMESTAMP + assert "ctime" not in directory_object + assert "modified" in directory_object and directory_object['modified'] == TIMESTAMP + assert "mtime" not in directory_object + assert "accessed" in directory_object and directory_object['accessed'] == TIMESTAMP + assert "atime" not in directory_object + + # process + process_object = TestTransform.get_first_of_type(objects.values(), 'process') + assert process_object, 'process object type not found' + assert "name" in process_object and process_object['name'] == DATA['process_name'] + assert "pid" in process_object and process_object['pid'] == DATA['process_id'] + assert "arguments" in process_object and process_object['arguments'] == DATA['process_arguments'] + assert "created" in process_object and process_object['created'] == TIMESTAMP + assert "binary_ref" in process_object + assert "image_ref" not in process_object def test_STIX_2_1_cybox_observables(self): - - result_bundle = json_to_stix_translator.convert_to_stix( - DATA_SOURCE, MAP_DATA, [DATA], TRANSFORMERS, {"stix_2.1": "true"}) + options = { + "stix_2.1": True + } + entry_point = EntryPoint(options=options) + result_bundle = entry_point.translate_results(json.dumps(DATA_SOURCE), json.dumps([DATA])) assert result_bundle['type'] == 'bundle' assert "spec_version" not in result_bundle @@ -182,40 +224,71 @@ def test_STIX_2_1_cybox_observables(self): assert "src_port" in network_traffic_object and network_traffic_object['src_port'] == 3000 assert "dst_port" in network_traffic_object and network_traffic_object['dst_port'] == 2000 assert "protocols" in network_traffic_object and network_traffic_object['protocols'] == ['tcp'] + assert "id" in network_traffic_object and network_traffic_object['id'] == CYBOX_ID["network-traffic"] # destination ipv4-addr destination_ipv4_object = TestTransform.get_first_cybox_of_id_stix_2_1(result_bundle_objects, network_traffic_object["dst_ref"]) assert "type" in destination_ipv4_object and destination_ipv4_object['type'] == 'ipv4-addr' assert "value" in destination_ipv4_object and destination_ipv4_object['value'] == DATA["dest_ipaddr"] + assert "id" in destination_ipv4_object and destination_ipv4_object['id'] == CYBOX_ID["dest-ipv4-addr"] # source ipv4-addr source_ipv4_object = TestTransform.get_first_cybox_of_id_stix_2_1(result_bundle_objects, network_traffic_object["src_ref"]) assert "type" in source_ipv4_object and source_ipv4_object['type'] == 'ipv4-addr' assert "value" in source_ipv4_object and source_ipv4_object['value'] == DATA["source_ipaddr"] + assert "id" in source_ipv4_object and source_ipv4_object['id'] == CYBOX_ID["source-ipv4-addr"] # url url_object = TestTransform.get_first_cybox_of_type_stix_2_1(result_bundle_objects, 'url') assert url_object, 'url object type not found' assert "value" in url_object and url_object['value'] == DATA['url'] + assert "id" in url_object and url_object['id'] == CYBOX_ID["url"] # user-account user_account_object = TestTransform.get_first_cybox_of_type_stix_2_1(result_bundle_objects, 'user-account') assert user_account_object, 'user-account object type not found' assert "user_id" in user_account_object and user_account_object['user_id'] == DATA['username'] + assert "id" in user_account_object and user_account_object['id'] == CYBOX_ID["user-account"] # file file_object = TestTransform.get_first_cybox_of_type_stix_2_1(result_bundle_objects, 'file') assert file_object, 'file object type not found' assert "name" in file_object and file_object['name'] == DATA['filename'] + assert "ctime" in file_object and file_object['ctime'] == TIMESTAMP + assert "created" not in file_object + assert "mtime" in file_object and file_object['mtime'] == TIMESTAMP + assert "modified" not in file_object + assert "atime" in file_object and file_object['atime'] == TIMESTAMP + assert "accessed" not in file_object + assert "parent_directory_ref" in file_object assert "hashes" in file_object hashes = file_object["hashes"] assert "MD5" in hashes and hashes["MD5"] == DATA["md5hash"] assert "SHA-256" in hashes and hashes["SHA-256"] == DATA["sha256hash"] assert "parent_directory_ref" in file_object + assert "id" in file_object and file_object['id'] == CYBOX_ID["file"] # directory directory_object = TestTransform.get_first_cybox_of_id_stix_2_1(result_bundle_objects, file_object["parent_directory_ref"]) assert directory_object, 'directory object type not found' assert "path" in directory_object and directory_object["path"] == DATA["file_path"] + assert "ctime" in directory_object and directory_object['ctime'] == TIMESTAMP + assert "created" not in directory_object + assert "mtime" in directory_object and directory_object['mtime'] == TIMESTAMP + assert "modified" not in directory_object + assert "atime" in directory_object and directory_object['atime'] == TIMESTAMP + assert "accessed" not in directory_object + + # process + process_object = TestTransform.get_first_of_type(result_bundle_objects, 'process') + assert process_object, 'process object type not found' + assert "name" not in process_object + assert "pid" in process_object and process_object['pid'] == DATA['process_id'] + assert "arguments" in process_object and process_object['pid'] == DATA['process_id'] + assert "created_time" in process_object and process_object['arguments'] == DATA['process_arguments'] + assert "created" not in process_object + assert "image_ref" in process_object + assert "binary_ref" not in process_object + assert "id" in directory_object and directory_object['id'] == CYBOX_ID["directory"] diff --git a/tests/stix_translation/test_transformers.py b/tests/stix_translation/test_transformers.py new file mode 100644 index 000000000..e0d359408 --- /dev/null +++ b/tests/stix_translation/test_transformers.py @@ -0,0 +1,19 @@ +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers + +MODULE = 'mysql' +TRANSFORMERS = get_module_transformers(MODULE) +epoch_to_timestamp_class = TRANSFORMERS.get('EpochToTimestamp') +timestamp_to_epoch_class = TRANSFORMERS.get('TimestampToMilliseconds') +EPOCH = 1634657528000 +TIMESTAMP = "2021-10-19T15:32:08.000Z" + +class TestTransform(object): + + def test_epoch_to_timestamp_conversion(self): + conversion = epoch_to_timestamp_class.transform(EPOCH) + assert conversion == TIMESTAMP + + def test_timestamp_to_epoch_conversion(self): + conversion = timestamp_to_epoch_class.transform(TIMESTAMP) + assert conversion == EPOCH + From e26fddce2f99007b0ef4ce3a7dd9ef81490ba568 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Fri, 26 Nov 2021 16:48:38 +0530 Subject: [PATCH 27/30] Ignoring .idea pycharm files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index adaf55ca9..4fab91332 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,5 @@ virtualenv/ *.sqlite deployment/ibm_cloud_pak_for_security/backup_* deployment/ibm_cloud_pak_for_security/bundle +.idea +.idea/ \ No newline at end of file From 3d9f761d7e83e7d73f8a770f3f252dd4e1b29350 Mon Sep 17 00:00:00 2001 From: JyotiWhcl <91883455+JyotiWhcl@users.noreply.github.com> Date: Fri, 26 Nov 2021 16:51:29 +0530 Subject: [PATCH 28/30] Deleting .idea directory Deleting .idea directory --- .idea/.gitignore | 0 .../inspectionProfiles/profiles_settings.xml | 6 - .idea/misc.xml | 7 - .idea/modules.xml | 8 - .../shelved.patch | 19 - ...mitted_changes_before_rebase__Changes_.xml | 4 - .idea/stix-shifter.iml | 12 - .idea/vcs.xml | 6 - .idea/workspace.xml | 424 ------------------ 9 files changed, 486 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/shelf/Uncommitted_changes_before_rebase_[Changes]/shelved.patch delete mode 100644 .idea/shelf/Uncommitted_changes_before_rebase__Changes_.xml delete mode 100644 .idea/stix-shifter.iml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da2..000000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 21b7dee60..000000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 0a6a40f57..000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/shelf/Uncommitted_changes_before_rebase_[Changes]/shelved.patch b/.idea/shelf/Uncommitted_changes_before_rebase_[Changes]/shelved.patch deleted file mode 100644 index 35fa70b12..000000000 --- a/.idea/shelf/Uncommitted_changes_before_rebase_[Changes]/shelved.patch +++ /dev/null @@ -1,19 +0,0 @@ -Index: stix_shifter_modules/proofpoint/stix_transmission/results_connector.py -IDEA additional info: -Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP -<+>import json\n\nfrom stix_shifter_utils.modules.base.stix_transmission.base_results_connector import BaseResultsConnector\nfrom stix_shifter_utils.utils.error_response import ErrorResponder\nfrom stix_shifter_utils.utils import logger\n\nclass ResultsConnector(BaseResultsConnector):\n def __init__(self, api_client):\n self.api_client = api_client\n self.logger = logger.set_logger(__name__)\n\n\n def create_results_connection(self, search_id, offset, length):\n try:\n min_range = int(offset)\n max_range = min_range + int(length)\n\n # Grab the response, extract the response code, and convert it to readable json\n response = self.api_client.get_search_results(search_id)\n response_code = response.code\n response_txt = response.read()\n # Construct a response object\n # print(\"response code={}, text={}\".format(response_code, response_txt))\n return_obj = dict()\n error_obj = dict()\n if response_code == 200:\n return_obj['success'] = True\n try:\n response_txt = response.read().decode('utf-8')\n data= json.loads(response_txt)\n newdata = list()\n for key, value in data.items():\n if isinstance(value, list) and value:\n newdata+=value\n\n # slice off the data count according to offset values\n if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range:\n newdata = newdata[min_range:max_range]\n\n for msg in newdata:\n if \"messageParts\" in msg:\n msg[\"is_multipart\"] = True\n else: msg[\"is_multipart\"] = False\n\n print(\"newdata :\", newdata)\n return_obj['data'] = newdata\n\n except json.decoder.JSONDecodeError as err:\n return_obj['success'] = False\n except Exception as err:\n return_obj['success'] = False\n self.logger.error('Response decode error: {}'.format(err))\n\n elif response_code > 200 and response_code <= 204:#empty results\n error_obj['code'] = 2000\n elif response_code == 400:#error from data source\n error_obj['code'] = 4000\n elif response_code == 401:#Authentication error\n error_obj['code'] = 4010\n elif response_code >= 402 and response_code <= 499:#All other client side errors\n error_obj['code'] = 4020\n else:\n #unknown errors\n error_obj['code'] = 7000\n if error_obj:\n error_msg = \"\"\n try:\n error_msg = str(response_txt.decode(\"utf-8\"))\n except Exception as err:\n self.logger.error('Response decode error: {}'.format(err))\n error_obj['message'] = error_msg\n ErrorResponder.fill_error(return_obj,\n error_obj,\n ['message'])\n err = 'error when getting search results: {}:{}'.format(str(response_code), error_msg)\n self.logger.error(err)\n # raise NoResultsFoundError(err)\n return return_obj\n except Exception as err:\n self.logger.error('error when getting search results: {}'.format(err))\n raise\n\nclass NoResultsFoundError(Exception):\n pass\n -Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP -<+>UTF-8 -=================================================================== -diff --git a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py ---- a/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py (revision ca9282c041bd9ba8581136708abe77290802c1d9) -+++ b/stix_shifter_modules/proofpoint/stix_transmission/results_connector.py (date 1637338725740) -@@ -34,7 +34,7 @@ - newdata+=value - - # slice off the data count according to offset values -- if newdata and min_range > 0 and max_range > 0 and len(newdata) > max_range: -+ if newdata and max_range > 0 and len(newdata) > max_range: - newdata = newdata[min_range:max_range] - - for msg in newdata: diff --git a/.idea/shelf/Uncommitted_changes_before_rebase__Changes_.xml b/.idea/shelf/Uncommitted_changes_before_rebase__Changes_.xml deleted file mode 100644 index 5bc59623a..000000000 --- a/.idea/shelf/Uncommitted_changes_before_rebase__Changes_.xml +++ /dev/null @@ -1,4 +0,0 @@ - - \ No newline at end of file diff --git a/.idea/stix-shifter.iml b/.idea/stix-shifter.iml deleted file mode 100644 index 4871eb071..000000000 --- a/.idea/stix-shifter.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f4..000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index cfd13f936..000000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,424 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1633433539700 - - - 1633436100542 - - - 1633674880650 - - - 1634021028673 - - - 1634618859315 - - - 1634649832103 - - - 1634723635852 - - - 1634894465227 - - - 1635159453857 - - - 1635160043136 - - - 1635165788579 - - - 1635166248747 - - - 1635166433818 - - - 1635233655540 - - - 1635233683219 - - - 1635236963324 - - - 1635257040425 - - - 1635327579105 - - - 1635338419503 - - - 1637226717777 - - - 1637229485097 - - - 1637294032908 - - - 1637338365309 - - - 1637339560253 - - - 1637832674207 - - - 1637847209146 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From ed8f16bb20cc6e4ed2340849eb888cb051920b6e Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 30 Nov 2021 17:32:53 +0530 Subject: [PATCH 29/30] deleted .idea entries --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 4fab91332..c6a503568 100644 --- a/.gitignore +++ b/.gitignore @@ -77,6 +77,4 @@ virtualenv/ .vscode *.sqlite deployment/ibm_cloud_pak_for_security/backup_* -deployment/ibm_cloud_pak_for_security/bundle -.idea -.idea/ \ No newline at end of file +deployment/ibm_cloud_pak_for_security/bundle \ No newline at end of file From 511eb4e39eedfacc20261c84103f75953cdb4790 Mon Sep 17 00:00:00 2001 From: "jyoti.wakchaure" Date: Tue, 30 Nov 2021 17:40:24 +0530 Subject: [PATCH 30/30] deleted ./idea entries --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c6a503568..adaf55ca9 100644 --- a/.gitignore +++ b/.gitignore @@ -77,4 +77,4 @@ virtualenv/ .vscode *.sqlite deployment/ibm_cloud_pak_for_security/backup_* -deployment/ibm_cloud_pak_for_security/bundle \ No newline at end of file +deployment/ibm_cloud_pak_for_security/bundle