diff --git a/stix_shifter_modules/cybereason/configuration/lang_en.json b/stix_shifter_modules/cybereason/configuration/lang_en.json index 481d1de8c..d66326680 100644 --- a/stix_shifter_modules/cybereason/configuration/lang_en.json +++ b/stix_shifter_modules/cybereason/configuration/lang_en.json @@ -3,7 +3,7 @@ "host": { "label": "Management IP address or Hostname", "placeholder": "192.168.1.10", - "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" + "description": "Specify the IP address or hostname of the data source" }, "port": { "label": "Host Port", diff --git a/stix_shifter_modules/cybereason/stix_translation/query_constructor.py b/stix_shifter_modules/cybereason/stix_translation/query_constructor.py index 690470b29..f4e4ec33b 100644 --- a/stix_shifter_modules/cybereason/stix_translation/query_constructor.py +++ b/stix_shifter_modules/cybereason/stix_translation/query_constructor.py @@ -78,7 +78,8 @@ def _format_datetime(value): """ try: time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ' - + if re.search(r"\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}Z", str(value)): # without milli seconds + time_pattern = '%Y-%m-%dT%H:%M:%SZ' epoch = datetime(1970, 1, 1) converted_time = int(((datetime.strptime(value, time_pattern) - epoch).total_seconds()) * 1000) diff --git a/stix_shifter_modules/cybereason/stix_transmission/connector.py b/stix_shifter_modules/cybereason/stix_transmission/connector.py index fe27397c5..923c4ee6f 100644 --- a/stix_shifter_modules/cybereason/stix_transmission/connector.py +++ b/stix_shifter_modules/cybereason/stix_transmission/connector.py @@ -53,7 +53,7 @@ def create_results_connection(self, query, offset, length): response_dict = json.loads(response_wrapper.read().decode('utf-8')) results = self.get_results_data(response_dict) - return_obj['data'] = results[offset:length] + return_obj['data'] = results[offset:(offset+length)] # session log out response_wrapper = self.api_client.session_log_out(response_wrapper) diff --git a/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py b/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py index 5aa16d825..fda68241e 100644 --- a/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py +++ b/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py @@ -1985,4 +1985,39 @@ def test_invalid_email_address(self): result = translation.translate('cybereason', 'query', '{}', stix_pattern) assert result['success'] is False assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code'] - assert 'Invalid email address' in result['error'] \ No newline at end of file + assert 'Invalid email address' in result['error'] + + def test_timestamp_in_seconds_and_milliseconds(self): + stix_pattern = "[network-traffic:src_port = 23]START t'2019-10-01T08:00:10Z' STOP t'2019-11-30T11:00:10Z' AND" \ + "[network-traffic:protocols[*] = 'tcp'] START t'2019-10-01T08:43:10.003Z' STOP " \ + "t'2019-11-30T10:43:10.005Z' " + query = translation.translate('cybereason', 'query', '{}', stix_pattern) + queries = [{'queryPath': [{'requestedType': 'Connection', 'filters': [{'facetName': 'transportProtocol', + 'filterType': 'Equals', + 'values': ['tcp']}, + {'facetName': 'creationTime', + 'filterType': 'Between', + 'values': [1569919390003, + 1575110590005]}, + {'facetName': 'localPort', + 'filterType': 'Equals', + 'values': [23]}, + {'facetName': 'creationTime', + 'filterType': 'Between', + 'values': [1569916810000, + 1575111610000]}], + 'isResult': True}], 'queryLimits': {'groupingFeature': + {'elementInstanceType': 'Connection', + 'featureName': 'elementDisplayName'}}, + 'perFeatureLimit': 1, + 'totalResultLimit': 9999, 'perGroupLimit': 1, 'templateContext': 'CUSTOM', + 'customFields': ['elementDisplayName', 'direction', 'ownerMachine', 'ownerProcess', 'serverPort', + 'serverAddress', 'portType', 'aggregatedReceivedBytesCount', + 'aggregatedTransmittedBytesCount', 'remoteAddressCountryName', 'dnsQuery', + 'calculatedCreationTime', 'domainName', 'endTime', 'localPort', 'portDescription', + 'remotePort', 'state', 'isExternalConnection', 'isIncoming', + 'remoteAddressInternalExternalLocal', 'transportProtocol', 'hasMalops', + 'hasSuspicions', 'relatedToMalop', 'isWellKnownPort', 'isProcessLegit', + 'isProcessMalware', 'localAddress', 'remoteAddress', 'urlDomains']}] + + self._test_query_assertions(query, queries) diff --git a/stix_shifter_modules/darktrace/configuration/lang_en.json b/stix_shifter_modules/darktrace/configuration/lang_en.json index 683853e55..cb95009dc 100644 --- a/stix_shifter_modules/darktrace/configuration/lang_en.json +++ b/stix_shifter_modules/darktrace/configuration/lang_en.json @@ -2,7 +2,7 @@ "connection": { "host": { "label": "Management IP address or Hostname", - "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" + "description": "Specify the IP address or hostname of the data source" }, "help": { "label": "Need additional help?", diff --git a/stix_shifter_modules/darktrace/test/stix_translation/test_darktrace_stix_to_query.py b/stix_shifter_modules/darktrace/test/stix_translation/test_darktrace_stix_to_query.py index 4a10d506c..1bcf2a75f 100644 --- a/stix_shifter_modules/darktrace/test/stix_translation/test_darktrace_stix_to_query.py +++ b/stix_shifter_modules/darktrace/test/stix_translation/test_darktrace_stix_to_query.py @@ -650,3 +650,20 @@ def test_combinedcomparison_and_or_operator(self): }] expected_query = _remove_timestamp_from_query(expected_query) self._test_query_assertions(actual_query, expected_query) + + def test_qualifier_without_milliseconds(self): + stix_pattern = "[x-oca-asset:hostname = '169.254.169.254'] " \ + "START t'2022-03-01T11:50:21Z' STOP t'2022-03-31T11:55:25Z'" + actual_query = translation.translate('darktrace', 'query', '{}', stix_pattern) + expected_query = [{ + "search": "(@fields.host:\"169.254.169.254\" AND (@fields.epochdate :>1646135421.0 " + "AND @fields.epochdate :<1648727725.0))", + "fields": [], + "timeframe": "custom", + "time": { + "from": "2022-03-01T11:50:21.000000Z", + "to": "2022-03-31T11:55:25.000000Z" + }, + "size": 10000 + }] + self._test_query_assertions(actual_query, expected_query) diff --git a/stix_shifter_modules/paloalto/configuration/lang_en.json b/stix_shifter_modules/paloalto/configuration/lang_en.json index be3f57075..a47affc0b 100644 --- a/stix_shifter_modules/paloalto/configuration/lang_en.json +++ b/stix_shifter_modules/paloalto/configuration/lang_en.json @@ -2,7 +2,7 @@ "connection": { "host": { "label": "Management IP address or Hostname", - "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" + "description": "Specify the IP address or hostname of the data source" }, "quota_threshold": { "label": "The quota limit for the API", diff --git a/stix_shifter_modules/paloalto/stix_translation/query_constructor.py b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py index 3ff5fae8d..a81381aef 100644 --- a/stix_shifter_modules/paloalto/stix_translation/query_constructor.py +++ b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py @@ -249,6 +249,8 @@ def _format_datetime(value): """ try: time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ' + if re.search(r"\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}Z", str(value)): # without milli seconds + time_pattern = '%Y-%m-%dT%H:%M:%SZ' epoch = datetime(1970, 1, 1) converted_time = int(((datetime.strptime(value, time_pattern) - epoch).total_seconds()) * 1000) diff --git a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py index 533614be8..fb7ecae06 100644 --- a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py +++ b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py @@ -126,6 +126,8 @@ def format_results_data(result_data, to_stix_mapping, mandatory_map): if value is not None and value != "NULL" and value != '' and field != 'dataset_name' \ and (field in to_stix_mapping[dataset_map].keys()): stix_data_map = to_stix_mapping[dataset_map][field] + if '\x00' in str(value): + value = value.replace('\x00', '') data = ResultsConnector.check_object(stix_data_map, mandatory_map, data, log, field, value) elif field == 'dataset_name': @@ -160,6 +162,8 @@ def format_stream_data(stream_data, to_stix_mapping, mandatory_map): if value is not None and value != "NULL" and value != '' and field != 'dataset_name' \ and (field in to_stix_mapping[dataset_map].keys()): stix_data_map = to_stix_mapping[dataset_map][field] + if '\x00' in str(value): + value = value.replace('\x00', '') data = ResultsConnector.check_object(stix_data_map, mandatory_map, data, log_dict, field, value) elif field == 'dataset_name': diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py index 68e7a80d9..4a6b4b5fb 100644 --- a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py +++ b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py @@ -559,3 +559,17 @@ def test_format_timestamp_fields(self): "'to': 1645636157746}}}"] queries = _remove_timestamp_from_query(queries) self._test_query_assertions(query, queries) + + def test_qualifier_without_milliseconds(self): + stix_pattern = "[ipv4-addr:value = '10.0.1.4' AND network-traffic:src_port = 52221] " \ + "START t'2022-02-01T08:43:10Z' STOP t'2022-04-07T10:43:10Z'" + query = translation.translate('paloalto', 'query', '{}', stix_pattern) + queries = [{'xdr_data': {'query': 'dataset = xdr_data | filter (action_local_port = 52221 ' + 'and (action_local_ip = "10.0.1.4" or action_remote_ip = "10.0.1.4"' + ' or agent_ip_addresses = "10.0.1.4") and ' + '(to_epoch(_time,"millis") >= 1643704990000 and ' + 'to_epoch(_time,"millis") <= 1649328190000)) | alter dataset_name = ' + '"xdr_data" | fields ' + all_fields + ' | limit 10000 ', + 'timeframe': {'from': 1643704990000, 'to': 1649328190000}}}] + + self._test_query_assertions(query, queries) diff --git a/stix_shifter_modules/sentinelone/configuration/lang_en.json b/stix_shifter_modules/sentinelone/configuration/lang_en.json index 9ad973dad..b00ad7f3c 100644 --- a/stix_shifter_modules/sentinelone/configuration/lang_en.json +++ b/stix_shifter_modules/sentinelone/configuration/lang_en.json @@ -2,7 +2,7 @@ "connection": { "host": { "label": "Management IP address or Hostname", - "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" + "description": "Specify the IP address or hostname of the data source" }, "port": { "label": "Host Port",