cybereason timestamp fix

opencybersecurityalliance · Dec 13, 2022 · a1773c2 · a1773c2
1 parent aca9c49
commit a1773c2
Show file tree

Hide file tree

Showing 11 changed files with 80 additions and 7 deletions.
diff --git a/stix_shifter_modules/cybereason/configuration/lang_en.json b/stix_shifter_modules/cybereason/configuration/lang_en.json
@@ -3,7 +3,7 @@
         "host": {
             "label": "Management IP address or Hostname",
             "placeholder": "192.168.1.10",
-            "description": "Specify the IP address or  hostname of the data source so that IBM Cloud Pak for Security can communicate with it"
+            "description": "Specify the IP address or hostname of the data source"
         },
         "port": {
             "label": "Host Port",

diff --git a/stix_shifter_modules/cybereason/stix_translation/query_constructor.py b/stix_shifter_modules/cybereason/stix_translation/query_constructor.py
@@ -78,7 +78,8 @@ def _format_datetime(value):
         """
         try:
             time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ'
-
+            if re.search(r"\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}Z", str(value)):  # without milli seconds
+                time_pattern = '%Y-%m-%dT%H:%M:%SZ'
             epoch = datetime(1970, 1, 1)
             converted_time = int(((datetime.strptime(value,
                                                      time_pattern) - epoch).total_seconds()) * 1000)

diff --git a/stix_shifter_modules/cybereason/stix_transmission/connector.py b/stix_shifter_modules/cybereason/stix_transmission/connector.py
@@ -53,7 +53,7 @@ def create_results_connection(self, query, offset, length):
 
             response_dict = json.loads(response_wrapper.read().decode('utf-8'))
             results = self.get_results_data(response_dict)
-            return_obj['data'] = results[offset:length]
+            return_obj['data'] = results[offset:(offset+length)]
 
             # session log out
             response_wrapper = self.api_client.session_log_out(response_wrapper)

diff --git a/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py b/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py
@@ -1985,4 +1985,39 @@ def test_invalid_email_address(self):
         result = translation.translate('cybereason', 'query', '{}', stix_pattern)
         assert result['success'] is False
         assert ErrorCode.TRANSLATION_NOTIMPLEMENTED_MODE.value == result['code']
-        assert 'Invalid email address' in result['error']
+        assert 'Invalid email address' in result['error']
+
+    def test_timestamp_in_seconds_and_milliseconds(self):
+        stix_pattern = "[network-traffic:src_port = 23]START t'2019-10-01T08:00:10Z' STOP t'2019-11-30T11:00:10Z' AND" \
+                       "[network-traffic:protocols[*] = 'tcp'] START t'2019-10-01T08:43:10.003Z' STOP " \
+                       "t'2019-11-30T10:43:10.005Z' "
+        query = translation.translate('cybereason', 'query', '{}', stix_pattern)
+        queries = [{'queryPath': [{'requestedType': 'Connection', 'filters': [{'facetName': 'transportProtocol',
+                                                                               'filterType': 'Equals',
+                                                                               'values': ['tcp']},
+                                                                              {'facetName': 'creationTime',
+                                                                               'filterType': 'Between',
+                                                                               'values': [1569919390003,
+                                                                                          1575110590005]},
+                                                                              {'facetName': 'localPort',
+                                                                               'filterType': 'Equals',
+                                                                               'values': [23]},
+                                                                              {'facetName': 'creationTime',
+                                                                               'filterType': 'Between',
+                                                                               'values': [1569916810000,
+                                                                                          1575111610000]}],
+                                   'isResult': True}], 'queryLimits': {'groupingFeature':
+                                                                           {'elementInstanceType': 'Connection',
+                                                                            'featureName': 'elementDisplayName'}},
+                    'perFeatureLimit': 1,
+                    'totalResultLimit': 9999, 'perGroupLimit': 1, 'templateContext': 'CUSTOM',
+                    'customFields': ['elementDisplayName', 'direction', 'ownerMachine', 'ownerProcess', 'serverPort',
+                                     'serverAddress', 'portType', 'aggregatedReceivedBytesCount',
+                                     'aggregatedTransmittedBytesCount', 'remoteAddressCountryName', 'dnsQuery',
+                                     'calculatedCreationTime', 'domainName', 'endTime', 'localPort', 'portDescription',
+                                     'remotePort', 'state', 'isExternalConnection', 'isIncoming',
+                                     'remoteAddressInternalExternalLocal', 'transportProtocol', 'hasMalops',
+                                     'hasSuspicions', 'relatedToMalop', 'isWellKnownPort', 'isProcessLegit',
+                                     'isProcessMalware', 'localAddress', 'remoteAddress', 'urlDomains']}]
+
+        self._test_query_assertions(query, queries)
diff --git a/stix_shifter_modules/darktrace/configuration/lang_en.json b/stix_shifter_modules/darktrace/configuration/lang_en.json
@@ -2,7 +2,7 @@
     "connection": {
         "host": {
             "label": "Management IP address or Hostname",
-            "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it"
+            "description": "Specify the IP address or hostname of the data source"
         },
         "help": {
             "label": "Need additional help?",

diff --git a/stix_shifter_modules/darktrace/test/stix_translation/test_darktrace_stix_to_query.py b/stix_shifter_modules/darktrace/test/stix_translation/test_darktrace_stix_to_query.py
@@ -650,3 +650,20 @@ def test_combinedcomparison_and_or_operator(self):
         }]
         expected_query = _remove_timestamp_from_query(expected_query)
         self._test_query_assertions(actual_query, expected_query)
+
+    def test_qualifier_without_milliseconds(self):
+        stix_pattern = "[x-oca-asset:hostname = '169.254.169.254'] " \
+                       "START t'2022-03-01T11:50:21Z' STOP t'2022-03-31T11:55:25Z'"
+        actual_query = translation.translate('darktrace', 'query', '{}', stix_pattern)
+        expected_query = [{
+            "search": "(@fields.host:\"169.254.169.254\" AND (@fields.epochdate :>1646135421.0 "
+                      "AND @fields.epochdate :<1648727725.0))",
+            "fields": [],
+            "timeframe": "custom",
+            "time": {
+                "from": "2022-03-01T11:50:21.000000Z",
+                "to": "2022-03-31T11:55:25.000000Z"
+            },
+            "size": 10000
+        }]
+        self._test_query_assertions(actual_query, expected_query)
diff --git a/stix_shifter_modules/paloalto/configuration/lang_en.json b/stix_shifter_modules/paloalto/configuration/lang_en.json
@@ -2,7 +2,7 @@
     "connection": {
         "host": {
             "label": "Management IP address or Hostname",
-            "description": "Specify the IP address or  hostname of the data source so that IBM Cloud Pak for Security can communicate with it"
+            "description": "Specify the IP address or hostname of the data source"
         },
         "quota_threshold": {
             "label": "The quota limit for the API",

diff --git a/stix_shifter_modules/paloalto/stix_translation/query_constructor.py b/stix_shifter_modules/paloalto/stix_translation/query_constructor.py
@@ -249,6 +249,8 @@ def _format_datetime(value):
         """
         try:
             time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ'
+            if re.search(r"\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}Z", str(value)):  # without milli seconds
+                time_pattern = '%Y-%m-%dT%H:%M:%SZ'
             epoch = datetime(1970, 1, 1)
             converted_time = int(((datetime.strptime(value,
                                                      time_pattern) - epoch).total_seconds()) * 1000)

diff --git a/stix_shifter_modules/paloalto/stix_transmission/results_connector.py b/stix_shifter_modules/paloalto/stix_transmission/results_connector.py
@@ -126,6 +126,8 @@ def format_results_data(result_data, to_stix_mapping, mandatory_map):
                     if value is not None and value != "NULL" and value != '' and field != 'dataset_name' \
                             and (field in to_stix_mapping[dataset_map].keys()):
                         stix_data_map = to_stix_mapping[dataset_map][field]
+                        if '\x00' in str(value):
+                            value = value.replace('\x00', '')
                         data = ResultsConnector.check_object(stix_data_map, mandatory_map, data, log,
                                                              field, value)
                     elif field == 'dataset_name':
@@ -160,6 +162,8 @@ def format_stream_data(stream_data, to_stix_mapping, mandatory_map):
                         if value is not None and value != "NULL" and value != '' and field != 'dataset_name' \
                                 and (field in to_stix_mapping[dataset_map].keys()):
                             stix_data_map = to_stix_mapping[dataset_map][field]
+                            if '\x00' in str(value):
+                                value = value.replace('\x00', '')
                             data = ResultsConnector.check_object(stix_data_map, mandatory_map,
                                                                  data, log_dict, field, value)
                         elif field == 'dataset_name':

diff --git a/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py b/stix_shifter_modules/paloalto/tests/stix_translation/test_paloalto_stix_to_query.py
@@ -559,3 +559,17 @@ def test_format_timestamp_fields(self):
                                                                                   "'to': 1645636157746}}}"]
         queries = _remove_timestamp_from_query(queries)
         self._test_query_assertions(query, queries)
+
+    def test_qualifier_without_milliseconds(self):
+        stix_pattern = "[ipv4-addr:value = '10.0.1.4' AND network-traffic:src_port = 52221] " \
+                       "START t'2022-02-01T08:43:10Z' STOP t'2022-04-07T10:43:10Z'"
+        query = translation.translate('paloalto', 'query', '{}', stix_pattern)
+        queries = [{'xdr_data': {'query': 'dataset = xdr_data | filter (action_local_port = 52221 '
+                                          'and (action_local_ip = "10.0.1.4" or action_remote_ip = "10.0.1.4"'
+                                          ' or agent_ip_addresses = "10.0.1.4")  and '
+                                          '(to_epoch(_time,"millis") >= 1643704990000 and '
+                                          'to_epoch(_time,"millis") <= 1649328190000)) | alter dataset_name = '
+                                          '"xdr_data" | fields ' + all_fields + ' | limit 10000 ',
+                                 'timeframe': {'from': 1643704990000, 'to': 1649328190000}}}]
+
+        self._test_query_assertions(query, queries)
diff --git a/stix_shifter_modules/sentinelone/configuration/lang_en.json b/stix_shifter_modules/sentinelone/configuration/lang_en.json
@@ -2,7 +2,7 @@
     "connection": {
         "host": {
             "label": "Management IP address or Hostname",
-            "description": "Specify the IP address or  hostname of the data source so that IBM Cloud Pak for Security can communicate with it"
+            "description": "Specify the IP address or hostname of the data source"
         },
         "port": {
             "label": "Host Port",