fixed timestamp issue for start and end filter and mapping correction (…

…#1142)
opencybersecurityalliance · Nov 9, 2022 · 822b6d3 · 822b6d3
1 parent 0072513
commit 822b6d3
Show file tree

Hide file tree

Showing 5 changed files with 73 additions and 57 deletions.
diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/from_stix_map.json
@@ -2,7 +2,7 @@
   "user-account": {
     "fields": {
       "user_id": [
-        "data.user_id"
+        "data.userid"
       ],
       "account_login": [
         "data.username"
@@ -83,9 +83,6 @@
       "extensions.'x-iam-ext'.target": [
         "data.target"
       ],
-      "extensions.'x-iam-ext'.deleted": [
-        "data.deleted"
-      ],
       "extensions.'x-iam-ext'.performedby_clientname": [
         "data.performedby_clientname"
       ],
@@ -113,12 +110,6 @@
       "extensions.'x-iam-ext'.country_name": [
         "geoip.country_name"
       ],
-      "extensions.x-iam-ext.location_lon": [
-        "geoip.lon"
-      ],
-      "extensions.'x-iam-ext'.location_lat": [
-        "geoip.lat"
-      ],
       "extensions.'x-iam-ext'.city_name": [
         "geoip.city_name"
       ],

diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/from_stix_map.json
@@ -2,7 +2,7 @@
   "user-account": {
     "fields": {
       "user_id": [
-        "data.user_id"
+        "data.userid"
       ],
       "account_login": [
         "data.username"
@@ -83,9 +83,6 @@
       "extensions.'x-iam-ext'.target": [
         "data.target"
       ],
-      "extensions.'x-iam-ext'.deleted": [
-        "data.deleted"
-      ],
       "extensions.'x-iam-ext'.performedby_clientname": [
         "data.performedby_clientname"
       ],
@@ -113,12 +110,6 @@
       "extensions.'x-iam-ext'.country_name": [
         "geoip.country_name"
       ],
-      "extensions.x-iam-ext.location_lon": [
-        "geoip.lon"
-      ],
-      "extensions.'x-iam-ext'.location_lat": [
-        "geoip.lat"
-      ],
       "extensions.'x-iam-ext'.city_name": [
         "geoip.city_name"
       ],

diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/stix_2_1/to_stix_map.json
@@ -1,31 +1,35 @@
 {
   "username": [
     {
-      "key": "user-account.user_id",
+      "key": "user-account.account_login",
       "object": "useraccount"
     },
     {
-      "key": "user-account.account_login",
+      "key": "x-oca-event.user_ref",
+      "object": "ocaevent",
+      "references": "useraccount"
+    }
+  ],
+  "userid": [
+    {
+      "key": "user-account.user_id",
       "object": "useraccount"
+    }
+  ],
+  "sourcetype": [
+    {
+      "key": "x-oca-event.agent",
+      "object": "ocaevent"
     },
     {
       "key": "user-account.account_type",
       "object": "useraccount"
-    },
-    {
-      "key": "x-oca-event.user_ref",
-      "object": "ocaevent",
-      "references": "useraccount"
     }
   ],
   "servicename": {
     "key": "x-oca-event.module",
     "object": "ocaevent"
   },
-  "sourcetype": {
-    "key": "x-oca-event.agent",
-    "object": "ocaevent"
-  },
   "ip": [
     {
       "key": "ipv4-addr.value",
@@ -142,12 +146,6 @@
     "key": "x-oca-event.extensions.x-iam-ext.taregetid_username",
     "object": "ocaevent"
   },
-  "userid": [
-    {
-      "key": "user-account.user_id",
-      "object": "useraccount"
-    }
-  ],
   "continent_name": {
     "key": "x-oca-event.extensions.x-iam-ext.continent_name",
     "object": "ocaevent"
@@ -215,5 +213,9 @@
   "lon": {
     "key": "x-oca-event.extensions.x-iam-ext.location_lon",
     "object": "ocaevent"
+  },
+  "add": {
+    "key": "x-oca-event.extensions.x-iam-ext.add",
+    "object": "ocaevent"
   }
 }
diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/json/to_stix_map.json b/stix_shifter_modules/ibm_security_verify/stix_translation/json/to_stix_map.json
@@ -1,31 +1,35 @@
 {
   "username": [
     {
-      "key": "user-account.user_id",
+      "key": "user-account.account_login",
       "object": "useraccount"
     },
     {
-      "key": "user-account.account_login",
+      "key": "x-oca-event.user_ref",
+      "object": "ocaevent",
+      "references": "useraccount"
+    }
+  ],
+  "userid": [
+    {
+      "key": "user-account.user_id",
       "object": "useraccount"
+    }
+  ],
+  "sourcetype": [
+    {
+      "key": "x-oca-event.agent",
+      "object": "ocaevent"
     },
     {
       "key": "user-account.account_type",
       "object": "useraccount"
-    },
-    {
-      "key": "x-oca-event.user_ref",
-      "object": "ocaevent",
-      "references": "useraccount"
     }
   ],
   "servicename": {
     "key": "x-oca-event.module",
     "object": "ocaevent"
   },
-  "sourcetype": {
-    "key": "x-oca-event.agent",
-    "object": "ocaevent"
-  },
   "ip": [
     {
       "key": "ipv4-addr.value",
@@ -112,6 +116,17 @@
       "key": "x-oca-event.created",
       "transformer": "EpochToTimestamp",
       "object": "ocaevent"
+    },
+    {
+
+      "key": "first_observed",
+      "transformer": "EpochToTimestamp",
+      "cybox": false
+    },
+    {
+      "key": "last_observed",
+      "transformer": "EpochToTimestamp",
+      "cybox": false
     }
   ],
   "performedby_username": {
@@ -142,12 +157,6 @@
     "key": "x-oca-event.extensions.x-iam-ext.taregetid_username",
     "object": "ocaevent"
   },
-  "userid": [
-    {
-      "key": "user-account.user_id",
-      "object": "useraccount"
-    }
-  ],
   "continent_name": {
     "key": "x-oca-event.extensions.x-iam-ext.continent_name",
     "object": "ocaevent"
@@ -215,5 +224,9 @@
   "lon": {
     "key": "x-oca-event.extensions.x-iam-ext.location_lon",
     "object": "ocaevent"
+  },
+  "add": {
+    "key": "x-oca-event.extensions.x-iam-ext.add",
+    "object": "ocaevent"
   }
 }
diff --git a/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py b/stix_shifter_modules/ibm_security_verify/stix_translation/query_constructor.py
@@ -1,6 +1,7 @@
 import json
 import logging
 import re
+from datetime import datetime, timedelta
 from typing import Union
 
 from stix_shifter_utils.stix_translation.src.json_to_stix import observable
@@ -140,9 +141,9 @@ def _format_start_stop_qualifier(self, expression, qualifier) -> str:
         start = qualifier_split[1]
         stop = qualifier_split[3]
         # convert timepestamp to millisecond which will be passed to rest service
-        start_epoach = transformer.transform(start)
-        stop_epoach = transformer.transform(stop)
-
+
+        start_epoach = self.get_epoch_time(start)
+        stop_epoach = self.get_epoch_time(stop)
         qualified_query = "%s&from=%s&to=%s" % (
             expression, start_epoach, stop_epoach)
         return qualified_query
@@ -238,6 +239,24 @@ def _parse_expression(self, expression, qualifier=None) -> Union[str, list]:
 
     def parse_expression(self, pattern: Pattern):
         return self._parse_expression(pattern)
+
+    @staticmethod
+    def get_epoch_time(timestamp):
+
+        """
+        Converting timestamp (YYYY-MM-DDThh:mm:ss.000Z) to 13-digit Unix time (epoch + milliseconds)
+        :param timestamp: str, timestamp
+        :return: int, epoch time
+        """
+        time_patterns = ['%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S.%fZ']
+        epoch = datetime(1970, 1, 1)
+        for time_pattern in time_patterns:
+            try:
+                converted_time = int(((datetime.strptime(timestamp, time_pattern) - epoch).total_seconds()) * 1000)
+                return converted_time
+            except ValueError:
+                pass
+        raise NotImplementedError("cannot convert the timestamp {} to milliseconds".format(timestamp))
 
 
 def translate_pattern(pattern: Pattern, data_model_mapping, options):