NGSIEM payload handling fixes. Relates to #1291.

CHANGELOG.md

@@ -7,6 +7,15 @@
 + Fixed: Uber Class path variable interpolation issue related to formatting differences within the endpoint module.
     - `_util/_uber.py`
 
++ Fixed: Payload handling issue with __StartSearchV1__ operation. New payload handler implemented.
+         Keyword argument handling issue using the `id` keyword with __GetSearchStatusV1__ operation.
+         Related to #1291. Thanks go out to @JCKelley-CYBR for reporting this issue. 🙇
+         - `_payload\__init__.py`
+         - `_payload\_ngsiem.py`
+         - `ngsiem.py`
+         > Unit testing expanded to complete code coverage.
+         - `tests/test_ngsiem.py`
+
 ---
 
 # Version 1.4.7

src/falconpy/_payload/__init__.py

@@ -64,6 +64,7 @@
 from ._incidents import incident_action_parameters
 from ._ioa import ioa_exclusion_payload, ioa_custom_payload
 from ._mobile_enrollment import mobile_enrollment_payload
+from ._ngsiem import ngsiem_search_payload
 from ._prevention_policy import prevention_policy_payload
 from ._sensor_update_policy import sensor_policy_payload
 from ._response_policy import response_policy_payload
@@ -150,5 +151,5 @@
     "idp_policy_payload", "delivery_settings_payload", "base_image_payload", "aspm_delete_tag_payload",
     "aspm_update_tag_payload", "aspm_violations_search_payload", "aspm_get_services_count_payload",
     "aspm_query_payload", "aspm_integration_payload", "aspm_integration_task_payload", "aspm_node_payload",
-    "aspm_application_payload", "correlation_rules_payload"
+    "aspm_application_payload", "correlation_rules_payload", "ngsiem_search_payload"
 ]

src/falconpy/_payload/_ngsiem.py

@@ -0,0 +1,91 @@
+"""Internal payload handling library - NGSIEM.
+
+ _______                        __ _______ __        __ __
+|   _   .----.-----.--.--.--.--|  |   _   |  |_.----|__|  |--.-----.
+|.  1___|   _|  _  |  |  |  |  _  |   1___|   _|   _|  |    <|  -__|
+|.  |___|__| |_____|________|_____|____   |____|__| |__|__|__|_____|
+|:  1   |                         |:  1   |
+|::.. . |   CROWDSTRIKE FALCON    |::.. . |    FalconPy
+`-------'                         `-------'
+
+OAuth2 API - Customer SDK
+
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>
+"""
+
+
+def ngsiem_search_payload(passed_keywords: dict) -> dict:
+    """Generate a properly formatted NGSIEM search payload.
+
+    {
+        "allowEventSkipping": boolean,
+        "arguments": {},
+        "around": {
+            "eventId": "string",
+            "numberOfEventsAfter": integer,
+            "numberOfEventsBefore": integer,
+            "timestamp": integer
+        },
+        "autobucketCount": integer,
+        "end": "string",
+        "ingestEnd": "string",
+        "ingestStart": "string",
+        "isLive": boolean,
+        "queryString": "string",
+        "start": "string",
+        "timeZone": "string",
+        "timeZoneOffsetMinutes": integer,
+        "useIngestTime": boolean
+    }
+    """
+    key_map = {
+        "allow_event_skipping": "allowEventSkipping",
+        "autobucket_count": "autobucketCount",
+        "ingest_end": "ingestEnd",
+        "ingest_start": "ingestStart",
+        "is_live": "isLive",
+        "query_string": "queryString",
+        "timezone": "timeZone",
+        "timezone_offset_minutes": "timeZoneOffsetMinutes",
+        "use_ingest_time": "useIngestTime"
+    }
+    returned: dict = {}
+    bool_int_keys = ["allow_event_skipping", "is_live", "use_ingest_time", "autobucket_count",
+                     "timezone_offset_minutes"
+                     ]
+    keys = ["arguments", "around", "end", "ingest_end", "ingest_start", "query_string", "start"
+            "timezone"
+            ]
+    for key in keys:
+        if passed_keywords.get(key, None):
+            keystr = key_map[key] if key in key_map else key
+            returned[keystr] = passed_keywords.get(key, None)
+    for key in bool_int_keys:
+        if passed_keywords.get(key, None) is not None:
+            keystr = key_map[key] if key in key_map else key
+            returned[keystr] = passed_keywords.get(key, None)
+
+    return returned

src/falconpy/ngsiem.py

@@ -38,6 +38,7 @@
 
 from typing import Dict, Union
 from ._util import force_default, process_service_request, generate_error_result
+from ._payload import ngsiem_search_payload
 from ._result import Result
 from ._service_class import ServiceClass
 from ._endpoint._ngsiem import _ngsiem_endpoints as Endpoints
@@ -269,16 +270,72 @@ def get_file_from_package(self: object,
     #         params=parameters
     #         )
 
-    @force_default(defaults=["parameters"], default_types=["dict"])
+    @force_default(defaults=["parameters", "body"], default_types=["dict", "dict"])
     def start_search(self: object,
+                     body: dict = None,
                      parameters: dict = None,
                      **kwargs) -> Union[Dict[str, Union[int, dict]], Result]:
         """Initiate search.
 
         Keyword arguments:
-        repository -- Name of repository. String.
-        search -- Search to perform. JSON formatted string.
-        parameters -- Full parameters payload dictionary. Not required if using other keywords.
+        allow_event_skipping -- Flag indicating if event skipping is allowed. Boolean.
+        arguments -- Search arguments in JSON format. Dictionary.
+        around -- Search proximity arguments. Dictionary.
+        autobucket_count -- Number of events per bucket. Integer.
+        body -- Full body payload as a JSON dictionary.
+                Not required if using the search argument or other keywords.
+                {
+                    "allowEventSkipping": boolean,
+                    "arguments": {},
+                    "around": {
+                        "eventId": "string",
+                        "numberOfEventsAfter": integer,
+                        "numberOfEventsBefore": integer,
+                        "timestamp": integer
+                    },
+                    "autobucketCount": integer,
+                    "end": "string",
+                    "ingestEnd": "string",
+                    "ingestStart": "string",
+                    "isLive": boolean,
+                    "queryString": "string",
+                    "start": "string",
+                    "timeZone": "string",
+                    "timeZoneOffsetMinutes": integer,
+                    "useIngestTime": boolean
+                }
+        end -- Last event limit. String.
+        ingest_end -- Ingest maximum. Integer.
+        ingest_start -- Ingest start. Integer.
+        is_live -- Flag indicating if this is a live search. Boolean.
+        parameters -- Full parameters payload dictionary. Not required if using repository keyword.
+        query_string -- Search query string. String.
+        repository -- Name of repository. Required. String.
+        search -- Search to perform. JSON formatted string. Can be used instead of body.
+                  Not required if using other keywords.
+                  {
+                    "allowEventSkipping": boolean,
+                    "arguments": {},
+                    "around": {
+                        "eventId": "string",
+                        "numberOfEventsAfter": integer,
+                        "numberOfEventsBefore": integer,
+                        "timestamp": integer
+                    },
+                    "autobucketCount": integer,
+                    "end": "string",
+                    "ingestEnd": "string",
+                    "ingestStart": "string",
+                    "isLive": boolean,
+                    "queryString": "string",
+                    "start": "string",
+                    "timeZone": "string",
+                    "timeZoneOffsetMinutes": integer,
+                    "useIngestTime": boolean
+                  }
+        start -- Search starting time range. Start.
+        timezone -- Timezone applied to the search. String.
+        timezone_offset_minutes -- Timezone offset. Integer.
 
         This method only supports keywords for providing arguments.
 
@@ -291,6 +348,10 @@ def start_search(self: object,
         """
         repository = kwargs.get("repository", None)
         search = kwargs.get("search", None)
+
+        if not body and not search:
+            search = ngsiem_search_payload(kwargs)
+
         if repository and search:
             # Pop the path variables from the keywords dictionary
             # before processing query string arguments.
@@ -309,7 +370,7 @@ def start_search(self: object,
                 returned.pop("body")
         else:
             returned = generate_error_result("You must provide a repository and search "
-                                             "argument in order to use this operation."
+                                             "arguments in order to use this operation."
                                              )
         return returned
 
@@ -321,7 +382,7 @@ def get_search_status(self: object,
 
         Keyword arguments:
         repository -- Name of repository. String.
-        search_id -- ID of query. String.
+        id -- ID of query. String.
         parameters -- Full parameters payload dictionary. Not required if using other keywords.
 
         This method only supports keywords for providing arguments.
@@ -334,12 +395,12 @@ def get_search_status(self: object,
         https://assets.falcon.crowdstrike.com/support/api/swagger.html#/humio-auth-proxy/GetSearchStatusV1
         """
         repository = kwargs.get("repository", None)
-        search_id = kwargs.get("search_id", None)
+        search_id = kwargs.get("id", None)
         if repository and search_id:
             # Pop the path variables from the keywords dictionary
             # before processing query string arguments.
             kwargs.pop("repository")
-            kwargs.pop("search_id")
+            kwargs.pop("id")
             returned = process_service_request(
                 calling_object=self,
                 endpoints=Endpoints,
@@ -350,7 +411,7 @@ def get_search_status(self: object,
                 search_id=search_id
                 )
         else:
-            returned = generate_error_result("You must provide a repository and search_id "
+            returned = generate_error_result("You must provide a repository and id "
                                              "argument in order to use this operation."
                                              )
 

tests/test_ngsiem.py

@@ -21,7 +21,7 @@
 class TestNGSIEM:
     def run_all_tests(self):
         test_search = {
-            "showQueryEventDistribution" : True,
+            #"showQueryEventDistribution" : True,
             "isLive" : False,
             "start" : "1d",
             "queryString" : "#event_simpleName=*"
@@ -38,40 +38,41 @@ def run_all_tests(self):
                                                                                                ),
             "GetLookupFromPackageWithNamespaceV1Fail": falcon.get_file_from_package_with_namespace(filename="manny",
                                                                                                    package="moe",
-                                                                                                   namespace="jack"
+                                                                                                   namespace="jack",
+                                                                                                   repository="search-all"
                                                                                                    ),
             "GetLookupFromPackageV1": falcon.get_file_from_package(repository="search-all",
                                                                    filename="manny",
                                                                    package="moe"
                                                                    ),
-            "StartSearchV1": falcon.start_search(repository="search-all", search=test_search),
+            "StartSearchV1": falcon.start_search(repository="search-all", search=test_search, is_live=False, start="1d", timezone_offset_minutes=0),
+            "StartSearchV1B": falcon.start_search(repository="search-all", query_string="#event_simpleName=*", is_live=False, start="1d", timezone_offset_minutes=0),
         }
         for key in tests:
             if tests[key]["status_code"] not in AllowedResponses:
                 error_checks = False
-                # if not error_checks:
-                #     print(tests[key])
             else:
                 if key == "StartSearchV1":
                     search_id = tests[key]["resources"].get("id", None)
 
         follow_up_tests = {
-            "GetSearchStatusV1": falcon.get_search_status(repository="search-all", search_id=search_id),
+            "GetSearchStatusV1": falcon.get_search_status(repository="search-all", id=search_id),
             "StopSearchV1": falcon.stop_search(repository="search-all", search_id=search_id)
         }
         for follow_key in follow_up_tests:
             if follow_up_tests[follow_key]["status_code"] not in AllowedResponses:
                 error_checks = False
 
         # Test lookup file download
-        try:
-            binary_download_test = falcon.get_file(repository="search-all", filename="testfile.csv", expand_result=True)[0]
-        except Exception:
-            pytest.skip("Skipping on failure")
+        # try:
+        # Expanding result so we can retrieve the status code from a binary response
+        binary_download_test = falcon.get_file(repository="search-all", filename="testfile.csv", expand_result=True)[0]
+        # except Exception:
+        #     pytest.skip("Skipping on failure")
         if binary_download_test not in AllowedResponses:
             error_checks = False
-        if not error_checks:
-            pytest.skip("Skipping on failure")  # Skip on failure for now
+        # if not error_checks:
+        #     pytest.skip("Skipping on failure")  # Skip on failure for now
         return error_checks
 
     def test_all_functionality(self):