diff --git a/OVERVIEW.md b/OVERVIEW.md index 84a2dffcb..d841fc9e3 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -470,11 +470,11 @@ An asynchronous data source will typically return a search ID supplied by the AP ### Status -Uses the data source API to look up the query status based on the `search_id` that is returned from the query call. This is only used for asynchronous data sources where the results are not returned right after making a query call. +Uses the data source API to look up the query status based on the `search_id` that is returned from the query call. This is only used for asynchronous data sources where the results are not returned right after making a query call. If the connector supports, you can specify `metadata` parameter which may contain extra information to make the status api call. #### CLI Command -`stix-shifter transmit '' '' status ` +`stix-shifter transmit '' '' status ` #### OUTPUT: @@ -484,17 +484,53 @@ The status can be one of: `COMPLETED`, `ERROR`, `CANCELLED`, `TIMEOUT`, or `RUNN ### Results -Uses the data source API to fetch the query results based on the search ID, offset, and length. +Uses the data source API to fetch the query results based on the search ID, offset, and length. + +If the connector supports, you can specify `metadata` parameter which may contain extra information to fetch the next batch of results from the datasource. This is a recomended parameter for the datasource that supports pagination. #### CLI Command -`stix-shifter transmit '' '' results ` +`stix-shifter transmit '' '' results ` + +The `OFFSET` and `LENGTH` control what pages/rows of data are returned in the query results. #### OUTPUT: `{'success': True, 'data': []}` -The `OFFSET` and `LENGTH` control what pages/rows of data are returned in the query results. +#### OUTPUT(with metadata): + +`{'success': True, 'data': [], 'metadata': }` + +#### Example: +``` +{ + "success": true, + "data": [ + { + "event": { + "securityEvent": { + "eventTimestamp": "2022-06-13T14:36:54.216539700Z", + "eventType": "FILE_CREATION", + "vendorName": "Microsoft", + "productEventType": "DeviceFileEvents", + "ingestedTimestamp": "2022-06-13T15:36:26.275010Z" + }, + "securityResult": [ + { + "summary": "FileCreated", + "category": "alert" + } + ] + } + } + ], + "metadata": { + "result_count": 2, + "next_page_token": "CgwIlqLjoAYQ2NfggwESCwiGl52VBhC0xKB" + } +} +``` ### Results as STIX @@ -554,6 +590,10 @@ You can redirect the output of your CLI command to a file to save the STIX resul `stix-shifter execute '' '' '' '' > results.json` +### OUTPUT: + +A bundle of STIX objects + ## Modules The `modules` command will return a JSON of the existing connectors along with their dialects and supported languages that are used in query translation. @@ -562,7 +602,7 @@ The `modules` command will return a JSON of the existing connectors along with t `python main.py modules` -returns +#### output ``` { "qradar": { @@ -589,12 +629,204 @@ returns } ``` +This command can also be used to get the dialects of a specific connector. + +`python main.py modules ` + +### CLI Command + +`python main.py modules qradar` + +#### output +``` +{ + "qradar": { + "flows": { + "language": "stix", + "default": true + }, + "events": { + "language": "stix", + "default": true + }, + "aql": { + "language": "aql", + "default": true + } + } +} +``` + In the above example, the QRadar connector can use three dialects: `flows`, `events`, and `aql`. When a connector only has a `default` dialect, such as with Security Advisor, only one dialect is used by the connector. Most dialects will use the `stix` language since they translate STIX patterns into native queries. QRadar's `aql` dialect uses the `aql` language since it is meant to accept an AQL query rather than a STIX pattern. See the [QRadar connector README](stix_shifter_modules/qradar/README.md) for more information on AQL passthrough. -### OUTPUT: +## configs -A bundle of STIX objects +The `configs` command returns the configuration pararmetes of the existing connectors. It basically returns a JSON of the existing connectors along with their connections and configuation objects that are specified in config.json. +### CLI Command + +`python main.py configs` +#### output +``` +{ + "alertflex": { + "connection": { + "type": { + "type": "connectorType", + "displayName": "Alertflex" + }, + "options": { + "type": "fields", + "async_call": { + "type": "text", + "hidden": true, + "optional": true + }, + "result_limit": { + "default": 10000, + "min": 1, + "max": 500000, + "type": "number", + "previous": "connection.resultSizeLimit" + }, + "time_range": { + "default": 5, + "min": 1, + "max": 10000, + "type": "number", + "previous": "connection.timerange", + "nullable": true + }, + ..... + } + } + } +} +``` + +Specifying the connector module name will return the configuration parameters of a specific connector. + +### CLI Command + +`python main.py configs ` +#### output +``` +{ + "qradar": { + "connection": { + "type": { + "type": "connectorType", + "displayName": "IBM\u00ae QRadar and QRadar On Cloud", + "group": "qradar" + }, + "options": { + "type": "fields", + "async_call": { + "type": "text", + "hidden": true, + "optional": true + }, + "result_limit": { + "default": 10000, + "min": 1, + "max": 500000, + "type": "number", + "previous": "connection.resultSizeLimit" + }, + "time_range": { + "default": 5, + "min": 1, + "max": 10000, + "type": "number", + "previous": "connection.timerange", + "nullable": true + }, + "timeout": { + "default": 30, + "min": 1, + "max": 60, + "hidden": true, + "type": "number", + "previous": "connection.timeoutLimit" + }, + "dialects": { + "type": "array", + "hidden": true, + "optional": true + }, + "language": { + "type": "string", + "default": "stix", + "optional": true, + "hidden": true + }, + "validate_pattern": { + "type": "boolean", + "optional": true, + "hidden": true, + "previous": "connection.validate_pattern", + "default": false + }, + "stix_validator": { + "type": "boolean", + "default": false, + "optional": true, + "hidden": true, + "previous": "connection.stix_validator" + }, + "mapping": { + "type": "json", + "optional": true, + "previous": "connection.mapping" + }, + "unmapped_fallback": { + "type": "boolean", + "default": true, + "optional": true, + "hidden": true + }, + "stix_2.1": { + "type": "boolean", + "default": false, + "optional": true, + "hidden": true + } + }, + "host": { + "type": "text", + "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_:/\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_:/\\-]*[A-Za-z0-9])$" + }, + "port": { + "type": "number", + "default": 443, + "min": 1, + "max": 65535 + }, + "help": { + "default": "data-sources-qradar.html", + "type": "link" + }, + "sni": { + "type": "text", + "optional": true + }, + "selfSignedCert": { + "type": "password", + "optional": true + } + }, + "configuration": { + "auth": { + "type": "fields", + "sec": { + "type": "password", + "previous": "configuration.auth.SEC" + } + } + } + } +} +``` ## Limitations STIX-Shifter has limitations on the length of a pattern that can be translated into a native query. As the pattern length increases, the translation time increases exponentially due to how ANTLR 4 parses the pattern. See [STIX-Shifter Limitations](adapter-guide/stix-shifter-limitations.md) for more details. diff --git a/adapter-guide/develop-transmission-module.md b/adapter-guide/develop-transmission-module.md index 665277dde..328f41496 100644 --- a/adapter-guide/develop-transmission-module.md +++ b/adapter-guide/develop-transmission-module.md @@ -62,6 +62,21 @@ Results from the data source need to be returned as an array of JSON objects bef For asynchronous sources, the search id that gets passed into the status, delete, and results methods is the ID returned by the data source when making the query API call. This is used to keep track of the original query, allowing the status and results to be fetched. However, in the case a synchronous data source, the search id is the entire query string; this is what gets passed into the results and delete methods. +### Returning metadata in the return object + +Additional values can be returned as a metadata paremeter in the status and results return object. The data type of the `metadata` parameter can be anything based on the requirements of the connector. The recomended types are python dictionary and string. Ideal use case for this paramater is pagination query. For example, if the connector needs to store the next page token or url and the previous results count to fetch next batch of results from the datasource then the metadata can look like this: + +``` +{ + "result_count": 1, + "next_page_token": "CgwImdHioAYQqKmUuQMSDAiHl52VBhD8g4" +} +``` +Here's an example of the return object of the results with metadata parameter: + +`{'success': True, 'data': [], 'metadata': }` + + [Back to top](#create-a-transmission-module) ## Step 4. Edit the error mapper file @@ -143,7 +158,11 @@ The connection and configuration objects are explained in the transmission secti ``` 2. You can set the offset and length command line arguments to 1. - 3. Visually confirm that query results are returned as JSON objects. These results can be compared to what is returned when running the query string used in test C directly on the data source API, either through a UI or the CLI. + 3. Optionally, you can set the metadata parameter if the connector supports it. Ideally used for pagination query. + ``` + python main.py transmit abc '{"host":"www.example.com", "port":1234}' '{"auth": {"username": "some_user_name", "password": "some_password"}}' results "" '' + ``` + 4. Visually confirm that query results are returned as JSON objects. These results can be compared to what is returned when running the query string used in test C directly on the data source API, either through a UI or the CLI. ### Test the transmission **delete** method. diff --git a/stix_shifter/scripts/stix_shifter.py b/stix_shifter/scripts/stix_shifter.py index 926efd926..d8b749adb 100644 --- a/stix_shifter/scripts/stix_shifter.py +++ b/stix_shifter/scripts/stix_shifter.py @@ -111,21 +111,30 @@ def main(): # operation subparser operation_subparser = transmit_parser.add_subparsers(title="operation", dest="operation_command") operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source") + query_operation_parser = operation_subparser.add_parser(stix_transmission.QUERY, help="Executes a query on the data source") query_operation_parser.add_argument('query_string', help='native datasource query string') + results_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS, help="Fetches the results of the data source query") results_operation_parser.add_argument('search_id', help='uuid of executed query') results_operation_parser.add_argument('offset', help='offset of results') results_operation_parser.add_argument('length', help='length of results') + results_operation_parser.add_argument('metadata', nargs='?', help='metadata to fetch results') + resultsstix_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS_STIX, help="Fetches the results of the data source query, response is translated in STIX") resultsstix_operation_parser.add_argument('search_id', help='uuid of executed query') resultsstix_operation_parser.add_argument('offset', help='offset of results') resultsstix_operation_parser.add_argument('length', help='length of results') resultsstix_operation_parser.add_argument('data_source', help='STIX identity object representing a datasource') + resultsstix_operation_parser.add_argument('metadata', nargs='?', help='metadata to fetch results') + status_operation_parser = operation_subparser.add_parser(stix_transmission.STATUS, help="Gets the current status of the query") status_operation_parser.add_argument('search_id', help='uuid of executed query') + status_operation_parser.add_argument('metadata', nargs='?', help='metadata to fetch results') + delete_operation_parser = operation_subparser.add_parser(stix_transmission.DELETE, help="Delete a running query on the data source") delete_operation_parser.add_argument('search_id', help='id of query to remove') + operation_subparser.add_parser(stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous') execute_parser = parent_subparsers.add_parser(EXECUTE, help='Translate and fully execute a query') @@ -384,29 +393,38 @@ def transmit(args): is_async > """ + log = utils_logger.set_logger(__name__) connection_dict = json.loads(args.connection) configuration_dict = json.loads(args.configuration) transmission = stix_transmission.StixTransmission(args.module, connection_dict, configuration_dict) operation_command = args.operation_command + if 'metadata' in args and args.metadata: + metadata = args.metadata + try: + metadata = json.loads(metadata) + except Exception as ex: + log.debug(exception_to_string(ex)) + log.error('Cannot convert supplied metadata string to json') + pass if operation_command == stix_transmission.QUERY: query = args.query_string result = transmission.query(query) elif operation_command == stix_transmission.STATUS: search_id = args.search_id - result = transmission.status(search_id) + result = transmission.status(search_id, metadata=None) elif operation_command == stix_transmission.RESULTS: search_id = args.search_id offset = args.offset length = args.length - result = transmission.results(search_id, offset, length) + result = transmission.results(search_id, offset, length, metadata=None) elif operation_command == stix_transmission.RESULTS_STIX: search_id = args.search_id offset = args.offset length = args.length data_source = args.data_source - result = transmission.results_stix(search_id, offset, length, data_source) + result = transmission.results_stix(search_id, offset, length, data_source, metadata=None) elif operation_command == stix_transmission.DELETE: search_id = args.search_id result = transmission.delete(search_id)