From ab2e5ef6d553c36f757e095e3707814b1fc632f7 Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh <92723742+thangaraj-ramesh@users.noreply.github.com> Date: Mon, 5 Dec 2022 13:43:19 +0000 Subject: [PATCH] Updated cybereason code to fix the issue #1215 (#1224) 1. Changes done to treat both AND as OR operator between multiple observations. 2. Updated code to display alternate error message "Cybereason does not support AND operator between fields" when unsupported Cybereason fields are attempted to be combined using AND operator. Added relevant unit test cases. --- stix_shifter_modules/cybereason/README.md | 366 +++++++++++--- .../stix_translation/json/operators.json | 5 +- .../stix_translation/query_constructor.py | 42 +- .../test_cybereason_stix_to_query.py | 462 ++++++++++-------- 4 files changed, 599 insertions(+), 276 deletions(-) diff --git a/stix_shifter_modules/cybereason/README.md b/stix_shifter_modules/cybereason/README.md index e2703b068..a32c76a9a 100644 --- a/stix_shifter_modules/cybereason/README.md +++ b/stix_shifter_modules/cybereason/README.md @@ -375,7 +375,7 @@ translate cybereason results ####STIX Translate query ```shell -translate cybereason query '{}' "([x-cybereason-process:integrity != 'trusted'] AND [x-cybereason-file:product_type IN ('Adobe')] AND [process:command_line LIKE 'Adobe\\Acrobat Reader DC']) START t'2021-02-10T11:43:08.000Z' STOP t'2021-11-12T11:00:00.003Z'" +translate cybereason query '{}' "([x-cybereason-process:integrity != 'trusted'] OR [x-cybereason-file:product_type IN ('Adobe')] AND [process:command_line LIKE 'Adobe\\Acrobat Reader DC']) START t'2021-02-10T11:43:08.000Z' STOP t'2021-11-12T11:00:00.003Z'" ``` #### STIX Translate query - output @@ -404,11 +404,222 @@ translate cybereason query '{}' "([x-cybereason-process:integrity != 'trusted'] ] } ], - "connectionFeature": { - "elementInstanceType": "Process", - "featureName": "imageFile" - } - }, + "isResult": true + } + ], + "queryLimits": { + "groupingFeature": { + "elementInstanceType": "Process", + "featureName": "elementDisplayName" + } + }, + "perFeatureLimit": 1, + "totalResultLimit": 9999, + "perGroupLimit": 1, + "templateContext": "CUSTOM", + "customFields": [ + "elementDisplayName", + "creationTime", + "endTime", + "commandLine", + "imageFile.maliciousClassificationType", + "productType", + "children", + "parentProcess", + "ownerMachine", + "calculatedUser", + "imageFile", + "imageFile.sha1String", + "imageFile.md5String", + "imageFile.sha256String", + "imageFile.companyName", + "imageFile.productName", + "applicablePid", + "imageFileExtensionType", + "integrity", + "tid", + "isAggregate", + "isDotNetProtected", + "hasMalops", + "hasSuspicions", + "relatedToMalop", + "multipleSizeForHashEvidence", + "isImageFileVerified", + "knownMaliciousToolSuspicion", + "knownMalwareSuspicion", + "knownUnwantedSuspicion", + "isMaliciousByHashEvidence", + "imageFileMultipleCompanyNamesEvidence", + "multipleHashForUnsignedPeInfoEvidence", + "multipleNameForHashEvidence", + "unknownEvidence", + "rareHasPeMismatchEvidence", + "imageFile.signedInternalOrExternal", + "unknownUnsignedBySigningCompany", + "imageFileUnsignedEvidence", + "imageFileUnsignedHasSignedVersionEvidence", + "unwantedModuleSuspicion", + "imageFile.signerInternalOrExternal", + "architecture", + "commandLineContainsTempEvidence", + "hasChildren", + "hasClassification", + "hasVisibleWindows", + "hasWindows", + "isInstaller", + "isIdentifiedProduct", + "hasModuleFromTempEvidence", + "nonExecutableExtensionEvidence", + "isNotShellRunner", + "runningFromTempEvidence", + "shellOfNonShellRunnerSuspicion", + "shellWithElevatedPrivilegesEvidence", + "systemUserEvidence", + "hasExternalConnection", + "hasExternalConnectionToWellKnownPortEvidence", + "hasIncomingConnection", + "hasInternalConnection", + "hasMailConnectionForNonMailProcessEvidence", + "hasListeningConnection", + "hasOutgoingConnection", + "hasUnresolvedDnsQueriesFromDomain", + "multipleUnresolvedRecordNotExistsEvidence", + "hasNonDefaultResolverEvidence", + "parentProcessNotMatchHierarchySuspicion", + "parentProcessNotAdminUserEvidence", + "parentProcessFromRemovableDeviceEvidence", + "autorun", + "childrenCreatedByThread", + "connections", + "elevatedPrivilegeChildren", + "hackerToolChildren", + "hostProcess", + "hostUser", + "hostedChildren", + "injectedChildren", + "loadedModules", + "logonSession", + "remoteSession", + "service", + "execedBy", + "connectionsToMaliciousDomain", + "connectionsToMalwareAddresses", + "externalConnections", + "absoluteHighVolumeMaliciousAddressConnections", + "absoluteHighVolumeExternalConnections", + "incomingConnections", + "incomingExternalConnections", + "incomingInternalConnections", + "internalConnections", + "listeningConnections", + "localConnections", + "mailConnections", + "outgoingConnections", + "outgoingExternalConnections", + "outgoingInternalConnections", + "suspiciousExternalConnections", + "suspiciousInternalConnections", + "wellKnownPortConnections", + "lowTtlDnsQueries", + "nonDefaultResolverQueries", + "resolvedDnsQueriesDomainToDomain", + "resolvedDnsQueriesDomainToIp", + "resolvedDnsQueriesIpToDomain", + "suspiciousDnsQueryDomainToDomain", + "unresolvedQueryFromSuspiciousDomain", + "dnsQueryFromSuspiciousDomain", + "dnsQueryToSuspiciousDomain", + "unresolvedRecordNotExist", + "unresolvedDnsQueriesFromDomain", + "unresolvedDnsQueriesFromIp", + "maliciousToolClassificationModules", + "malwareClassificationModules", + "modulesNotInLoaderDbList", + "modulesFromTemp", + "unsignedWithSignedVersionModules", + "unwantedClassificationModules", + "accessToMalwareAddressInfectedProcess", + "connectingToBadReputationAddressSuspicion", + "hasMaliciousConnectionEvidence", + "hasSuspiciousExternalConnectionSuspicion", + "highNumberOfExternalConnectionsSuspicion", + "nonDefaultResolverSuspicion", + "hasRareExternalConnectionEvidence", + "hasRareRemoteAddressEvidence", + "suspiciousMailConnections", + "accessToMalwareAddressByUnknownProcess", + "hasAbsoluteHighVolumeConnectionToMaliciousAddressEvidence", + "hasAbsoluteHighVolumeExternalOutgoingConnectionEvidence", + "highDataTransmittedSuspicion", + "highDataVolumeTransmittedToMaliciousAddressSuspicion", + "highDataVolumeTransmittedByUnknownProcess", + "absoluteHighNumberOfInternalOutgoingEmbryonicConnectionsEvidence", + "dgaSuspicion", + "hasLowTtlDnsQueryEvidence", + "highUnresolvedToResolvedRateEvidence", + "manyUnresolvedRecordNotExistsEvidence", + "hasChildKnownHackerToolEvidence", + "hackingToolOfNonToolRunnerEvidence", + "hackingToolOfNonToolRunnerSuspicion", + "hasRareChildProcessKnownHackerToolEvidence", + "maliciousToolModuleSuspicion", + "deletedParentProcessEvidence", + "malwareModuleSuspicion", + "dualExtensionNameEvidence", + "hiddenFileExtensionEvidence", + "rightToLeftFileExtensionEvidence", + "screenSaverWithChildrenEvidence", + "suspicionsScreenSaverEvidence", + "hasPeFloatingCodeEvidence", + "hasSectionMismatchEvidence", + "detectedInjectedEvidence", + "detectedInjectingEvidence", + "detectedInjectingToProtectedProcessEvidence", + "hasInjectedChildren", + "hostingInjectedThreadEvidence", + "injectedProtectedProcessEvidence", + "maliciousInjectingCodeSuspicion", + "injectionMethod", + "isHostingInjectedThread", + "maliciousInjectedCodeSuspicion", + "maliciousPeExecutionSuspicion", + "hasSuspiciousInternalConnectionEvidence", + "highInternalOutgoingEmbryonicConnectionRateEvidence", + "highNumberOfInternalConnectionsEvidence", + "newProcessesAboveThresholdEvidence", + "hasRareInternalConnectionEvidence", + "elevatingPrivilegesToChildEvidence", + "parentProcessNotSystemUserEvidence", + "privilegeEscalationEvidence", + "firstExecutionOfDownloadedProcessEvidence", + "hasAutorun", + "newProcessEvidence", + "markedForPrevention", + "ransomwareAutoRemediationSuspended", + "totalNumOfInstances", + "lastMinuteNumOfInstances", + "lastSeenTimeStamp", + "wmiQueryStrings", + "isExectuedByWmi", + "absoluteHighNumberOfInternalConnectionsEvidence", + "scanningProcessSuspicion", + "imageFile.isDownloadedFromInternet", + "imageFile.downloadedFromDomain", + "imageFile.downloadedFromIpAddress", + "imageFile.downloadedFromUrl", + "imageFile.downloadedFromUrlReferrer", + "imageFile.downloadedFromEmailFrom", + "imageFile.downloadedFromEmailMessageId", + "imageFile.downloadedFromEmailSubject", + "rpcRequests", + "iconBase64", + "executionPrevented", + "isWhiteListClassification", + "matchedWhiteListRuleIds" + ] + }, + { + "queryPath": [ { "requestedType": "File", "filters": [ @@ -428,12 +639,84 @@ translate cybereason query '{}' "([x-cybereason-process:integrity != 'trusted'] ] } ], - "connectionFeature": { - "elementInstanceType": "Process", - "featureName": "imageFile" - }, - "isReversed": true - }, + "isResult": true + } + ], + "queryLimits": { + "groupingFeature": { + "elementInstanceType": "File", + "featureName": "elementDisplayName" + } + }, + "perFeatureLimit": 1, + "totalResultLimit": 9999, + "perGroupLimit": 1, + "templateContext": "CUSTOM", + "customFields": [ + "elementDisplayName", + "avRemediationStatus", + "signerInternalOrExternal", + "fileHash", + "autoruns", + "ownerMachine", + "mount", + "autorun", + "dualExtensionEvidence", + "hiddenFileExtensionEvidence", + "rightToLeftFileExtensionEvidence", + "hasMalops", + "hasSuspicions", + "maliciousClassificationType", + "hackingToolClassificationEvidence", + "classificationLink", + "isPEFile", + "executedByProcessEvidence", + "hasAutorun", + "isInstallerProperties", + "isFromRemovableDevice", + "productType", + "secondExtensionType", + "temporaryFolderEvidence", + "multipleCompanyNamesEvidence", + "multipleHashForUnsignedPeInfoEvidence", + "unsignedHasSignedVersionEvidence", + "classificationComment", + "signedInternalOrExternal", + "signatureVerifiedInternalOrExternal", + "classificationBlocking", + "isDownloadedFromInternet", + "downloadedFromDomain", + "downloadedFromIpAddress", + "downloadedFromUrl", + "downloadedFromUrlReferrer", + "downloadedFromEmailFrom", + "downloadedFromEmailMessageId", + "downloadedFromEmailSubject", + "legalCopyright", + "legalTrademarks", + "privateBuild", + "specialBuild", + "companyName", + "createdTime", + "extensionType", + "fileDescription", + "internalName", + "md5String", + "modifiedTime", + "originalFileName", + "correctedPath", + "productName", + "productVersion", + "sha1String", + "size", + "comments", + "fileVersion", + "applicationIdentifier", + "sha256String" + ] + }, + { + "queryPath": [ { "requestedType": "Process", "filters": [ @@ -669,55 +952,6 @@ translate cybereason query '{}' "([x-cybereason-process:integrity != 'trusted'] }, { "queryPath": [ - { - "requestedType": "Process", - "filters": [ - { - "facetName": "integrity", - "filterType": "NotEquals", - "values": [ - "trusted" - ] - }, - { - "facetName": "creationTime", - "filterType": "Between", - "values": [ - 1612957388000, - 1636714800003 - ] - } - ], - "connectionFeature": { - "elementInstanceType": "Process", - "featureName": "imageFile" - } - }, - { - "requestedType": "File", - "filters": [ - { - "facetName": "productType", - "filterType": "Equals", - "values": [ - "Adobe" - ] - }, - { - "facetName": "createdTime", - "filterType": "Between", - "values": [ - 1612957388000, - 1636714800003 - ] - } - ], - "connectionFeature": { - "elementInstanceType": "Process", - "featureName": "imageFile" - }, - "isReversed": true - }, { "requestedType": "Process", "filters": [ @@ -953,6 +1187,7 @@ translate cybereason query '{}' "([x-cybereason-process:integrity != 'trusted'] } ] } + ``` @@ -1312,9 +1547,10 @@ ping ``` ### Limitations -- Cybereason does not support “OR” operator between the elements and features. It supports only "AND" operator through "connectionFeature" and "Filters". - -### Observations -- Cybereason doesnt support regex based search. It supports only substring based search . Hence wildcard characters cannot be used for searches using LIKE or MATCHES operator +- Cybereason does not support “OR” operator between Combined Comparison. It supports only "AND" operator. +### Observations +- Cybereason doesnt support regex based search. It supports only substring based search . Hence wildcard characters cannot be used for searches using LIKE or MATCHES operator. +- AND operator between stix fields , can be performed only when there is a link (the relationship with the next field in the chain) available between two fields. If “AND” is given for fields which have no link, connector will throw error. + All the allowed links between different fields is given in config_map.json file in stix_translation\json folder. diff --git a/stix_shifter_modules/cybereason/stix_translation/json/operators.json b/stix_shifter_modules/cybereason/stix_translation/json/operators.json index b3a384ebd..f9dc5f91a 100644 --- a/stix_shifter_modules/cybereason/stix_translation/json/operators.json +++ b/stix_shifter_modules/cybereason/stix_translation/json/operators.json @@ -9,5 +9,6 @@ "ComparisonComparators.Like": "ContainsIgnoreCase", "ComparisonComparators.In": "Equals", "ComparisonComparators.Matches": "ContainsIgnoreCase", - "ObservationOperators.And": "AND" -} + "ObservationOperators.And": "OR", + "ObservationOperators.Or": "OR" +} \ No newline at end of file diff --git a/stix_shifter_modules/cybereason/stix_translation/query_constructor.py b/stix_shifter_modules/cybereason/stix_translation/query_constructor.py index f4e4ec33b..9d563b697 100644 --- a/stix_shifter_modules/cybereason/stix_translation/query_constructor.py +++ b/stix_shifter_modules/cybereason/stix_translation/query_constructor.py @@ -36,6 +36,9 @@ def __init__(self, pattern: Pattern, data_model_mapper, options): self.options = options self.config_map = self.load_json(CONFIG_MAP_PATH) self.qualified_queries = [] + self.stix_object_fields = [] + self.link_not_found_messages = [] + self.link_not_found_flag = False self.parse_expression(pattern) @staticmethod @@ -231,9 +234,9 @@ def _formatted_query(add_query_path, add_custom_fields, options): } }, "perFeatureLimit": 1, - "totalResultLimit": options["result_limit"] - 1, # Cybereason returns 1 more than totalResultLimit records along with - "perGroupLimit": 1, # the options perfeaturelimit=1 and perGroupLimit=1 - "templateContext": "CUSTOM", + "totalResultLimit": options["result_limit"] - 1, # Cybereason returns 1 more than totalResultLimit records + "perGroupLimit": 1, # along with the options perfeaturelimit=1 + "templateContext": "CUSTOM", # and perGroupLimit=1 "customFields": add_custom_fields } @@ -351,6 +354,8 @@ def _and_operator_query(self, previous_all_queries, current_all_queries): :return : list """ merged_query = [] + self.link_not_found_messages[-1].append([]) + for previous_queries in previous_all_queries: for current_queries in current_all_queries: current_query = copy.deepcopy(current_queries) @@ -364,8 +369,14 @@ def _and_operator_query(self, previous_all_queries, current_all_queries): elif current_requested_type in self.config_map["linked_fields"][previous_requested_type].keys(): self._merge_linked_element(previous_query, current_query, merged_query) + else: + link_error_msg = f"{self.stix_object_fields[-2]} and {self.stix_object_fields[-1]}" + self.link_not_found_messages[-1][-1].append(link_error_msg) + # set the flag to true if no link is found between elements if not merged_query: - raise LinkNotFoundException('Link is not found between elements') + self.link_not_found_flag = True + else: + self.link_not_found_messages[-1][-1] = [] return merged_query def _parse_mapped_fields(self, comparator, value, mapped_fields_array, qualifier): @@ -390,13 +401,16 @@ def __eval_observation_expression(self, expression, qualifier): :param expression: expression object :param qualifier: qualifier """ + self.link_not_found_flag = False self.qualified_queries.append([]) + self.link_not_found_messages.append([]) self._parse_expression(expression.comparison_expression, qualifier) - if len(self.qualified_queries) > 1: - current_query = self.qualified_queries.pop() - previous_query = self.qualified_queries.pop() - merged_query = self._and_operator_query(previous_query, current_query) - self.qualified_queries.append(merged_query) + self.link_not_found_messages[-1] = {item for sublist in self.link_not_found_messages[-1] for item in sublist} + # queries should not be added when a link is not found between elements in an observation + if self.link_not_found_flag: + self.qualified_queries[-1] = [] + elif self.qualified_queries[-1]: + self.link_not_found_messages[-1] = [] def _parse_expression(self, expression, qualifier=None): """ @@ -406,6 +420,7 @@ def _parse_expression(self, expression, qualifier=None): :return :None or str """ if isinstance(expression, ComparisonExpression): # Base Case + self.stix_object_fields.append(expression.object_path) stix_object, stix_field = expression.object_path.split(':') mapped_fields_array = self.dmm.map_field(stix_object, stix_field) comparator = self.comparator_lookup[str(expression.comparator)] @@ -437,7 +452,7 @@ def _parse_expression(self, expression, qualifier=None): f'type(expression)={type(expression)}') def parse_expression(self, pattern: Pattern): - if "ComparisonExpressionOperators.Or" in str(pattern) or "ObservationOperators.Or" in str(pattern): + if "ComparisonExpressionOperators.Or" in str(pattern): raise NotImplementedError("OR operator is not supported in Cybereason") self._parse_expression(pattern) @@ -453,5 +468,12 @@ def translate_pattern(pattern: Pattern, data_model_mapping, options): """ translated_query_strings = QueryStringPatternTranslator(pattern, data_model_mapping, options) queries = translated_query_strings.qualified_queries + link_not_found_message = translated_query_strings.link_not_found_messages final_queries = [item for sublist in queries for item in sublist] + link_not_found_messages = {item for sublist in link_not_found_message for item in sublist} + if not final_queries: + raise LinkNotFoundException(f"Cybereason does not allow AND operation " + f"between {', '.join(link_not_found_messages)}") + if link_not_found_messages: + logger.error("Cybereason does not allow AND operation between %s", ', '.join(link_not_found_messages)) return final_queries diff --git a/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py b/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py index fda68241e..029ac34df 100644 --- a/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py +++ b/stix_shifter_modules/cybereason/test/stix_translation/test_cybereason_stix_to_query.py @@ -795,7 +795,6 @@ def test_process_created_query(self): "'process.calculatedName', 'process.calculatedUser', 'process.creationTime', 'process.endTime', " "'process.imageFile.maliciousClassificationType']}"] - queries = _remove_timestamp_from_query(queries) self._test_query_assertions(query, queries) @@ -1162,42 +1161,54 @@ def test_multiple_observation_query(self): "t'2019-10-01T00:00:00.030Z' STOP t'2021-10-07T00:00:00.030Z' " query = translation.translate('cybereason', 'query', '{}', stix_pattern) - query['queries'] = _remove_timestamp_from_query(query['queries']) queries = ["{'queryPath': [{'requestedType': 'RegistryEvent', 'filters': [{'facetName': " "'detectionTimesNumber', 'filterType': 'Equals', 'values': [1]}, {'facetName': 'firstTime', " "'filterType': 'Between', 'values': [1569888000030, 1633564800030]}], 'connectionFeature': {" "'elementInstanceType': 'RegistryEvent', 'featureName': 'registryEntry'}}, {'requestedType': " "'Autorun', 'filters': [{'facetName': 'elementDisplayName', 'filterType': 'Equals', 'values': [" - "'name']}, {'facetName': 'endTime', " - "'filterType': 'Between', 'values': [1569888000030, 1633564800030]}], " - "'connectionFeature': {'elementInstanceType': 'Autorun', 'featureName': " + "'name']}, {'facetName': 'endTime', 'filterType': 'Between', 'values': [1569888000030, " + "1633564800030]}], 'connectionFeature': {'elementInstanceType': 'Autorun', 'featureName': " "'dependInFile'}}, {'requestedType': 'File', 'filters': [{'facetName': 'md5String', 'filterType': " - "'Equals', 'values': ['MD5']}, {'facetName': 'createdTime', " - "'filterType': 'Between', 'values': [1569888000030, 1633564800030]}], " - "'connectionFeature': {'elementInstanceType': 'Driver', " - "'featureName': 'file'}, 'isReversed': True}, {'requestedType': 'Driver', 'filters': [{" - "'facetName': 'ownerMachine', 'filterType': 'Equals', 'values': ['username']}, {'facetName': " - "'endTime', 'filterType': 'Between', 'values': [1569888000030, 1633564800030]}], " - "'connectionFeature': {'elementInstanceType': 'Driver', 'featureName': 'ownerMachine'}}, " - "{'requestedType': 'Machine', 'filters': [{'facetName': 'timezoneUTCOffsetMinutes', 'filterType': " - "'Equals', 'values': [4]}, {'facetName': 'lastSeenTimeStamp', " - "'filterType': 'Between', 'values': [1569888000030, 1633564800030]}], 'isResult': True}]," - " 'queryLimits': {'groupingFeature': {" - "'elementInstanceType': 'Machine', 'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, " + "'Equals', 'values': ['MD5']}, {'facetName': 'createdTime', 'filterType': 'Between', 'values': [" + "1569888000030, 1633564800030]}], 'isResult': True}], 'queryLimits': {'groupingFeature': {" + "'elementInstanceType': 'File', 'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, " "'totalResultLimit': 9999, 'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': [" - "'elementDisplayName', 'mountPoints', 'processes', 'services', 'logonSessions', " - "'hasRemovableDevice', 'timezoneUTCOffsetMinutes', 'osVersionType', 'platformArchitecture', " - "'mbrHashString', 'osType', 'domainFqdn', 'ownerOrganization', 'pylumId', 'adSid', 'adOU', " - "'adOrganization', 'adCanonicalName', 'adCompany', 'adDNSHostName', 'adDepartment', " - "'adDisplayName', 'adLocation', 'adMachineRole', 'adDescription', 'freeDiskSpace', " - "'totalDiskSpace', 'freeMemory', 'totalMemory', 'cpuCount', 'isLaptop', 'deviceModel', " - "'isActiveProbeConnected', 'uptime', 'isIsolated', 'lastSeenTimeStamp', " - "'timeStampSinceLastConnectionTime', 'hasMalops', 'hasSuspicions', " + "'elementDisplayName', 'avRemediationStatus', 'signerInternalOrExternal', 'fileHash', 'autoruns', " + "'ownerMachine', 'mount', 'autorun', 'dualExtensionEvidence', 'hiddenFileExtensionEvidence', " + "'rightToLeftFileExtensionEvidence', 'hasMalops', 'hasSuspicions', 'maliciousClassificationType', " + "'hackingToolClassificationEvidence', 'classificationLink', 'isPEFile', " + "'executedByProcessEvidence', 'hasAutorun', 'isInstallerProperties', 'isFromRemovableDevice', " + "'productType', 'secondExtensionType', 'temporaryFolderEvidence', 'multipleCompanyNamesEvidence', " + "'multipleHashForUnsignedPeInfoEvidence', 'unsignedHasSignedVersionEvidence', " + "'classificationComment', 'signedInternalOrExternal', 'signatureVerifiedInternalOrExternal', " + "'classificationBlocking', 'isDownloadedFromInternet', 'downloadedFromDomain', " + "'downloadedFromIpAddress', 'downloadedFromUrl', 'downloadedFromUrlReferrer', " + "'downloadedFromEmailFrom', 'downloadedFromEmailMessageId', 'downloadedFromEmailSubject', " + "'legalCopyright', 'legalTrademarks', 'privateBuild', 'specialBuild', 'companyName', " + "'createdTime', 'extensionType', 'fileDescription', 'internalName', 'md5String', 'modifiedTime', " + "'originalFileName', 'correctedPath', 'productName', 'productVersion', 'sha1String', 'size', " + "'comments', 'fileVersion', 'applicationIdentifier', 'sha256String']}", + "{'queryPath': [{" + "'requestedType': 'Driver', 'filters': [{'facetName': 'ownerMachine', 'filterType': 'Equals', " + "'values': ['username']}, {'facetName': 'endTime', 'filterType': 'Between', 'values': [" + "1569888000030, 1633564800030]}], 'connectionFeature': {'elementInstanceType': 'Driver', " + "'featureName': 'ownerMachine'}}, {'requestedType': 'Machine', 'filters': [{'facetName': " + "'timezoneUTCOffsetMinutes', 'filterType': 'Equals', 'values': [4]}, {'facetName': " + "'lastSeenTimeStamp', 'filterType': 'Between', 'values': [1569888000030, 1633564800030]}], " + "'isResult': True}], 'queryLimits': {'groupingFeature': {'elementInstanceType': 'Machine', " + "'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, " + "'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', " + "'mountPoints', 'processes', 'services', 'logonSessions', 'hasRemovableDevice', " + "'timezoneUTCOffsetMinutes', 'osVersionType', 'platformArchitecture', 'mbrHashString', 'osType', " + "'domainFqdn', 'ownerOrganization', 'pylumId', 'adSid', 'adOU', 'adOrganization', " + "'adCanonicalName', 'adCompany', 'adDNSHostName', 'adDepartment', 'adDisplayName', 'adLocation', " + "'adMachineRole', 'adDescription', 'freeDiskSpace', 'totalDiskSpace', 'freeMemory', 'totalMemory', " + "'cpuCount', 'isLaptop', 'deviceModel', 'isActiveProbeConnected', 'uptime', 'isIsolated', " + "'lastSeenTimeStamp', 'timeStampSinceLastConnectionTime', 'hasMalops', 'hasSuspicions', " "'isSuspiciousOrHasSuspiciousProcessOrFile', 'maliciousTools', 'maliciousProcesses', " "'suspiciousProcesses']}"] - queries = _remove_timestamp_from_query(queries) self._test_query_assertions(query, queries) @@ -1523,157 +1534,145 @@ def test_multiple_observation_with_qualifier_query(self): query = translation.translate('cybereason', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) queries = ["{'queryPath': [{'requestedType': 'File', 'filters': [{'facetName': 'size', 'filterType': " - "'GreaterThan', 'values': [10]}, {'facetName': 'createdTime', " - "'filterType': 'Between', 'values': [1601510400030, 1633564800030]}]," - " 'connectionFeature': {'elementInstanceType': 'File', " - "'featureName': 'ownerMachine'}}, {'requestedType': 'Machine', 'filters': [{'facetName': " - "'osVersionType', 'filterType': 'Equals', 'values': ['Windows_Server_2016']}, {'facetName': " - "'lastSeenTimeStamp', 'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], " - "'connectionFeature': {'elementInstanceType': 'Machine', 'featureName': 'services'}}, " - "{'requestedType': 'Service', 'filters': [{'facetName': 'displayName', 'filterType': 'Equals', " - "'values': ['Windows Push Notifications User Service_2d02eb']}, " - "{'facetName': 'endTime', 'filterType': 'Between', 'values': [1638180024181, 1638180324181]}]," - " 'isResult': True}], " - "'queryLimits': {'groupingFeature': {'elementInstanceType': 'Service', 'featureName': " - "'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, 'perGroupLimit': 1, " - "'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', 'binaryFile', 'ownerMachine'," - " " - "'process', 'serviceStartName', 'commandLineArguments', 'description', 'displayName', 'endTime', " - "'isActive', 'startType', 'unitFilePath', 'serviceState', 'serviceSubState', 'isAutoRestartService'," - " 'hasSuspicions', 'newServiceEvidence', 'rareServiceEvidence', 'serviceType', 'driver']}", - "{'queryPath': [{'requestedType': 'File', 'filters': [{'facetName': 'size', 'filterType': " - "'GreaterThan', 'values': [10]}, " - "{'facetName': 'createdTime', 'filterType': 'Between', 'values': [1601510400030, 1633564800030]}]," - " 'connectionFeature': {'elementInstanceType': 'File', " - "'featureName': 'ownerMachine'}}, {'requestedType': 'Machine', 'filters': [{'facetName': " - "'osVersionType', 'filterType': 'Equals', 'values': ['Windows_Server_2016']}, {'facetName': " - "'lastSeenTimeStamp', 'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], " - "'connectionFeature': {'elementInstanceType': 'Machine', 'featureName': 'services'}}, " - "{'requestedType': 'Service', 'filters': [{'facetName': 'oldServiceStartName', 'filterType': " - "'Equals', 'values': ['Windows Push Notifications User Service_2d02eb']}, {'facetName': 'endTime', " - "'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], 'isResult': True}], " - "'queryLimits': {'groupingFeature': {'elementInstanceType': 'Service', 'featureName': " - "'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, 'perGroupLimit': 1, " - "'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', 'binaryFile', 'ownerMachine'," - " " - "'process', 'serviceStartName', 'commandLineArguments', 'description', 'displayName', 'endTime', " - "'isActive', 'startType', 'unitFilePath', 'serviceState', 'serviceSubState', 'isAutoRestartService'," - " " - "'hasSuspicions', 'newServiceEvidence', 'rareServiceEvidence', 'serviceType', 'driver']}", - "{'queryPath': [{'requestedType': 'File', 'filters': [{'facetName': 'size', 'filterType': " - "'GreaterThan', 'values': [10]}, " - "{'facetName': 'createdTime', 'filterType': 'Between', 'values': [1601510400030, 1633564800030]}]," - " 'connectionFeature': {'elementInstanceType': 'File', " - "'featureName': 'ownerMachine'}}, {'requestedType': 'Machine', 'filters': [{'facetName': " - "'osVersionType', 'filterType': 'Equals', 'values': ['Windows_Server_2016']}, {'facetName': " - "'lastSeenTimeStamp', 'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], " - "'connectionFeature': {'elementInstanceType': 'Machine', 'featureName': 'services'}}, " - "{'requestedType': 'Service', 'filters': [{'facetName': 'elementDisplayName', 'filterType': " - "'Equals', 'values': ['Windows Push Notifications User Service_2d02eb']}, {'facetName': 'endTime', " - "'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], 'isResult': True}], " - "'queryLimits': {'groupingFeature': {'elementInstanceType': 'Service', 'featureName': " - "'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, 'perGroupLimit': 1, " - "'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', 'binaryFile', 'ownerMachine'," - " " - "'process', 'serviceStartName', 'commandLineArguments', 'description', 'displayName', 'endTime', " - "'isActive', 'startType', 'unitFilePath', 'serviceState', 'serviceSubState', 'isAutoRestartService'," - " " - "'hasSuspicions', 'newServiceEvidence', 'rareServiceEvidence', 'serviceType', 'driver']}", - "{'queryPath': [{'requestedType': 'File', 'filters': [{'facetName': 'size', 'filterType': " - "'GreaterThan', 'values': [10]}, " - "{'facetName': 'createdTime', 'filterType': 'Between', 'values': [1601510400030, 1633564800030]}]," - " 'connectionFeature': {'elementInstanceType': 'File', " - "'featureName': 'ownerMachine'}}, {'requestedType': 'Machine', 'filters': [{'facetName': " - "'osVersionType', 'filterType': 'Equals', 'values': ['Windows_Server_2016']}, {'facetName': " - "'lastSeenTimeStamp', 'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], " - "'connectionFeature': {'elementInstanceType': 'Machine', 'featureName': 'services'}}, " - "{'requestedType': 'Service', 'filters': [{'facetName': 'serviceStartName', 'filterType': 'Equals', " - "'values': ['Windows Push Notifications User Service_2d02eb']}, {'facetName': 'endTime', " - "'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], 'isResult': True}], " - "'queryLimits': {'groupingFeature': {'elementInstanceType': 'Service', 'featureName': " - "'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, 'perGroupLimit': 1, " - "'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', 'binaryFile', 'ownerMachine'," - " " - "'process', 'serviceStartName', 'commandLineArguments', 'description', 'displayName', 'endTime', " - "'isActive', 'startType', 'unitFilePath', 'serviceState', 'serviceSubState', 'isAutoRestartService'," - " " - "'hasSuspicions', 'newServiceEvidence', 'rareServiceEvidence', 'serviceType', 'driver']}", - "{'queryPath': [{'requestedType': 'File', 'filters': [{'facetName': 'size', 'filterType': " - "'GreaterThan', 'values': [10]}," - " {'facetName': 'createdTime', 'filterType': 'Between', 'values': [1601510400030, 1633564800030]}]," - " 'connectionFeature': {'elementInstanceType': 'File', " - "'featureName': 'ownerMachine'}}, {'requestedType': 'Machine', 'filters': [{'facetName': " + "'GreaterThan', 'values': [10]}, {'facetName': 'createdTime', 'filterType': 'Between', 'values': [" + "1601510400030, 1633564800030]}], 'isResult': True}], 'queryLimits': {'groupingFeature': {" + "'elementInstanceType': 'File', 'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, " + "'totalResultLimit': 9999, 'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': [" + "'elementDisplayName', 'avRemediationStatus', 'signerInternalOrExternal', 'fileHash', 'autoruns', " + "'ownerMachine', 'mount', 'autorun', 'dualExtensionEvidence', 'hiddenFileExtensionEvidence', " + "'rightToLeftFileExtensionEvidence', 'hasMalops', 'hasSuspicions', 'maliciousClassificationType', " + "'hackingToolClassificationEvidence', 'classificationLink', 'isPEFile', " + "'executedByProcessEvidence', 'hasAutorun', 'isInstallerProperties', 'isFromRemovableDevice', " + "'productType', 'secondExtensionType', 'temporaryFolderEvidence', 'multipleCompanyNamesEvidence', " + "'multipleHashForUnsignedPeInfoEvidence', 'unsignedHasSignedVersionEvidence', " + "'classificationComment', 'signedInternalOrExternal', 'signatureVerifiedInternalOrExternal', " + "'classificationBlocking', 'isDownloadedFromInternet', 'downloadedFromDomain', " + "'downloadedFromIpAddress', 'downloadedFromUrl', 'downloadedFromUrlReferrer', " + "'downloadedFromEmailFrom', 'downloadedFromEmailMessageId', 'downloadedFromEmailSubject', " + "'legalCopyright', 'legalTrademarks', 'privateBuild', 'specialBuild', 'companyName', " + "'createdTime', 'extensionType', 'fileDescription', 'internalName', 'md5String', 'modifiedTime', " + "'originalFileName', 'correctedPath', 'productName', 'productVersion', 'sha1String', 'size', " + "'comments', 'fileVersion', 'applicationIdentifier', 'sha256String']}", + "{'queryPath': [{" + "'requestedType': 'Machine', 'filters': [{'facetName': 'osVersionType', 'filterType': 'Equals', " + "'values': ['Windows_Server_2016']}, {'facetName': 'lastSeenTimeStamp', 'filterType': 'Between', " + "'values': [1669869087615, 1669869387615]}], 'connectionFeature': {'elementInstanceType': " + "'Machine', 'featureName': 'services'}}, {'requestedType': 'Service', 'filters': [{'facetName': " + "'displayName', 'filterType': 'Equals', 'values': ['Windows Push Notifications User " + "Service_2d02eb']}, {'facetName': 'endTime', 'filterType': 'Between', 'values': [1669869087615, " + "1669869387615]}], 'isResult': True}], 'queryLimits': {'groupingFeature': {'elementInstanceType': " + "'Service', 'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, " + "'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', " + "'binaryFile', 'ownerMachine', 'process', 'serviceStartName', 'commandLineArguments', " + "'description', 'displayName', 'endTime', 'isActive', 'startType', 'unitFilePath', 'serviceState', " + "'serviceSubState', 'isAutoRestartService', 'hasSuspicions', 'newServiceEvidence', " + "'rareServiceEvidence', 'serviceType', 'driver']}", + "{'queryPath': [{'requestedType': 'Machine', " + "'filters': [{'facetName': 'osVersionType', 'filterType': 'Equals', 'values': [" + "'Windows_Server_2016']}, {'facetName': 'lastSeenTimeStamp', 'filterType': 'Between', 'values': [" + "1669869087615, 1669869387615]}], 'connectionFeature': {'elementInstanceType': 'Machine', " + "'featureName': 'services'}}, {'requestedType': 'Service', 'filters': [{'facetName': " + "'oldServiceStartName', 'filterType': 'Equals', 'values': ['Windows Push Notifications User " + "Service_2d02eb']}, {'facetName': 'endTime', 'filterType': 'Between', 'values': [1669869087615, " + "1669869387615]}], 'isResult': True}], 'queryLimits': {'groupingFeature': {'elementInstanceType': " + "'Service', 'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, " + "'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', " + "'binaryFile', 'ownerMachine', 'process', 'serviceStartName', 'commandLineArguments', " + "'description', 'displayName', 'endTime', 'isActive', 'startType', 'unitFilePath', 'serviceState', " + "'serviceSubState', 'isAutoRestartService', 'hasSuspicions', 'newServiceEvidence', " + "'rareServiceEvidence', 'serviceType', 'driver']}", + "{'queryPath': [{'requestedType': 'Machine', " + "'filters': [{'facetName': 'osVersionType', 'filterType': 'Equals', 'values': [" + "'Windows_Server_2016']}, {'facetName': 'lastSeenTimeStamp', 'filterType': 'Between', 'values': [" + "1669869087615, 1669869387615]}], 'connectionFeature': {'elementInstanceType': 'Machine', " + "'featureName': 'services'}}, {'requestedType': 'Service', 'filters': [{'facetName': " + "'elementDisplayName', 'filterType': 'Equals', 'values': ['Windows Push Notifications User " + "Service_2d02eb']}, {'facetName': 'endTime', 'filterType': 'Between', 'values': [1669869087615, " + "1669869387615]}], 'isResult': True}], 'queryLimits': {'groupingFeature': {'elementInstanceType': " + "'Service', 'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, " + "'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', " + "'binaryFile', 'ownerMachine', 'process', 'serviceStartName', 'commandLineArguments', " + "'description', 'displayName', 'endTime', 'isActive', 'startType', 'unitFilePath', 'serviceState', " + "'serviceSubState', 'isAutoRestartService', 'hasSuspicions', 'newServiceEvidence', " + "'rareServiceEvidence', 'serviceType', 'driver']}", + "{'queryPath': [{'requestedType': 'Machine', " + "'filters': [{'facetName': 'osVersionType', 'filterType': 'Equals', 'values': [" + "'Windows_Server_2016']}, {'facetName': 'lastSeenTimeStamp', 'filterType': 'Between', 'values': [" + "1669869087615, 1669869387615]}], 'connectionFeature': {'elementInstanceType': 'Machine', " + "'featureName': 'services'}}, {'requestedType': 'Service', 'filters': [{'facetName': " + "'serviceStartName', 'filterType': 'Equals', 'values': ['Windows Push Notifications User " + "Service_2d02eb']}, {'facetName': 'endTime', 'filterType': 'Between', 'values': [1669869087615, " + "1669869387615]}], 'isResult': True}], 'queryLimits': {'groupingFeature': {'elementInstanceType': " + "'Service', 'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, " + "'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', " + "'binaryFile', 'ownerMachine', 'process', 'serviceStartName', 'commandLineArguments', " + "'description', 'displayName', 'endTime', 'isActive', 'startType', 'unitFilePath', 'serviceState', " + "'serviceSubState', 'isAutoRestartService', 'hasSuspicions', 'newServiceEvidence', " + "'rareServiceEvidence', 'serviceType', 'driver']}", + "{'queryPath': [{'requestedType': 'Machine', " + "'filters': [{'facetName': 'osVersionType', 'filterType': 'Equals', 'values': [" + "'Windows_Server_2016']}, {'facetName': 'lastSeenTimeStamp', 'filterType': 'Between', 'values': [" + "1669869087615, 1669869387615]}], 'connectionFeature': {'elementInstanceType': 'Machine', " + "'featureName': 'drivers'}}, {'requestedType': 'Driver', 'filters': [{'facetName': 'service', " + "'filterType': 'Equals', 'values': ['Windows Push Notifications User Service_2d02eb']}, " + "{'facetName': 'endTime', 'filterType': 'Between', 'values': [1669869087615, 1669869387615]}], " + "'isResult': True}], 'queryLimits': {'groupingFeature': {'elementInstanceType': 'Driver', " + "'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, " + "'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', " + "'creationTime', 'file', 'ownerMachine', 'service', 'endTime', 'newDriverEvidence', " + "'hasSuspicions']}", + "{'queryPath': [{'requestedType': 'Machine', 'filters': [{'facetName': " "'osVersionType', 'filterType': 'Equals', 'values': ['Windows_Server_2016']}, {'facetName': " - "'lastSeenTimeStamp', 'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], " - "'connectionFeature': {'elementInstanceType': 'Machine', 'featureName': 'drivers'}}, " - "{'requestedType': 'Driver', 'filters': [{'facetName': 'service', 'filterType': 'Equals', " - "'values': ['Windows Push Notifications User Service_2d02eb']}, {'facetName': 'endTime', " - "'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], 'isResult': True}], " - "'queryLimits': {'groupingFeature': {'elementInstanceType': 'Driver', 'featureName': " + "'lastSeenTimeStamp', 'filterType': 'Between', 'values': [1669869087615, 1669869387615]}], " + "'connectionFeature': {'elementInstanceType': 'Machine', 'featureName': 'processes'}}, " + "{'requestedType': 'Process', 'filters': [{'facetName': 'service', 'filterType': 'Equals', " + "'values': ['Windows Push Notifications User Service_2d02eb']}, {'facetName': 'creationTime', " + "'filterType': 'Between', 'values': [1669869087615, 1669869387615]}], 'isResult': True}], " + "'queryLimits': {'groupingFeature': {'elementInstanceType': 'Process', 'featureName': " "'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, 'perGroupLimit': 1, " - "'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', 'creationTime', 'file', " - "'ownerMachine', 'service', 'endTime', 'newDriverEvidence', 'hasSuspicions']}", - "{'queryPath': [{" - "'requestedType': 'File', 'filters': [{'facetName': 'size', 'filterType': 'GreaterThan', 'values': [" - "10]}, {'facetName': 'createdTime', " - "'filterType': 'Between', 'values': [1601510400030, 1633564800030]}], " - "'connectionFeature': {'elementInstanceType': 'File', 'featureName': " - "'ownerMachine'}}, {'requestedType': 'Machine', 'filters': [{'facetName': 'osVersionType', " - "'filterType': 'Equals', 'values': ['Windows_Server_2016']}, {'facetName': 'lastSeenTimeStamp', " - "'filterType': 'Between', 'values': [1638180024181, 1638180324181]}], 'connectionFeature': {" - "'elementInstanceType': 'Machine', 'featureName': 'processes'}}, {'requestedType': 'Process', " - "'filters': [{'facetName': 'service', 'filterType': 'Equals', 'values': ['Windows Push Notifications" - " " - "User Service_2d02eb']}," - " {'facetName': 'creationTime', 'filterType': 'Between', 'values': [1638180024181, 1638180324181]}]," - " 'isResult': True}], 'queryLimits': {'groupingFeature': {" - "'elementInstanceType': 'Process', 'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, " - "'totalResultLimit': 9999, 'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': [" - "'elementDisplayName', 'creationTime', 'endTime', 'commandLine', " - "'imageFile.maliciousClassificationType', 'productType', 'children', 'parentProcess', " - "'ownerMachine', 'calculatedUser', 'imageFile', 'imageFile.sha1String', 'imageFile.md5String', " - "'imageFile.sha256String', 'imageFile.companyName', 'imageFile.productName', 'applicablePid', " - "'imageFileExtensionType', 'integrity', 'tid', 'isAggregate', 'isDotNetProtected', 'hasMalops', " - "'hasSuspicions', 'relatedToMalop', 'multipleSizeForHashEvidence', 'isImageFileVerified', " - "'knownMaliciousToolSuspicion', 'knownMalwareSuspicion', 'knownUnwantedSuspicion', " - "'isMaliciousByHashEvidence', 'imageFileMultipleCompanyNamesEvidence', " - "'multipleHashForUnsignedPeInfoEvidence', 'multipleNameForHashEvidence', 'unknownEvidence', " - "'rareHasPeMismatchEvidence', 'imageFile.signedInternalOrExternal', " - "'unknownUnsignedBySigningCompany', 'imageFileUnsignedEvidence', " - "'imageFileUnsignedHasSignedVersionEvidence', 'unwantedModuleSuspicion', " - "'imageFile.signerInternalOrExternal', 'architecture', 'commandLineContainsTempEvidence', " - "'hasChildren', 'hasClassification', 'hasVisibleWindows', 'hasWindows', 'isInstaller', " - "'isIdentifiedProduct', 'hasModuleFromTempEvidence', 'nonExecutableExtensionEvidence', " - "'isNotShellRunner', 'runningFromTempEvidence', 'shellOfNonShellRunnerSuspicion', " - "'shellWithElevatedPrivilegesEvidence', 'systemUserEvidence', 'hasExternalConnection', " - "'hasExternalConnectionToWellKnownPortEvidence', 'hasIncomingConnection', 'hasInternalConnection', " - "'hasMailConnectionForNonMailProcessEvidence', 'hasListeningConnection', 'hasOutgoingConnection', " - "'hasUnresolvedDnsQueriesFromDomain', 'multipleUnresolvedRecordNotExistsEvidence', " - "'hasNonDefaultResolverEvidence', 'parentProcessNotMatchHierarchySuspicion', " - "'parentProcessNotAdminUserEvidence', 'parentProcessFromRemovableDeviceEvidence', 'autorun', " - "'childrenCreatedByThread', 'connections', 'elevatedPrivilegeChildren', 'hackerToolChildren', " - "'hostProcess', 'hostUser', 'hostedChildren', 'injectedChildren', 'loadedModules', 'logonSession', " - "'remoteSession', 'service', 'execedBy', 'connectionsToMaliciousDomain', " - "'connectionsToMalwareAddresses', 'externalConnections', " + "'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', 'creationTime', 'endTime', " + "'commandLine', 'imageFile.maliciousClassificationType', 'productType', 'children', " + "'parentProcess', 'ownerMachine', 'calculatedUser', 'imageFile', 'imageFile.sha1String', " + "'imageFile.md5String', 'imageFile.sha256String', 'imageFile.companyName', " + "'imageFile.productName', 'applicablePid', 'imageFileExtensionType', 'integrity', 'tid', " + "'isAggregate', 'isDotNetProtected', 'hasMalops', 'hasSuspicions', 'relatedToMalop', " + "'multipleSizeForHashEvidence', 'isImageFileVerified', 'knownMaliciousToolSuspicion', " + "'knownMalwareSuspicion', 'knownUnwantedSuspicion', 'isMaliciousByHashEvidence', " + "'imageFileMultipleCompanyNamesEvidence', 'multipleHashForUnsignedPeInfoEvidence', " + "'multipleNameForHashEvidence', 'unknownEvidence', 'rareHasPeMismatchEvidence', " + "'imageFile.signedInternalOrExternal', 'unknownUnsignedBySigningCompany', " + "'imageFileUnsignedEvidence', 'imageFileUnsignedHasSignedVersionEvidence', " + "'unwantedModuleSuspicion', 'imageFile.signerInternalOrExternal', 'architecture', " + "'commandLineContainsTempEvidence', 'hasChildren', 'hasClassification', 'hasVisibleWindows', " + "'hasWindows', 'isInstaller', 'isIdentifiedProduct', 'hasModuleFromTempEvidence', " + "'nonExecutableExtensionEvidence', 'isNotShellRunner', 'runningFromTempEvidence', " + "'shellOfNonShellRunnerSuspicion', 'shellWithElevatedPrivilegesEvidence', 'systemUserEvidence', " + "'hasExternalConnection', 'hasExternalConnectionToWellKnownPortEvidence', 'hasIncomingConnection', " + "'hasInternalConnection', 'hasMailConnectionForNonMailProcessEvidence', 'hasListeningConnection', " + "'hasOutgoingConnection', 'hasUnresolvedDnsQueriesFromDomain', " + "'multipleUnresolvedRecordNotExistsEvidence', 'hasNonDefaultResolverEvidence', " + "'parentProcessNotMatchHierarchySuspicion', 'parentProcessNotAdminUserEvidence', " + "'parentProcessFromRemovableDeviceEvidence', 'autorun', 'childrenCreatedByThread', 'connections', " + "'elevatedPrivilegeChildren', 'hackerToolChildren', 'hostProcess', 'hostUser', 'hostedChildren', " + "'injectedChildren', 'loadedModules', 'logonSession', 'remoteSession', 'service', 'execedBy', " + "'connectionsToMaliciousDomain', 'connectionsToMalwareAddresses', 'externalConnections', " "'absoluteHighVolumeMaliciousAddressConnections', 'absoluteHighVolumeExternalConnections', " "'incomingConnections', 'incomingExternalConnections', 'incomingInternalConnections', " "'internalConnections', 'listeningConnections', 'localConnections', 'mailConnections', " "'outgoingConnections', 'outgoingExternalConnections', 'outgoingInternalConnections', " "'suspiciousExternalConnections', 'suspiciousInternalConnections', 'wellKnownPortConnections', " "'lowTtlDnsQueries', 'nonDefaultResolverQueries', 'resolvedDnsQueriesDomainToDomain', " - "'resolvedDnsQueriesDomainToIp', 'resolvedDnsQueriesIpToDomain', 'suspiciousDnsQueryDomainToDomain'," - " " - "'unresolvedQueryFromSuspiciousDomain', 'dnsQueryFromSuspiciousDomain', " - "'dnsQueryToSuspiciousDomain', 'unresolvedRecordNotExist', 'unresolvedDnsQueriesFromDomain', " - "'unresolvedDnsQueriesFromIp', 'maliciousToolClassificationModules', 'malwareClassificationModules'," - " " - "'modulesNotInLoaderDbList', 'modulesFromTemp', 'unsignedWithSignedVersionModules', " - "'unwantedClassificationModules', 'accessToMalwareAddressInfectedProcess', " - "'connectingToBadReputationAddressSuspicion', 'hasMaliciousConnectionEvidence', " - "'hasSuspiciousExternalConnectionSuspicion', 'highNumberOfExternalConnectionsSuspicion', " - "'nonDefaultResolverSuspicion', 'hasRareExternalConnectionEvidence', 'hasRareRemoteAddressEvidence'," - " " - "'suspiciousMailConnections', 'accessToMalwareAddressByUnknownProcess', " + "'resolvedDnsQueriesDomainToIp', 'resolvedDnsQueriesIpToDomain', " + "'suspiciousDnsQueryDomainToDomain', 'unresolvedQueryFromSuspiciousDomain', " + "'dnsQueryFromSuspiciousDomain', 'dnsQueryToSuspiciousDomain', 'unresolvedRecordNotExist', " + "'unresolvedDnsQueriesFromDomain', 'unresolvedDnsQueriesFromIp', " + "'maliciousToolClassificationModules', 'malwareClassificationModules', 'modulesNotInLoaderDbList', " + "'modulesFromTemp', 'unsignedWithSignedVersionModules', 'unwantedClassificationModules', " + "'accessToMalwareAddressInfectedProcess', 'connectingToBadReputationAddressSuspicion', " + "'hasMaliciousConnectionEvidence', 'hasSuspiciousExternalConnectionSuspicion', " + "'highNumberOfExternalConnectionsSuspicion', 'nonDefaultResolverSuspicion', " + "'hasRareExternalConnectionEvidence', 'hasRareRemoteAddressEvidence', 'suspiciousMailConnections', " + "'accessToMalwareAddressByUnknownProcess', " "'hasAbsoluteHighVolumeConnectionToMaliciousAddressEvidence', " "'hasAbsoluteHighVolumeExternalOutgoingConnectionEvidence', 'highDataTransmittedSuspicion', " "'highDataVolumeTransmittedToMaliciousAddressSuspicion', " @@ -1704,6 +1703,7 @@ def test_multiple_observation_with_qualifier_query(self): "'imageFile.downloadedFromUrlReferrer', 'imageFile.downloadedFromEmailFrom', " "'imageFile.downloadedFromEmailMessageId', 'imageFile.downloadedFromEmailSubject', 'rpcRequests', " "'iconBase64', 'executionPrevented', 'isWhiteListClassification', 'matchedWhiteListRuleIds']}"] + queries = _remove_timestamp_from_query(queries) self._test_query_assertions(query, queries) @@ -1792,7 +1792,6 @@ def test_not_include_filter_query(self): "'hasMaliciousProcess', 'hasSuspicions', 'hasSuspiciousProcess', " "'runningMaliciousProcessEvidence', 'hasRareProcessWithExternalConnections']}"] - queries = _remove_timestamp_from_query(queries) self._test_query_assertions(query, queries) @@ -1960,11 +1959,12 @@ def test_invalid_operator_for_int(self): assert 'LIKE OR MATCHES operator is not supported for Integer/Timestamp/Boolean fields' in result['error'] def test_observation_with_invalid_link_between_element(self): - stix_pattern = "[x-cybereason-connection:port_type LIKE 'HTTP' AND x-oca-event:file_event_user LIKE 'FET_DELETE']" + stix_pattern = "[x-cybereason-connection:port_type LIKE 'HTTP' AND x-oca-event:file_event_user " \ + "LIKE 'FET_DELETE']" result = translation.translate('cybereason', 'query', '{}', stix_pattern) assert result['success'] is False assert result['code'] == 'invalid_parameter' - assert 'Link is not found between elements' in result['error'] + assert 'Cybereason does not allow AND operation' in result['error'] def test_observation_with_invalid_operator_for_string(self): stix_pattern = "[x-cybereason-connection:port_type > 'HTTP']" @@ -1992,32 +1992,96 @@ def test_timestamp_in_seconds_and_milliseconds(self): "[network-traffic:protocols[*] = 'tcp'] START t'2019-10-01T08:43:10.003Z' STOP " \ "t'2019-11-30T10:43:10.005Z' " query = translation.translate('cybereason', 'query', '{}', stix_pattern) - queries = [{'queryPath': [{'requestedType': 'Connection', 'filters': [{'facetName': 'transportProtocol', - 'filterType': 'Equals', - 'values': ['tcp']}, - {'facetName': 'creationTime', - 'filterType': 'Between', - 'values': [1569919390003, - 1575110590005]}, - {'facetName': 'localPort', - 'filterType': 'Equals', - 'values': [23]}, - {'facetName': 'creationTime', - 'filterType': 'Between', - 'values': [1569916810000, - 1575111610000]}], - 'isResult': True}], 'queryLimits': {'groupingFeature': - {'elementInstanceType': 'Connection', - 'featureName': 'elementDisplayName'}}, - 'perFeatureLimit': 1, - 'totalResultLimit': 9999, 'perGroupLimit': 1, 'templateContext': 'CUSTOM', - 'customFields': ['elementDisplayName', 'direction', 'ownerMachine', 'ownerProcess', 'serverPort', - 'serverAddress', 'portType', 'aggregatedReceivedBytesCount', - 'aggregatedTransmittedBytesCount', 'remoteAddressCountryName', 'dnsQuery', - 'calculatedCreationTime', 'domainName', 'endTime', 'localPort', 'portDescription', - 'remotePort', 'state', 'isExternalConnection', 'isIncoming', - 'remoteAddressInternalExternalLocal', 'transportProtocol', 'hasMalops', - 'hasSuspicions', 'relatedToMalop', 'isWellKnownPort', 'isProcessLegit', - 'isProcessMalware', 'localAddress', 'remoteAddress', 'urlDomains']}] + queries = [{ + 'queryPath': [{ + 'requestedType': 'Connection', + 'filters': [{ + 'facetName': 'localPort', + 'filterType': 'Equals', + 'values': [23] + }, { + 'facetName': 'creationTime', + 'filterType': 'Between', + 'values': [1569916810000, 1575111610000] + }], + 'isResult': True + }], + 'queryLimits': { + 'groupingFeature': { + 'elementInstanceType': 'Connection', + 'featureName': 'elementDisplayName' + } + }, + 'perFeatureLimit': 1, + 'totalResultLimit': 9999, + 'perGroupLimit': 1, + 'templateContext': 'CUSTOM', + 'customFields': ['elementDisplayName', 'direction', 'ownerMachine', 'ownerProcess', 'serverPort', + 'serverAddress', 'portType', 'aggregatedReceivedBytesCount', + 'aggregatedTransmittedBytesCount', 'remoteAddressCountryName', 'dnsQuery', + 'calculatedCreationTime', 'domainName', 'endTime', 'localPort', 'portDescription', + 'remotePort', 'state', 'isExternalConnection', 'isIncoming', + 'remoteAddressInternalExternalLocal', 'transportProtocol', 'hasMalops', 'hasSuspicions', + 'relatedToMalop', 'isWellKnownPort', 'isProcessLegit', 'isProcessMalware', 'localAddress', + 'remoteAddress', 'urlDomains'] + }, { + 'queryPath': [{ + 'requestedType': 'Connection', + 'filters': [{ + 'facetName': 'transportProtocol', + 'filterType': 'Equals', + 'values': ['tcp'] + }, { + 'facetName': 'creationTime', + 'filterType': 'Between', + 'values': [1569919390003, 1575110590005] + }], + 'isResult': True + }], + 'queryLimits': { + 'groupingFeature': { + 'elementInstanceType': 'Connection', + 'featureName': 'elementDisplayName' + } + }, + 'perFeatureLimit': 1, + 'totalResultLimit': 9999, + 'perGroupLimit': 1, + 'templateContext': 'CUSTOM', + 'customFields': ['elementDisplayName', 'direction', 'ownerMachine', 'ownerProcess', 'serverPort', + 'serverAddress', 'portType', 'aggregatedReceivedBytesCount', + 'aggregatedTransmittedBytesCount', 'remoteAddressCountryName', 'dnsQuery', + 'calculatedCreationTime', 'domainName', 'endTime', 'localPort', 'portDescription', + 'remotePort', 'state', 'isExternalConnection', 'isIncoming', + 'remoteAddressInternalExternalLocal', 'transportProtocol', 'hasMalops', 'hasSuspicions', + 'relatedToMalop', 'isWellKnownPort', 'isProcessLegit', 'isProcessMalware', 'localAddress', + 'remoteAddress', 'urlDomains'] + }] + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_and_without_linked_element(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1' AND x-cybereason-driver:name = 'test_driver'] " \ + "OR [AND x-oca-asset:os_type = 'windows' AND x-cybereason-service:description LIKE 'service' ]" + query = translation.translate('cybereason', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ["{'queryPath': [{'requestedType': 'Service', 'filters': [{'facetName': 'description', " + "'filterType': 'ContainsIgnoreCase', 'values': ['service']}, {'facetName': 'endTime', " + "'filterType': 'Between', 'values': [1669871110786, 1669871410786]}], 'connectionFeature': {" + "'elementInstanceType': 'Service', 'featureName': 'ownerMachine'}}, {'requestedType': 'Machine', " + "'filters': [{'facetName': 'osType', 'filterType': 'Equals', 'values': ['windows']}, {'facetName': " + "'lastSeenTimeStamp', 'filterType': 'Between', 'values': [1669871110786, 1669871410786]}], " + "'isResult': True}], 'queryLimits': {'groupingFeature': {'elementInstanceType': 'Machine', " + "'featureName': 'elementDisplayName'}}, 'perFeatureLimit': 1, 'totalResultLimit': 9999, " + "'perGroupLimit': 1, 'templateContext': 'CUSTOM', 'customFields': ['elementDisplayName', " + "'mountPoints', 'processes', 'services', 'logonSessions', 'hasRemovableDevice', " + "'timezoneUTCOffsetMinutes', 'osVersionType', 'platformArchitecture', 'mbrHashString', 'osType', " + "'domainFqdn', 'ownerOrganization', 'pylumId', 'adSid', 'adOU', 'adOrganization', " + "'adCanonicalName', 'adCompany', 'adDNSHostName', 'adDepartment', 'adDisplayName', 'adLocation', " + "'adMachineRole', 'adDescription', 'freeDiskSpace', 'totalDiskSpace', 'freeMemory', 'totalMemory', " + "'cpuCount', 'isLaptop', 'deviceModel', 'isActiveProbeConnected', 'uptime', 'isIsolated', " + "'lastSeenTimeStamp', 'timeStampSinceLastConnectionTime', 'hasMalops', 'hasSuspicions', " + "'isSuspiciousOrHasSuspiciousProcessOrFile', 'maliciousTools', 'maliciousProcesses', " + "'suspiciousProcesses']}"] + queries = _remove_timestamp_from_query(queries) self._test_query_assertions(query, queries)