Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Virus total connector #1458

Merged
Merged
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
46550a3
Add AbuseIPDB Connector
SalmanMesia Apr 21, 2023
ea4b7ba
Add SANS ISC DShield Connector
SalmanMesia Apr 21, 2023
be4c222
Add Mandiant Connector
SalmanMesia Apr 21, 2023
08d156e
Add Maxmind Connector
SalmanMesia Apr 21, 2023
d727c2a
Add Recorded Future Connector
SalmanMesia Apr 21, 2023
43f5345
Update ReversingLabs connector
SalmanMesia Apr 21, 2023
9fab461
Add ThreatQ Connector
SalmanMesia Apr 21, 2023
b1af968
Add Cisco Secure Malware Analytics Connector
SalmanMesia Apr 21, 2023
97a0584
Add VirusTotal Connector
SalmanMesia Apr 21, 2023
cff88a4
Add Intezer Connector
SalmanMesia Apr 21, 2023
4d79667
Add Alienvault OTX Connector
SalmanMesia Apr 21, 2023
71b25f5
Merge remote-tracking branch 'upstream' into develop
SalmanMesia Apr 23, 2023
3ccb669
add IPv6Network import
SalmanMesia Apr 23, 2023
ccfb452
Merge branch 'develop' of https://github.com/opencybersecurityallianc…
SalmanMesia Apr 26, 2023
564bd68
update req.txt
SalmanMesia Apr 26, 2023
41e1c18
Update query_contructor.json to use operators.json ; Update requireme…
SalmanMesia Apr 26, 2023
8ab7652
Merge branch 'opencybersecurityalliance:develop' into develop
SalmanMesia Apr 26, 2023
096a747
Merge branch 'opencybersecurityalliance:develop' into develop
SalmanMesia Apr 27, 2023
417a7ce
Merge branch 'opencybersecurityalliance:develop' into develop
SalmanMesia Apr 27, 2023
981d62a
Merge branch 'opencybersecurityalliance:develop' into develop
SalmanMesia May 2, 2023
564b5a2
Add VirusTotal Connector
SalmanMesia May 2, 2023
2a5ab6a
Merge branch 'develop' into virus_total-connector
SalmanMesia May 3, 2023
7e5b27a
Merge branch 'develop' into virus_total-connector
May 12, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update query_contructor.json to use operators.json ; Update requireme…
…nts.txt ; Updated config.json and lang_en.json
SalmanMesia committed Apr 26, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 41e1c1859d45f8306fb6fec0c8d96e5c756c8153
35 changes: 1 addition & 34 deletions stix_shifter_modules/abuseipdb/configuration/config.json
Original file line number Diff line number Diff line change
@@ -2,11 +2,8 @@
"connection": {
"type": {
"id": "AbuseIPDB_Connector",
"type": "connectorType",
"displayName": "AbuseIPDB",
"description": "Determine whether an IP was reported or not as malicious by AbuseIPDB.",
"maxConnections": 1,
"maxConfigurations": 1
"description": "Determine whether an IP was reported or not as malicious by AbuseIPDB."
},
"options": {
"type": "fields",
@@ -40,36 +37,6 @@
"max": 60,
"type": "number",
"previous": "connection.timeoutLimit"
},
"language": {
"type": "string",
"default": "stix",
"optional": true,
"hidden": true
},
"validate_pattern": {
"type": "boolean",
"optional": true,
"hidden": true,
"previous": "connection.validate_pattern"
},
"stix_validator": {
"type": "boolean",
"default": false,
"optional": true,
"hidden": true,
"previous": "connection.stix_validator"
},
"mapping": {
"type": "json",
"optional": true,
"previous": "connection.mapping"
},
"unmapped_fallback": {
"type": "boolean",
"default": false,
"optional": true,
"hidden": true
}
},
"help": {
78 changes: 32 additions & 46 deletions stix_shifter_modules/abuseipdb/configuration/lang_en.json
Original file line number Diff line number Diff line change
@@ -1,74 +1,60 @@
{
"connection": {
"name": {
"label": "Data source name",
"placeholder": "AbuseIPDB"
},
"description": {
"label": "Data source description",
"placeholder": "Connector for AbuseIPDB"
},
"options": {
"concurrent": {
"label": "Concurrent Search Limit",
"description": "The number of simultaneous connections that can be made between IBM Cloud Pak™ for Security and the data source. Valid input range is {{min}} to {{max}}."
"description": "The number of simultaneous connections that can be made between the host and the data source. Valid input range is {{min}} to {{max}}."
},
"search_timeout": {
"label": "Query Search Timeout Limit",
"description": "The limit on how long the query will run, in minutes, on the data source."
},
"result_limit": {
"label": "Result Size Limit",
"description": "The maximum number of entries or objects that are returned by search query. Valid input range is {{min}} to {{max}}."
},
"time_range": {
"label": "Query Time Range",
"description": "Time range for the search, in minutes, represented as last x minutes. Valid input range is {{min}} to {{max}}."
},
"timeout": {
"label": "Query Response Timeout Limit",
"description": "The limit on how long to wait for the data source response, in seconds. Valid input range is {{min}} to {{max}}."
},
"validate_pattern": {
"label": "STIX Pattern Validator",
"description": "Validate STIX patterns that needs to be translated into native data source query"
},
"stix_validator": {
"label": "STIX Object Validator",
"description": "Validate translated STIX Objects"
},
"mapping": {
"label": "Custom Mapping",
"description": "Custom stix mapping if default mapping needs to be replaced"
}
},
"host": {
"label": "Management IP address or Hostname",
"placeholder": "192.168.1.10",
"description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it"
"description": "Specify the OCP Cluster hostname or the XForce API host URL"
},
"port": {
"label": "Host Port",
"description": "Set the port number that is associated with the Host name or IP"
},
"help": {
"label": "Need additional help?",
"description": "More details on the data source setting can be found in the specified link"
"namespace": {
"label": "The UUID Namespace to generate unique ",
"description": "Supply a UUID to generate deterministic UUIDs for the resulting STIX bundle"
}
},
"configuration": {
"name": {
"label": "Configuration Name",
"placeholder": "Add a configuration name"
},
"description": {
"label": "Configuration Description",
"placeholder": "Add a configuration description"
},
"auth": {
"key": {
"label": "Key",
"description": "API Key for AbuseIPDB"
"description": "The APIKey for AbuseIPDB Threat Feed"
}
},
"rateLimit": {
"rateLimit": {
"label": "Rate Limit",
"description": "The number of queries allowed by AbuseIPDB"
},
"rateUnit": {
"label": "Rate Unit",
"description": "The rate unit for rate limit in [seconds, minutes, days, months, years ...]"
}
},
"cacheDuration": {
"cacheDuration": {
"label": "Cache Duration",
"description": "How long should we cache the results of the STIX Bundle execution?"
},
"unit": {
"label": "Rate Unit",
"description": "The unit for cache in [seconds, minutes, days, months, years ...]"
}
},
"dataTypeList": {
"ip": {
"label": "IP Address",
"description": "Whether IP Address lookup queries are supported by AbuseIPDB based on the User's API Provisioning"
}
}
}
Original file line number Diff line number Diff line change
@@ -19,27 +19,9 @@

class QueryStringPatternTranslator:
# Change comparator values to match with supported data source operators
comparator_lookup = {
ComparisonExpressionOperators.And: "AND",
ComparisonExpressionOperators.Or: "OR",
ComparisonComparators.GreaterThan: ">",
ComparisonComparators.GreaterThanOrEqual: ">=",
ComparisonComparators.LessThan: "<",
ComparisonComparators.LessThanOrEqual: "<=",
ComparisonComparators.Equal: "=",
ComparisonComparators.NotEqual: "!=",
ComparisonComparators.Like: "LIKE",
ComparisonComparators.In: "IN",
ComparisonComparators.Matches: 'LIKE',
# ComparisonComparators.IsSubSet: '',
# ComparisonComparators.IsSuperSet: '',
ObservationOperators.Or: 'OR',
# Treat AND's as OR's -- Unsure how two ObsExps wouldn't cancel each other out.
ObservationOperators.And: 'OR'
}

def __init__(self, pattern: Pattern, data_model_mapper):
self.dmm = data_model_mapper
self.comparator_lookup = self.dmm.map_comparator()
self.pattern = pattern
self.translated = self.parse_expression(pattern)

@@ -125,29 +107,19 @@ def _is_reference_value(stix_field):

@staticmethod
def _lookup_comparison_operator(self, expression_operator):
if expression_operator not in self.comparator_lookup:
if str(expression_operator) not in self.comparator_lookup:
raise NotImplementedError("Comparison operator {} unsupported for Dummy connector".format(expression_operator.name))
return self.comparator_lookup[expression_operator]
return self.comparator_lookup[str(expression_operator)]

def _parse_expression(self, expression, qualifier=None) -> str:
if isinstance(expression, ComparisonExpression): # Base Case
# Resolve STIX Object Path to a field in the target Data Model
stix_object, stix_field = expression.object_path.split(':')
#TODO DEBUGGING
# print("STIX Object: ", stix_object)
# print("STIX Field: ", stix_field)
# Multiple data source fields may map to the same STIX Object
mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
#TODO DEBUGGING
# print("MAPPED FIELDS ARRAY:", mapped_fields_array)
# Resolve the comparison symbol to use in the query string (usually just ':')
comparator = self._lookup_comparison_operator(self, expression.comparator)
#TODO DEBUGGING
# print("COMPARATOR: ", comparator)
if stix_field == 'start' or stix_field == 'end':
transformer = TimestampToMilliseconds()
expression.value = transformer.transform(expression.value)

# Some values are formatted differently based on how they're being compared
if expression.comparator == ComparisonComparators.Matches: # needs forward slashes
value = self._format_match(expression.value)
@@ -163,8 +135,6 @@ def _parse_expression(self, expression, qualifier=None) -> str:
else:
value = self._escape_value(expression.value)

#TODO DEBUGGING
# print("VALUE: ", value)
get_data_source_query(stix_field=stix_field, stix_object=stix_object, value=value)

comparison_string = self._parse_mapped_fields(self, expression, value, comparator, stix_field, mapped_fields_array)
@@ -230,22 +200,12 @@ def parse_expression(self, pattern: Pattern):

def translate_pattern(pattern: Pattern, data_model_mapping, options):
# Query result limit and time range can be passed into the QueryStringPatternTranslator if supported by the data source.
# result_limit = options['result_limit']
# time_range = options['time_range']
query = QueryStringPatternTranslator(pattern, data_model_mapping).translated
# Add space around START STOP qualifiers
query = re.sub("START", "START ", query)
query = re.sub("STOP", " STOP ", query)

#TODO DEBUGGING
# print("Query is: ", query)
# print("PATTERN: " , pattern)

# If supported by the query language, a limit on the number of results should be added to the query as defined by options['result_limit'].
# Translated patterns must be returned as a list of one or more native query strings.
# A list is returned because some query languages require the STIX pattern to be split into multiple query strings.
translated_query = {"data": data, "dataType": dataType}
# return [translated_query]
return [str(translated_query)]

def get_data_source_query(stix_field, stix_object, value):
35 changes: 1 addition & 34 deletions stix_shifter_modules/alienvault_otx/configuration/config.json
Original file line number Diff line number Diff line change
@@ -2,11 +2,8 @@
"connection": {
"type": {
"id": "OTXQuery_Connector",
"type": "connectorType",
"displayName": "AlienVault OTX",
"description": "Query AlienVault OTX for IPs, domains, URLs, or file hashes",
"maxConnections": 1,
"maxConfigurations": 1
"description": "Query AlienVault OTX for IPs, domains, URLs, or file hashes"
},
"help": {
"default": "www.ibm.com",
@@ -44,36 +41,6 @@
"max": 60,
"type": "number",
"previous": "connection.timeoutLimit"
},
"language": {
"type": "string",
"default": "stix",
"optional": true,
"hidden": true
},
"validate_pattern": {
"type": "boolean",
"optional": true,
"hidden": true,
"previous": "connection.validate_pattern"
},
"stix_validator": {
"type": "boolean",
"default": false,
"optional": true,
"hidden": true,
"previous": "connection.stix_validator"
},
"mapping": {
"type": "json",
"optional": true,
"previous": "connection.mapping"
},
"unmapped_fallback": {
"type": "boolean",
"default": false,
"optional": true,
"hidden": true
}
},
"namespace":{
Loading