[Marketplace Contribution] Github Maltrail Feed

Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py

@@ -0,0 +1,209 @@
+import demistomock as demisto  # noqa: F401
+from CommonServerPython import *  # noqa: F401
+import requests
+import base64
+from datetime import datetime
+import regex
+
+# CONSTANTS
+SOURCE_NAME = "Github Maltrail Feed"
+DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+COMMIT_LIMIT = 100
+
+# ############################## OVERWRITE REGEX FORMATTING ###############################
+regexFlags = re.M  # Multi line matching
+RGX_IP = r"\b(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(?:\[\.\]|\.)){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\b"
+
+
+class Client(BaseClient):
+
+    def __init__(self, params: dict):
+        self._verify: bool = not params.get('insecure', False)
+        self.user = params.get('user')
+        self.token = (params.get('api_token') or {}).get('password', '')
+        self.repo = params.get('repository')
+        self.url = params.get('base_url')
+        # self.base_url = f'{self.url}/{self.user}/{self.repo}'
+        self.base_url = urljoin(self.url, self.user)
+        self.base_url = urljoin(self.base_url, self.repo)
+        handle_proxy()
+
+    def http_request_indicators(self):
+        res = requests.get(
+            url=self.base_url,
+            verify=self._verify
+        )
+        try:
+            res.raise_for_status()
+        except Exception:
+            demisto.info(f'Github Maltrail Feed - exception in request: {res.status_code!r} {res.content!r}')
+            raise
+        return res.text
+
+    def getclienturl(self):
+        return self.base_url
+
+    def http_request(self, url_endpoint, params: dict = None):
+        """The HTTP request for daily feeds.
+        Returns:
+            list. A list of indicators fetched from the feed.
+        """
+        self.headers = {
+            'Authorization': "Bearer " + self.token
+        }
+        res = requests.request(
+            method="GET",
+            url=urljoin(self.base_url, url_endpoint),
+            verify=self._verify,
+            headers=self.headers,
+            params=params
+        )
+        return res
+
+
+def fetch_indicators(client: Client, url: str, limit: int = None, params: dict = None):
+    if params:
+        feed_tags = argToList(params.get('feedTags', []))
+        tlp_color = params.get('tlp_color')
+    response = client.http_request(url)
+    indicators_list = []
+    demisto.debug('Fetch of indicators started ###')
+
+    if response.ok:
+        content = response.json()["content"]
+        file_content = base64.b64decode(content).decode("utf-8")
+        lines = file_content.split("\n")
+        for line in lines:
+            if '#' not in line and line != '':
+                type_ = auto_detect_indicator_type(line)
+                if regex.search(RGX_IP, line):
+                    if line.startswith('http://'):
+                        line = line.removeprefix('http://')
+                    elif line.startswith('https://'):
+                        line = line.removeprefix('https://')
+                    else:
+                        line = line.split(':')[0]
+                    type_ = "IP"
+                elif type_ == "URL":
+                    if not line.startswith('http://') and not line.startswith('https://'):
+                        line = 'http://' + line
+                raw_data = {
+                    'value': line,
+                    'type': type_,
+                }
+                indicator_obj = {
+                    'value': line,
+                    'type': type_,
+                    'service': "GitHub Maltrail Feed",
+                    'fields': {},
+                    'rawJSON': raw_data
+                }
+                if feed_tags:
+                    indicator_obj['fields']['tags'] = feed_tags
+                if tlp_color:
+                    indicator_obj['fields']['trafficlightprotocol'] = tlp_color
+                indicators_list.append(indicator_obj)
+                # If limit is reached, break loop
+                if limit and isinstance(limit, int):
+                    if len(indicators_list) >= limit:
+                        break
+    else:
+        demisto.error(f"Error: {response.status_code} - {response.json()['message']}")
+    return indicators_list
+
+
+def get_last_commit_date(client):
+    api_url = "/commits"
+    response = client.http_request(api_url)
+    last_commit_date = None
+    if response.ok:
+        commits = []
+        page = 1
+        while response.ok and page < COMMIT_LIMIT:
+            commits.extend(response.json())
+            link_header = response.headers.get('Link')
+            if not link_header or 'rel="next"' not in link_header:
+                break
+            page += 1
+            response = client.http_request(api_url, params={'page': page})
+        for commit in commits:
+            if 'qakbot' in commit['commit']['message']:
+                commit_date = date_to_timestamp(parse_date_string(commit['commit']['author']['date'], DATE_FORMAT))
+                if not last_commit_date:
+                    last_commit_date = commit_date
+                elif commit_date > last_commit_date:
+                    last_commit_date = commit_date
+
+    return last_commit_date
+
+
+def fetch_indicators_command(client: Client, params: dict = None):
+    integration_context = get_integration_context()
+    api_url = "/contents/trails/static/malware/qakbot.txt"
+    indicators_list = []
+    # First Fetch
+    if not integration_context:
+        time_of_first_fetch = date_to_timestamp(datetime.now(), DATE_FORMAT)
+        set_integration_context({'time_of_last_fetch': time_of_first_fetch})
+        indicators_list = fetch_indicators(client, api_url, None, params)
+    else:
+        time_from_last_update = integration_context.get('time_of_last_fetch')
+        now = date_to_timestamp(datetime.now(), DATE_FORMAT)
+        last_commit_date = get_last_commit_date(client)
+        if last_commit_date > time_from_last_update:
+            indicators_list = fetch_indicators(client, api_url, None, params)
+            set_integration_context({'time_of_last_fetch': now})
+        else:
+            demisto.debug('### Nothing to fetch')
+
+    return indicators_list
+
+
+def get_indicators_command(client: Client, params: dict, args: dict):
+    try:
+        limit = int(args.get('limit', 50))
+    except ValueError:
+        raise ValueError('The limit argument must be a number.')
+    api_url = "/contents/trails/static/malware/qakbot.txt"
+    indicators_list = fetch_indicators(client, api_url, limit, params)
+    entry_result = indicators_list[:limit]
+    human_readable = tableToMarkdown("Indicators from Github Maltrail:", entry_result,
+                                     headers=['value', 'type', 'firstseenbysource', 'lastseenbysource', 'name'],
+                                     removeNull=True)
+    return human_readable, {}, entry_result
+
+
+def test_module_command(client: Client, params: dict, args: dict):
+    client.http_request_indicators()
+    return 'ok', {}, {}
+
+
+def main():
+    params = demisto.params()
+    args = demisto.args()
+
+    command = demisto.command()
+    demisto.info(f'Command being called is {command}')
+
+    # Switch case
+    commands = {
+        'test-module': test_module_command,
+        'gh-maltrail-get-indicators': get_indicators_command
+    }
+
+    try:
+        client = Client(params)
+        if command == 'fetch-indicators':
+            indicators = fetch_indicators_command(client, params)
+            for b in batch(indicators, batch_size=2000):
+                demisto.createIndicators(b)
+        else:
+            readable_output, outputs, raw_response = commands[command](client, params, args)
+            return_outputs(readable_output, outputs, raw_response)
+
+    except Exception as e:
+        raise Exception(f'Error in {SOURCE_NAME} Integration [{e}]')
+
+
+if __name__ in ('__main__', '__builtin__', 'builtins'):
+    main()

Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.yml

@@ -0,0 +1,126 @@
+category: Utilities
+commonfields:
+  id: Github Maltrail Feed
+  version: -1
+configuration:
+- additionalinfo: API Token
+  display: ""
+  displaypassword: API Token
+  hiddenusername: true
+  name: api_token
+  required: true
+  type: 9
+- display: 'Username of the repository owner, for example: github.com/repos/{user}/{repo}/issues'
+  name: user
+  required: true
+  type: 0
+- defaultvalue: https://api.github.com/repos
+  display: Base URL
+  name: base_url
+  required: true
+  type: 0
+- display: 'The name of the requested repository, for example: github.com/repos/{user}/{repo}/issues'
+  name: repository
+  required: true
+  type: 0
+- advanced: true
+  display: Trust any certificate (not secure)
+  name: insecure
+  required: false
+  section: Connect
+  type: 8
+- advanced: true
+  display: Use system proxy settings
+  name: proxy
+  required: false
+  section: Connect
+  type: 8
+- defaultvalue: "15"
+  display: Feed Fetch Interval
+  name: feedFetchInterval
+  required: false
+  type: 19
+- defaultvalue: "true"
+  display: Fetch indicators
+  name: feed
+  required: false
+  type: 8
+- additionalinfo: Indicators from this integration instance will be marked with this reputation
+  defaultvalue: Bad
+  display: Indicator Reputation
+  name: feedReputation
+  options:
+  - None
+  - Good
+  - Suspicious
+  - Bad
+  required: false
+  type: 18
+- additionalinfo: Reliability of the source providing the intelligence data
+  defaultvalue: F - Reliability cannot be judged
+  display: Source Reliability
+  name: feedReliability
+  options:
+  - A - Completely reliable
+  - B - Usually reliable
+  - C - Fairly reliable
+  - D - Not usually reliable
+  - E - Unreliable
+  - F - Reliability cannot be judged
+  required: true
+  type: 15
+- defaultvalue: indicatorType
+  display: ''
+  name: feedExpirationPolicy
+  options:
+  - never
+  - interval
+  - indicatorType
+  - suddenDeath
+  required: false
+  type: 17
+- defaultvalue: "20160"
+  display: ''
+  name: feedExpirationInterval
+  required: false
+  type: 1
+- additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system.
+  display: Bypass exclusion list
+  name: feedBypassExclusionList
+  required: false
+  type: 8
+- additionalinfo: Supports CSV values.
+  display: Tags
+  name: feedTags
+  required: false
+  type: 0
+- additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed
+  display: Traffic Light Protocol Color
+  name: tlp_color
+  options:
+  - RED
+  - AMBER
+  - GREEN
+  - WHITE
+  required: false
+  type: 15
+description: Fetches Indicators from Github Repo https://github.com/stamparm/maltrail
+display: Github Maltrail Feed
+name: Github Maltrail Feed
+script:
+  commands:
+  - arguments:
+    - name: limit
+      description: The maximum number of results to return to the output.
+      defaultValue: "50"
+    name: gh-maltrail-get-indicators
+    description: Get indicators from the feed.
+  dockerimage: demisto/python3:3.10.13.78960
+  feed: true
+  runonce: false
+  script: ''
+  subtype: python3
+  type: python
+tests:
+- No tests (auto formatted)
+fromversion: 6.10.0

Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed_description.md

@@ -0,0 +1 @@
+Fetches Indicators from Github Repo https://github.com/stamparm/maltrail

Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/README.md

@@ -0,0 +1,49 @@
+Fetches Indicators from Github Repo https://github.com/stamparm/maltrail
+
+## Configure Github Maltrail Feed on Cortex XSOAR
+
+1. Navigate to **Settings** > **Integrations** > **Servers & Services**.
+2. Search for Github Maltrail Feed.
+3. Click **Add instance** to create and configure a new integration instance.
+
+    | **Parameter** | **Description** | **Required** |
+    | --- | --- | --- |
+    | API Token | API Token | True |
+    | Username of the repository owner, for example: github.com/repos/{user}/{repo}/issues |  | True |
+    | Base URL |  | True |
+    | The name of the requested repository, for example: github.com/repos/{user}/{repo}/issues |  | True |
+    | Trust any certificate (not secure) |  | False |
+    | Use system proxy settings |  | False |
+    | Feed Fetch Interval |  | False |
+    | Fetch indicators |  | False |
+    | Indicator Reputation | Indicators from this integration instance will be marked with this reputation | False |
+    | Source Reliability | Reliability of the source providing the intelligence data | True |
+    | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False |
+    | Tags | Supports CSV values. | False |
+    | Traffic Light Protocol Color | The Traffic Light Protocol \(TLP\) designation to apply to indicators fetched from the feed | False |
+
+4. Click **Test** to validate the URLs, token, and connection.
+
+## Commands
+
+You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook.
+After you successfully execute a command, a DBot message appears in the War Room with the command details.
+
+### gh-maltrail-get-indicators
+
+***
+Get indicators from the feed.
+
+#### Base Command
+
+`gh-maltrail-get-indicators`
+
+#### Input
+
+| **Argument Name** | **Description** | **Required** |
+| --- | --- | --- |
+| limit | The maximum number of results to return to the output. Default is 50. | Optional | 
+
+#### Context Output
+
+There is no context output for this command.