Phishing framework concurrency and tweaks

api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py

@@ -32,6 +32,7 @@ class PhishingFormCompiler(FileAnalyzer):
     pin_matching: list = []
     cvv_matching: list = []
     expiration_date_matching: list = []
+    user_agent: str = ""
 
     def __init__(
         self,
@@ -196,9 +197,13 @@ def perform_request_to_form(self, form) -> Response:
         params = self.compile_form_field(form)
         dest_url = self.extract_action_attribute(form)
         logger.info(f"Job #{self.job_id}: Sending {params=} to submit url {dest_url}")
+        headers = {
+            "User-Agent": self.user_agent,
+        }
         response = requests.post(
             url=dest_url,
             data=params,
+            headers=headers,
             proxies=(
                 {"http": self.proxy_address, "https": self.proxy_address}
                 if self.proxy_address

api_app/analyzers_manager/migrations/0143_alter_analyzer_config_phishing_extractor_and_form_compiler.py

@@ -0,0 +1,107 @@
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+    pm_extractor = PythonModule.objects.get(
+        module="phishing.phishing_extractor.PhishingExtractor",
+        base_path="api_app.analyzers_manager.observable_analyzers",
+    )
+    pm_form_compiler = PythonModule.objects.get(
+        module="phishing.phishing_form_compiler.PhishingFormCompiler",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+    p_extractor = Parameter.objects.create(
+        name="user_agent",
+        type="str",
+        description="Custom user agent for the Phishing Extractor Selenium browser.",
+        is_secret=False,
+        required=False,
+        python_module=pm_extractor,
+    )
+    p_form_compiler = Parameter.objects.create(
+        name="user_agent",
+        type="str",
+        description="Custom user agent for the compilation of form.",
+        is_secret=False,
+        required=False,
+        python_module=pm_form_compiler,
+    )
+    for config in pm_extractor.analyzerconfigs.all():
+        PluginConfig.objects.create(
+            parameter=p_extractor,
+            analyzer_config=config,
+            value="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3",
+            owner=None,
+            for_organization=False,
+        )
+
+    for config in pm_form_compiler.analyzerconfigs.all():
+        PluginConfig.objects.create(
+            parameter=p_form_compiler,
+            analyzer_config=config,
+            value="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3",
+            owner=None,
+            for_organization=False,
+        )
+
+
+def reverse_migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+    pm_extractor = PythonModule.objects.get(
+        module="phishing.phishing_extractor.PhishingExtractor",
+        base_path="api_app.analyzers_manager.observable_analyzers",
+    )
+    pm_form_compiler = PythonModule.objects.get(
+        module="phishing.phishing_form_compiler.PhishingFormCompiler",
+        base_path="api_app.analyzers_manager.file_analyzers",
+    )
+
+    p_extractor = Parameter.objects.get(
+        name="user_agent",
+        type="str",
+        description="Custom user agent for the Phishing Extractor Selenium browser.",
+        is_secret=False,
+        required=False,
+        python_module=pm_extractor,
+    )
+    p_form_compiler = Parameter.objects.get(
+        name="user_agent",
+        type="str",
+        description="Custom user agent for the compilation of form",
+        is_secret=False,
+        required=False,
+        python_module=pm_form_compiler,
+    )
+
+    for config in pm_extractor.analyzerconfigs.all():
+        PluginConfig.objects.create(
+            parameter=p_extractor,
+            analyzer_config=config,
+        )
+
+    for config in pm_form_compiler.analyzerconfigs.all():
+        PluginConfig.objects.create(
+            parameter=p_form_compiler,
+            analyzer_config=config,
+        )
+
+    p_extractor.delete()
+    p_form_compiler.delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0062_alter_parameter_python_module"),
+        (
+            "analyzers_manager",
+            "0142_alter_analyzerreport_data_model_content_type_and_more",
+        ),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

api_app/analyzers_manager/observable_analyzers/phishing/phishing_extractor.py

@@ -17,6 +17,7 @@ class PhishingExtractor(ObservableAnalyzer, DockerBasedAnalyzer):
     proxy_address: str = ""
     window_width: int
     window_height: int
+    user_agent: str = ""
 
     def __init__(
         self,
@@ -40,6 +41,8 @@ def config(self, runtime_configuration: Dict):
             self.args.append(f"--window_width={self.window_width}")
         if self.window_height:
             self.args.append(f"--window_height={self.window_height}")
+        if self.user_agent:
+            self.args.append(f"--user_agent={self.user_agent}")
 
     def run(self):
         req_data: {} = {

api_app/playbooks_manager/migrations/0057_alter_phishing_extractor_add_domain.py

@@ -0,0 +1,32 @@
+from django.db import migrations
+
+from api_app.analyzers_manager.constants import ObservableTypes
+
+
+def migrate(apps, schema_editor):
+    PlaybookConfig = apps.get_model("playbooks_manager", "PlaybookConfig")
+    config = PlaybookConfig.objects.get(name="PhishingExtractor")
+    config.type = [
+        ObservableTypes.URL,
+        ObservableTypes.DOMAIN,
+    ]
+    config.full_clean()
+    config.save()
+
+
+def reverse_migrate(apps, schema_editor):
+    PlaybookConfig = apps.get_model("playbooks_manager", "PlaybookConfig")
+    config = PlaybookConfig.objects.get(name="PhishingExtractor")
+    config.type = [
+        ObservableTypes.URL,
+    ]
+    config.full_clean()
+    config.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playbooks_manager", "0056_download_sample_vt"),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

integrations/phishing_analyzers/analyzers/driver_wrapper.py

@@ -1,13 +1,15 @@
 import functools
 import logging
 import os
+from random import randint
 from typing import Iterator
 
 from selenium.common import WebDriverException
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.wait import WebDriverWait
 from seleniumwire.request import Request
+from seleniumwire.thirdparty.mitmproxy.exceptions import ServerException
 from seleniumwire.webdriver import ChromeOptions, Remote
 
 LOG_NAME = "driver_wrapper"
@@ -39,7 +41,7 @@ def handle_exception(self, *args, **kwargs):
         try:
             return func(self, *args, **kwargs)
         except WebDriverException as e:
-            logger.error(
+            logger.exception(
                 f"Error while performing {func.__name__}"
                 f"{' for url=' + url if func.__name__ == 'navigate' else ''}: {e}"
             )
@@ -56,23 +58,61 @@ def __init__(
         proxy_address: str = "",
         window_width: int = 1920,
         window_height: int = 1080,
+        user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3",
     ):
         self.proxy: str = proxy_address
         self.window_width: int = window_width
         self.window_height: int = window_height
+        self.user_agent: str = user_agent
         self.last_url: str = ""
-        self._driver: Remote = self._init_driver(self.window_width, self.window_height)
+        self.base_port = 17000
+        self.port_pool_size = 100
+        self._driver: Remote = self._init_driver(
+            self.window_width, self.window_height, self.user_agent
+        )
 
-    def _init_driver(self, window_width: int, window_height: int) -> Remote:
+    def _pick_free_port_from_pool(
+        self, sw_options: {}, options: ChromeOptions
+    ) -> Remote:
+        tries: int = 0
+        while tries < self.port_pool_size:
+            picked_port = randint(self.base_port, self.base_port + self.port_pool_size)
+            sw_options.update({"port": picked_port})
+
+            # traffic must go back to host running selenium-wire
+            options.add_argument(
+                f"--proxy-server=http://phishing_analyzers:{picked_port}"
+            )
+            try:
+                driver = Remote(
+                    command_executor="http://selenium-hub:4444/wd/hub",
+                    options=options,
+                    seleniumwire_options=sw_options,
+                )
+            except ServerException:
+                logger.info(
+                    f"Failed to create driver with {picked_port=}. Trying with another one..."
+                )
+                tries += 1
+            else:
+                logger.info(f"Found free port {picked_port}. Creating driver...")
+                return driver
+        raise RuntimeError(
+            "Failed to retrieve a free port for MitM proxy! Try restarting the job"
+        )
+
+    def _init_driver(
+        self, window_width: int, window_height: int, user_agent: str
+    ) -> Remote:
         logger.info(f"Adding proxy with option: {self.proxy}")
         logger.info("Creating Chrome driver...")
         sw_options: {} = {
             "auto_config": False,  # Ensure this is set to False
             "enable_har": True,
             # https://github.com/wkeeling/selenium-wire/issues/220#issuecomment-794308386
             # config to have local seleniumwire proxy compatible with another proxy
-            "addr": "0.0.0.0",  # nosec B104 # where selenium-wire proxy will run
-            "port": 7007,
+            "addr": "0.0.0.0",  # where selenium-wire proxy will run
+            "port": 0,
         }
         if self.proxy:
             sw_options["proxy"] = {"http": self.proxy, "https": self.proxy}
@@ -85,23 +125,22 @@ def _init_driver(self, window_width: int, window_height: int) -> Remote:
         options.add_argument("--headless=new")
         options.add_argument("--ignore-certificate-errors")
         options.add_argument(f"--window-size={window_width},{window_height}")
-        # traffic must go back to host running selenium-wire
-        options.add_argument("--proxy-server=http://phishing_analyzers:7007")
-        driver = Remote(
-            command_executor="http://selenium-hub:4444/wd/hub",
-            options=options,
-            seleniumwire_options=sw_options,
-        )
-        return driver
+        options.add_argument(f"--user-agent={user_agent}")
+
+        return self._pick_free_port_from_pool(sw_options, options)
 
     def restart(self, motivation: str = "", timeout_wait_page: int = 0):
-        logger.info(f"Restarting driver: {motivation=}")
+        logger.info(f"{self._driver.session_id}: Restarting driver: {motivation=}")
         self._driver.quit()
         self._driver = self._init_driver(
-            window_width=self.window_width, window_height=self.window_height
+            window_width=self.window_width,
+            window_height=self.window_height,
+            user_agent=self.user_agent,
         )
         if self.last_url:
-            logger.info(f"Navigating to {self.last_url} after driver has restarted")
+            logger.info(
+                f"{self._driver.session_id}: Navigating to {self.last_url} after driver has restarted"
+            )
             self.navigate(self.last_url, timeout_wait_page=timeout_wait_page)
 
     @driver_exception_handler
@@ -111,7 +150,7 @@ def navigate(self, url: str = "", timeout_wait_page: int = 0):
             return
 
         self.last_url = url
-        logger.info(f"Navigating to {url=}")
+        logger.info(f"{self._driver.session_id}: Navigating to {url=}")
         self._driver.get(url)
         # dinamically wait for page to load its content with a fallback
         # of `timeout_wait_page` seconds.
@@ -123,17 +162,21 @@ def navigate(self, url: str = "", timeout_wait_page: int = 0):
 
     @driver_exception_handler
     def get_page_source(self) -> str:
-        logger.info(f"Extracting page source for url {self.last_url}")
+        logger.info(
+            f"{self._driver.session_id}: Extracting page source for url {self.last_url}"
+        )
         return self._driver.page_source
 
     @driver_exception_handler
     def get_current_url(self) -> str:
-        logger.info("Extracting current URL of page")
+        logger.info(f"{self._driver.session_id}: Extracting current URL of page")
         return self._driver.current_url
 
     @driver_exception_handler
     def get_base64_screenshot(self) -> str:
-        logger.info(f"Extracting screenshot of page as base64 for url {self.last_url}")
+        logger.info(
+            f"{self._driver.session_id}: Extracting screenshot of page as base64 for url {self.last_url}"
+        )
         return self._driver.get_screenshot_as_base64()
 
     def iter_requests(self) -> Iterator[Request]:
@@ -142,5 +185,10 @@ def iter_requests(self) -> Iterator[Request]:
     def get_har(self) -> str:
         return self._driver.har
 
+    def close(self):
+        logger.info(f"{self._driver.session_id}: Closing")
+        self._driver.close()
+
     def quit(self):
+        logger.info(f"{self._driver.session_id}: Quitting")
         self._driver.quit()