Fix the issue wapiti-scanner#559

Fixing the errors output

tests/attack/test_mod_wapp.py

@@ -1,12 +1,14 @@
 import os
 from asyncio import Event
-from unittest.mock import AsyncMock, patch, MagicMock
+from unittest.mock import AsyncMock, patch, MagicMock, PropertyMock, Mock
 
 import httpx
+from httpx import RequestError
 import pytest
 import respx
 
 from wapitiCore.attack.mod_wapp import ModuleWapp
+from wapitiCore.controller.wapiti import InvalidOptionValue
 from wapitiCore.net.crawler import AsyncCrawler
 from wapitiCore.net.classes import CrawlerConfiguration
 from wapitiCore.net import Request
@@ -589,10 +591,45 @@ async def test_merge_with_and_without_redirection():
 
         assert sorted(results) == sorted(expected_results)
 
+@pytest.mark.asyncio
+@respx.mock
+async def test_exception_none_valid_db_url():
+    respx.get("http://perdu.com/").mock(
+        return_value=httpx.Response(
+            200,
+            content="Hello")
+    )
+    respx.get(url__regex=r"http://perdu.com/.*").mock(
+        return_value=httpx.Response(
+            404
+        )
+    )
+
+    cat_url = "http://perdu.com/src/categories.json"
+    group_url = "http://perdu.com/src/groups.json"
+    tech_url = "http://perdu.com/src/technologies/"
+    persister = AsyncMock()
+    crawler = AsyncMock()
+    crawler_configuration = CrawlerConfiguration(Request("http://perdu.com/"))
+    options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com"}
+    with patch.object(
+            ModuleWapp,
+            "_load_wapp_database",
+            return_value=ValueError
+    ) as mock_load_wapp_database:
+        module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
+
+        try:
+            await module._load_wapp_database(cat_url, tech_url, group_url)
+        except (IOError, ValueError):
+            pytest.fail("Unexpected IOError ..")
+        assert mock_load_wapp_database.assert_called_once
 
 @pytest.mark.asyncio
 @respx.mock
 async def test_exception_json():
+    """Tests that a ValueError is raised when calling _dump_url_content_to_file with invalid technologies.json file."""
+
     json_string = {
     "1C-Bitrix": {
         "cats": [1, 6],
@@ -628,17 +665,218 @@ async def test_exception_json():
             content="Test''''")
     )
 
-    request = Request("http://perdu.com/")
+    url = "http://perdu.com/"
+    persister = AsyncMock()
+    crawler = AsyncMock()
+    crawler_configuration = CrawlerConfiguration(Request("http://perdu.com/"))
+    options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com"}
+    with patch.object(
+            ModuleWapp,
+            "_dump_url_content_to_file",
+            return_value=ValueError
+    ) as mock_dump_url_content_to_file:
+        module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
+
+        try:
+            await module._dump_url_content_to_file(url, "tech.json")
+        except (IOError, ValueError):
+            pytest.fail("Unexpected IOError ..")
+        assert mock_dump_url_content_to_file.assert_called_once
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_raise_on_invalid_json():
+    """Tests that a ValueError is raised when calling _dump_url_content_to_file with invalid or empty Json."""
+
+    respx.get("http://perdu.com/src/categories.json").mock(
+        return_value=httpx.Response(
+            200,
+            content="Test")
+    )
+
+    persister = AsyncMock()
+    crawler_configuration = CrawlerConfiguration(Request("http://perdu.com/"))
+    async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
+        options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com"}
+        module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
+
+        with pytest.raises(ValueError) as exc_info:
+            await module._dump_url_content_to_file("http://perdu.com/src/categories.json", "test.json")
+
+        assert exc_info.value.args[0] == "Invalid or empty JSON response for http://perdu.com/src/categories.json"
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_raise_on_not_valid_db_url():
+    """Tests that a ValueError is raised when the URL doesn't contain a Wapp DB."""
+    cat_url = "http://perdu.com/src/categories.json"
+    group_url = "http://perdu.com/src/groups.json"
+    tech_url = "http://perdu.com/src/technologies/"
+
+    respx.get(url__regex=r"http://perdu.com/.*").mock(
+        return_value=httpx.Response(
+            404,
+            content="Not Found")
+    )
     persister = AsyncMock()
     crawler_configuration = CrawlerConfiguration(Request("http://perdu.com/"))
     async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
         options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com/"}
 
         module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
 
-        with patch("builtins.open", MagicMock(side_effect=IOError)) as open_mock:
-            try:
-                await module.attack(request)
-                pytest.fail("Should raise an exception ..")
-            except (IOError, ValueError):
-                open_mock.assert_called_with(open_mock.mock_calls[0][1][0], 'r', encoding='utf-8')
+        with pytest.raises(ValueError) as exc_info:
+            await module._load_wapp_database(cat_url, tech_url, group_url)
+
+        assert exc_info.value.args[0] == "http://perdu.com/src/technologies/ is not a valid URL for a wapp database"
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_raise_on_value_error():
+    """Tests that a ValueError is raised when calling the _load_wapp_database function when the json is not valid."""
+
+    example_json_content = {
+        "2B Advice": {
+            "cats": [67],
+            "description": "2B Advice provides a plug-in to manage GDPR cookie consent.",
+            "icon": "2badvice.png",
+            "js": {
+                "BBCookieControler": ""
+            },
+            "saas": True,
+            "scriptSrc": "2badvice-cdn\\.azureedge\\.net",
+            "website": "https://www.2b-advice.com/en/data-privacy-software/cookie-consent-plugin/"
+        },
+        "30namaPlayer": {
+            "cats": [14],
+            "description": "30namaPlayer is a modified version of Video.",
+            "dom": "section[class*='player30nama']",
+            "icon": "30namaPlayer.png",
+            "website": "https://30nama.com/"
+        }}
+
+    cat_url = "http://perdu.com/src/categories.json"
+    group_url = "http://perdu.com/src/groups.json"
+    tech_url = "http://perdu.com/src/technologies/"
+
+    respx.get(url__regex=r"http://perdu.com/src/technologies/.*").mock(
+        return_value=httpx.Response(
+            200,
+            content=str(example_json_content))
+    )
+    respx.get(url__regex=r"http://perdu.com/.*").mock(
+        return_value=httpx.Response(
+            200,
+            content="No Json")
+    )
+    persister = AsyncMock()
+    crawler_configuration = CrawlerConfiguration(Request("http://perdu.com/"))
+    async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
+        options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com/"}
+
+        module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
+
+        with pytest.raises(ValueError) as exc_info:
+            await module._load_wapp_database(cat_url, tech_url, group_url)
+
+        assert exc_info.value.args[0] == "Invalid or empty JSON response for http://perdu.com/src/categories.json"
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_raise_on_request_error():
+    """Tests that a RequestError is raised when calling the _load_wapp_database function with wrong URL."""
+
+    cat_url = "http://perdu.com/src/categories.json"
+    group_url = "http://perdu.com/src/groups.json"
+    tech_url = "http://perdu.com/src/technologies/"
+
+    respx.get(url__regex=r"http://perdu.com/.*").mock(side_effect=RequestError("RequestError occurred: [Errno -2] Name or service not known"))
+    persister = AsyncMock()
+    crawler_configuration = CrawlerConfiguration(Request("http://perdu.com/"))
+    async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
+        options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com/"}
+
+        module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
+
+        with pytest.raises(RequestError) as exc_info:
+            await module._load_wapp_database(cat_url, tech_url, group_url)
+
+        assert exc_info.value.args[0] == "RequestError occurred: [Errno -2] Name or service not known"
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_raise_on_request_error_for_dump_url():
+    """Tests that a RequestError is raised when calling the _dump_url_content_to_file function with wrong URL."""
+
+    url = "http://perdu.com/"
+    group_url = "http://perdu.com/src/groups.json"
+    tech_url = "http://perdu.com/src/technologies/"
+
+    respx.get(url__regex=r"http://perdu.com/.*").mock(side_effect=RequestError("RequestError occurred: [Errno -2] Name or service not known"))
+    persister = AsyncMock()
+    crawler_configuration = CrawlerConfiguration(Request("http://perduu.com/"))
+    async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
+        options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com/"}
+
+        module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
+
+        with pytest.raises(RequestError) as exc_info:
+            await module._dump_url_content_to_file(url, "cat.json")
+
+        assert exc_info.value.args[0] == "RequestError occurred: [Errno -2] Name or service not known"
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_raise_on_request_error_for_update():
+    """Tests that a RequestError is raised when calling the _dump_url_content_to_file function with wrong URL."""
+
+    url = "http://perdu.com/"
+    group_url = "http://perdu.com/src/groups.json"
+    tech_url = "http://perdu.com/src/technologies/"
+
+    respx.get(url__regex=r"http://perdu.com/.*").mock(side_effect=RequestError("RequestError occurred: [Errno -2] Name or service not known"))
+    persister = AsyncMock()
+    crawler_configuration = CrawlerConfiguration(Request("http://perduu.com/"))
+    async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
+        options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com/"}
+
+        module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
+
+        with pytest.raises(RequestError) as exc_info:
+            await module.update()
+
+        assert exc_info.value.args[0] == "RequestError occurred: [Errno -2] Name or service not known"
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_raise_on_value_error_for_update():
+    """Tests that a RequestError is raised when calling the _dump_url_content_to_file function with wrong URL."""
+
+    respx.get(url__regex=r"http://perdu.com/src/technologies/.*").mock(
+        return_value=httpx.Response(
+            200,
+            content=str("{}"))
+    )
+    respx.get(url__regex=r"http://perdu.com/.*").mock(
+        return_value=httpx.Response(
+            200,
+            content="No Json")
+    )
+
+    persister = AsyncMock()
+    crawler_configuration = CrawlerConfiguration(Request("http://perduu.com/"))
+    async with AsyncCrawler.with_configuration(crawler_configuration) as crawler:
+        options = {"timeout": 10, "level": 2, "wapp_url": "http://perdu.com/"}
+
+        module = ModuleWapp(crawler, persister, options, Event(), crawler_configuration)
+
+        with pytest.raises(ValueError) as exc_info:
+            await module.update()
+
+        assert exc_info.value.args[0] == "Invalid or empty JSON response for http://perdu.com/src/categories.json"

tests/cli/test_options.py

@@ -175,6 +175,16 @@ async def test_update_with_modules(mock_update):
             mock_update.assert_called_once_with("wapp,nikto")
 
 
+@pytest.mark.asyncio
+@mock.patch("wapitiCore.main.wapiti.is_valid_url")
+async def test_update_with_not_valid_url(mock_valid_url):
+    testargs = ["wapiti", "--update", "-m", "wapp", "--wapp-url", "htp:/perdu"]
+    with mock.patch.object(sys, 'argv', testargs):
+        with pytest.raises(SystemExit) as ve:
+            await wapiti_main()
+            mock_valid_url.assert_called_once_with("htp:/perdu")
+
+
 @pytest.mark.asyncio
 @mock.patch("wapitiCore.main.wapiti.Wapiti.update")
 async def test_update_without_modules(mock_update):

wapitiCore/attack/mod_wapp.py

@@ -29,7 +29,9 @@
 from arsenic.errors import JavascriptError, UnknownError, ArsenicError
 
 from wapitiCore.main.log import logging, log_blue
+from wapitiCore.main.wapiti import is_valid_url
 from wapitiCore.attack.attack import Attack
+from wapitiCore.controller.wapiti import InvalidOptionValue
 from wapitiCore.net.response import Response
 from wapitiCore.wappalyzer.wappalyzer import Wappalyzer, ApplicationData, ApplicationDataException
 from wapitiCore.definitions.fingerprint import NAME as TECHNO_DETECTED, WSTG_CODE as TECHNO_DETECTED_WSTG_CODE
@@ -110,15 +112,24 @@ async def update(self):
         wapp_categories_url = f"{self.BASE_URL}src/categories.json"
         wapp_technologies_base_url = f"{self.BASE_URL}src/technologies/"
         wapp_groups_url = f"{self.BASE_URL}src/groups.json"
-
+        if not is_valid_url(self.BASE_URL):
+            raise InvalidOptionValue(
+                "--wapp-url", self.BASE_URL
+            )
         try:
             await self._load_wapp_database(
                 wapp_categories_url,
                 wapp_technologies_base_url,
                 wapp_groups_url
             )
+        except RequestError as e:
+            logging.error(f"RequestError occurred: {e}")
+            raise
         except IOError:
             logging.error("Error downloading wapp database.")
+        except ValueError as e:
+            logging.error(f"Value error: {e}")
+            raise
 
     async def must_attack(self, request: Request, response: Optional[Response] = None):
         if self.finished:
@@ -136,7 +147,10 @@ async def attack(self, request: Request, response: Optional[Response] = None):
         groups_file_path = os.path.join(self.user_config_dir, self.WAPP_GROUPS)
         technologies_file_path = os.path.join(self.user_config_dir, self.WAPP_TECHNOLOGIES)
 
-        await self._verify_wapp_database(categories_file_path, technologies_file_path, groups_file_path)
+        try:
+            await self._verify_wapp_database(categories_file_path, technologies_file_path, groups_file_path)
+        except ValueError:
+            return
 
         try:
             application_data = ApplicationData(categories_file_path, groups_file_path, technologies_file_path)
@@ -226,6 +240,8 @@ async def _dump_url_content_to_file(self, url: str, file_path: str):
             response = await self.crawler.async_send(request)
         except RequestError:
             self.network_errors += 1
+            raise
+        if response.status != 200:
             logging.error(f"Error: Non-200 status code for {url}")
             return
         if not _is_valid_json(response):
@@ -247,19 +263,23 @@ async def _load_wapp_database(self, categories_url: str, technologies_base_url:
                 response: Response = await self.crawler.async_send(request)
             except RequestError:
                 self.network_errors += 1
-                logging.error(f"Error: Non-200 status code for {technology_file_name}. Skipping.")
-                return
-                # Merging all technologies in one object
+                raise
+            if response.status != 200:
+                raise ValueError(f"{technologies_base_url} is not a valid URL for a wapp database")
+            # Merging all technologies in one object
             for technology_name in response.json:
                 technologies[technology_name] = response.json[technology_name]
             try:
             # Saving categories & groups
                 await asyncio.gather(
                     self._dump_url_content_to_file(categories_url, categories_file_path),
                     self._dump_url_content_to_file(groups_url, groups_file_path))
-            except ValueError:
-                logging.error(f"Invalid or empty JSON response for {categories_url} or {groups_url}")
-                return
+            except RequestError as req_error:
+                logging.error(f"Caught a ValueError: {req_error}")
+                raise
+            except ValueError as ve:
+                logging.error(f"Caught a ValueError: {ve}")
+                raise
             # Saving technologies
             with open(technologies_file_path, 'w', encoding='utf-8') as file:
                 json.dump(technologies, file)