From 84583b3b631bf9873d1f747ce1f21b8a7796b9e8 Mon Sep 17 00:00:00 2001 From: Colin Stubbs <3059577+colin-stubbs@users.noreply.github.com> Date: Sun, 4 Aug 2024 11:30:00 +1000 Subject: [PATCH 001/254] initial commit for securitytxt module --- bbot/modules/securitytxt.py | 134 ++++++++++++++++++ .../module_tests/test_module_securitytxt.py | 52 +++++++ 2 files changed, 186 insertions(+) create mode 100644 bbot/modules/securitytxt.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_securitytxt.py diff --git a/bbot/modules/securitytxt.py b/bbot/modules/securitytxt.py new file mode 100644 index 000000000..640956a23 --- /dev/null +++ b/bbot/modules/securitytxt.py @@ -0,0 +1,134 @@ +# securitytxt.py +# +# Checks for/parses https://target.domain/.well-known/security.txt +# +# Refer to: https://securitytxt.org/ +# +# security.txt may contain email addresses and URL's, and possibly IP addresses. +# +# Example security.txt: +# +# Contact: mailto:security.reports@example.com +# Expires: 2028-05-31T14:00:00.000Z +# Encryption: https://example.com/security.pgp +# Preferred-Languages: en, es +# Canonical: https://example.com/.well-known/security.txt +# Canonical: https://www.example.com/.well-known/security.txt +# Policy: https://example.com/security-policy.html +# Hiring: https://example.com/jobs.html +# +# Example security.txt with PGP signature: +# +# -----BEGIN PGP SIGNED MESSAGE----- +# Hash: SHA512 +# +# Contact: https://vdp.example.com +# Expires: 2025-01-01T00:00:00.000Z +# Preferred-Languages: fr, en +# Canonical: https://example.com/.well-known/security.txt +# Policy: https://example.com/cert +# Hiring: https://www.careers.example.com +# -----BEGIN PGP SIGNATURE----- +# +# iQIzBAEBCgAdFiEELC1a63jHPhyV60KPsvWy9dDkrigFAmJBypcACgkQsvWy9dDk +# rijXHQ//Qya3hUSy5PYW+fI3eFP1+ak6gYq3Cbzkf57cqiBhxGetIGIGNJ6mxgjS +# KAuvXLMUWgZD73r//fjZ5v1lpuWmpt54+ecat4DgcVCvFKYpaH+KBlay8SX7XtQH +# 9T2NXMcez353TMR3EUOdLwdBzGZprf0Ekg9EzaHKMk0k+A4D9CnSb8Y6BKDPC7wr +# eadwDIR9ESo0va4sjjcllCG9MF5hqK25SfsKriCSEAMhse2FToEBbw8ImkPKowMN +# whJ4MIVlBxybu6XoIyk3n7HRRduijywy7uV80pAkhk/hL6wiW3M956FiahfRI6ad +# +Gky/Ri5TjwAE/x5DhUH8O2toPsn71DeIE4geKfz5d/v41K0yncdrHjzbj0CAHu3 +# wVWLKnEp8RVqTlOR8jU0HqQUQy8iZk4LY91ROv+QjG/jUTWlwun8Ljh+YUeJTMRp +# MGftCdCrrYjIy5aEQqWztt+dXKac/9e1plq3yyfuW1L+wG3zS7X+NpIJgygMvEwT +# L3dqfQf63sjk8kWIZMVnicHBlc6BiLqUn020l+pkIOr4MuuJmIlByhlnfqH7YM8k +# VShwDx7rs4Hj08C7NVCYIySaM2jM4eNKGt9V5k1F1sklCVfYaT8OqOhJrzhcisOC +# YcQDhjt/iZTR8SzrHO7kFZbaskIp2P7JMaPax2fov15AnNHQQq8= +# =8vfR +# -----END PGP SIGNATURE----- + +from bbot.modules.base import BaseModule + +import re + +from bbot.core.helpers.regexes import email_regex, url_regexes + +_securitytxt_regex = r"^(?P\w+): *(?P.*)$" +securitytxt_regex = re.compile(_securitytxt_regex, re.I | re.M) + + +class securitytxt(BaseModule): + watched_events = ["DNS_NAME"] + produced_events = ["EMAIL_ADDRESS", "URL_UNVERIFIED"] + flags = ["subdomain-enum", "cloud-enum", "active", "web-basic", "safe"] + meta = { + "description": "Check for security.txt content", + "author": "@colin-stubbs", + "created_date": "2024-05-26", + } + options = { + "in_scope_only": True, + "emails": True, + "urls": True, + } + options_desc = { + "in_scope_only": "Only emit events related to in-scope domains", + "emails": "emit EMAIL_ADDRESS events", + "urls": "emit URL_UNVERIFIED events", + } + # accept DNS_NAMEs out to 2 hops if in_scope_only is False + scope_distance_modifier = 2 + + async def setup(self): + self.in_scope_only = self.config.get("in_scope_only", True) + self._emails = self.config.get("emails", True) + self._urls = self.config.get("urls", True) + return await super().setup() + + def _incoming_dedup_hash(self, event): + # dedupe by parent + parent_domain = self.helpers.parent_domain(event.data) + return hash(parent_domain), "already processed parent domain" + + async def filter_event(self, event): + if "_wildcard" in str(event.host).split("."): + return False, "event is wildcard" + + # scope filtering + if event.scope_distance > 0 and self.in_scope_only: + return False, "event is not in scope" + + return True + + async def handle_event(self, event): + tags = ["securitytxt-policy"] + url = f"https://{event.host}/.well-known/security.txt" + + r = await self.helpers.request(url, method="GET") + + if r is None or r.status_code != 200: + # it doesn't look like we got a valid response... + return + + s = r.text + + # avoid parsing the response unless it looks, at a very basic level, like an actual security.txt + if "contact: " in s.lower() or "expires: " in s.lower(): + for securitytxt_match in securitytxt_regex.finditer(s): + v = securitytxt_match.group("v") + + for match in email_regex.finditer(v): + start, end = match.span() + email = v[start:end] + + if self._emails: + await self.emit_event(email, "EMAIL_ADDRESS", parent=event, tags=tags) + + for url_regex in url_regexes: + for match in url_regex.finditer(v): + start, end = match.span() + found_url = v[start:end] + + if found_url != url and self._urls == True: + await self.emit_event(found_url, "URL_UNVERIFIED", parent=event, tags=tags) + + +# EOF diff --git a/bbot/test/test_step_2/module_tests/test_module_securitytxt.py b/bbot/test/test_step_2/module_tests/test_module_securitytxt.py new file mode 100644 index 000000000..3d64a4eb6 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_securitytxt.py @@ -0,0 +1,52 @@ +from .base import ModuleTestBase + + +class TestSecurityTxt(ModuleTestBase): + targets = ["blacklanternsecurity.notreal"] + modules_overrides = ["securitytxt", "speculate"] + + async def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://blacklanternsecurity.notreal/.well-known/security.txt", + text="-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\nContact: mailto:joe.smith@blacklanternsecurity.notreal\nContact: mailto:vdp@example.com\nContact: https://vdp.example.com\nExpires: 2025-01-01T00:00:00.000Z\nPreferred-Languages: fr, en\nCanonical: https://blacklanternsecurity.notreal/.well-known/security.txt\nPolicy: https://example.com/cert\nHiring: https://www.careers.example.com\n-----BEGIN PGP SIGNATURE-----\n\nSIGNATURE\n\n-----END PGP SIGNATURE-----", + ) + + async def setup_after_prep(self, module_test): + await module_test.mock_dns( + { + "blacklanternsecurity.notreal": { + "A": ["127.0.0.11"], + }, + } + ) + + def check(self, module_test, events): + assert any( + e.type == "EMAIL_ADDRESS" and e.data == "joe.smith@blacklanternsecurity.notreal" for e in events + ), "Failed to detect email address" + assert not any( + e.type == "URL_UNVERIFIED" and e.data == "https://blacklanternsecurity.notreal/.well-known/security.txt" + for e in events + ), "Failed to filter Canonical URL to self" + assert not any(str(e.data) == "vdp@example.com" for e in events) + + +class TestSecurityTxtInScopeFalse(TestSecurityTxt): + config_overrides = { + "scope": {"report_distance": 1}, + "modules": {"securitytxt": {"in_scope_only": False}}, + } + + def check(self, module_test, events): + assert any( + e.type == "EMAIL_ADDRESS" and e.data == "vdp@example.com" for e in events + ), "Failed to detect email address" + assert any( + e.type == "URL_UNVERIFIED" and e.data == "https://vdp.example.com/" for e in events + ), "Failed to detect URL" + assert any( + e.type == "URL_UNVERIFIED" and e.data == "https://example.com/cert" for e in events + ), "Failed to detect URL" + assert any( + e.type == "URL_UNVERIFIED" and e.data == "https://www.careers.example.com/" for e in events + ), "Failed to detect URL" From 12a42877ba3a8130340f427502dbfbfe001c333b Mon Sep 17 00:00:00 2001 From: Colin Stubbs <3059577+colin-stubbs@users.noreply.github.com> Date: Mon, 5 Aug 2024 04:59:05 +1000 Subject: [PATCH 002/254] Remove unnecessary distance modifier adjustment --- bbot/modules/securitytxt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/modules/securitytxt.py b/bbot/modules/securitytxt.py index 640956a23..a84e14149 100644 --- a/bbot/modules/securitytxt.py +++ b/bbot/modules/securitytxt.py @@ -74,8 +74,6 @@ class securitytxt(BaseModule): "emails": "emit EMAIL_ADDRESS events", "urls": "emit URL_UNVERIFIED events", } - # accept DNS_NAMEs out to 2 hops if in_scope_only is False - scope_distance_modifier = 2 async def setup(self): self.in_scope_only = self.config.get("in_scope_only", True) From ab70dbd4d4733fb484bf1a671c72cba5abba580a Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 11 Aug 2024 17:34:48 -0400 Subject: [PATCH 003/254] small error-handling / speed improvement --- bbot/modules/securitytxt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bbot/modules/securitytxt.py b/bbot/modules/securitytxt.py index a84e14149..31dd0436a 100644 --- a/bbot/modules/securitytxt.py +++ b/bbot/modules/securitytxt.py @@ -106,10 +106,14 @@ async def handle_event(self, event): # it doesn't look like we got a valid response... return - s = r.text + try: + s = r.text + except Exception: + s = "" # avoid parsing the response unless it looks, at a very basic level, like an actual security.txt - if "contact: " in s.lower() or "expires: " in s.lower(): + s_lower = s.lower() + if "contact: " in s_lower or "expires: " in s_lower: for securitytxt_match in securitytxt_regex.finditer(s): v = securitytxt_match.group("v") From 3a4b7ce8dd02a0b0bb7abef4d561f46520f242a1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 16 Aug 2024 23:58:05 -0400 Subject: [PATCH 004/254] separate discovery_context <-> parent_chain --- bbot/core/event/base.py | 21 ++++++-- bbot/test/test_step_1/test_events.py | 25 ++++----- .../module_tests/test_module_json.py | 3 +- docs/scanning/events.md | 51 +++++++++---------- 4 files changed, 57 insertions(+), 43 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 89b5a84a3..d098213a0 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -388,10 +388,20 @@ def discovery_path(self): """ This event's full discovery context, including those of all its parents """ - parent_path = [] + discovery_path = [] if self.parent is not None and self.parent is not self: - parent_path = self.parent.discovery_path - return parent_path + [[self.id, self.discovery_context]] + discovery_path = self.parent.discovery_path + return discovery_path + [self.discovery_context] + + @property + def parent_chain(self): + """ + This event's full discovery context, including those of all its parents + """ + parent_chain = [] + if self.parent is not None and self.parent is not self: + parent_chain = self.parent.parent_chain + return parent_chain + [self.id] @property def words(self): @@ -775,6 +785,7 @@ def json(self, mode="json", siem_friendly=False): # discovery context j["discovery_context"] = self.discovery_context j["discovery_path"] = self.discovery_path + j["parent_chain"] = self.parent_chain # normalize non-primitive python objects for k, v in list(j.items()): @@ -914,6 +925,10 @@ def _data_human(self): def discovery_path(self): return [] + @property + def parent_chain(self): + return [] + class FINISHED(BaseEvent): """ diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index e9d1edeaf..1cde37c3a 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -409,7 +409,8 @@ async def test_events(events, helpers): db_event._resolved_hosts = {"127.0.0.1"} db_event.scope_distance = 1 assert db_event.discovery_context == "test context" - assert db_event.discovery_path == [["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]] + assert db_event.discovery_path == ["test context"] + assert db_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"] timestamp = db_event.timestamp.isoformat() json_event = db_event.json() assert json_event["scope_distance"] == 1 @@ -418,7 +419,8 @@ async def test_events(events, helpers): assert json_event["host"] == "evilcorp.com" assert json_event["timestamp"] == timestamp assert json_event["discovery_context"] == "test context" - assert json_event["discovery_path"] == [["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"]] + assert json_event["discovery_path"] == ["test context"] + assert json_event["parent_chain"] == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"] reconstituted_event = event_from_json(json_event) assert reconstituted_event.scope_distance == 1 assert reconstituted_event.timestamp.isoformat() == timestamp @@ -426,9 +428,8 @@ async def test_events(events, helpers): assert reconstituted_event.type == "OPEN_TCP_PORT" assert reconstituted_event.host == "evilcorp.com" assert reconstituted_event.discovery_context == "test context" - assert reconstituted_event.discovery_path == [ - ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac", "test context"] - ] + assert reconstituted_event.discovery_path == ["test context"] + assert reconstituted_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"] assert "127.0.0.1" in reconstituted_event.resolved_hosts hostless_event = scan.make_event("asdf", "ASDF", dummy=True) hostless_event_json = hostless_event.json() @@ -614,7 +615,7 @@ async def handle_event(self, event): if e.type == "DNS_NAME" and e.data == "evilcorp.com" and e.discovery_context == f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com" - and [_[-1] for _ in e.discovery_path] == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"] + and e.discovery_path == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"] ] ) assert 1 == len( @@ -624,7 +625,7 @@ async def handle_event(self, event): if e.type == "DNS_NAME" and e.data == "one.evilcorp.com" and e.discovery_context == "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com" - and [_[-1] for _ in e.discovery_path] + and e.discovery_path == [ f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com", "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com", @@ -639,7 +640,7 @@ async def handle_event(self, event): and e.data == "two.evilcorp.com" and e.discovery_context == "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com" - and [_[-1] for _ in e.discovery_path] + and e.discovery_path == [ f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com", "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com", @@ -654,7 +655,7 @@ async def handle_event(self, event): if e.type == "DNS_NAME" and e.data == "three.evilcorp.com" and e.discovery_context == "module_2 asked nicely and was given DNS_NAME three.evilcorp.com" - and [_[-1] for _ in e.discovery_path] + and e.discovery_path == [ f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com", "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com", @@ -676,11 +677,11 @@ async def handle_event(self, event): if e.type == "DNS_NAME" and e.data == "four.evilcorp.com" and e.discovery_context == "module_2 used brute force to obtain DNS_NAME four.evilcorp.com" - and [_[-1] for _ in e.discovery_path] == final_path + and e.discovery_path == final_path ] assert 1 == len(final_event) j = final_event[0].json() - assert [_[-1] for _ in j["discovery_path"]] == final_path + assert j["discovery_path"] == final_path await scan._cleanup() @@ -693,7 +694,7 @@ async def handle_event(self, event): events = [e async for e in scan.async_start()] blsops_event = [e for e in events if e.type == "DNS_NAME" and e.data == "blsops.com"] assert len(blsops_event) == 1 - assert blsops_event[0].discovery_path[1][-1] == "URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com" + assert blsops_event[0].discovery_path[1] == "URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com" await scan._cleanup() diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 6ccf4d847..7c5d163e0 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -38,8 +38,9 @@ def check(self, module_test, events): assert dns_reconstructed.data == dns_data assert dns_reconstructed.discovery_context == context_data assert dns_reconstructed.discovery_path == [ - ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", context_data] + context_data ] + assert dns_reconstructed.parent_chain == ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc"] class TestJSONSIEMFriendly(ModuleTestBase): diff --git a/docs/scanning/events.md b/docs/scanning/events.md index ea4fa3c82..8e1a6c1c9 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -21,43 +21,40 @@ These attributes allow us to construct a visual graph of events (e.g. in [Neo4j] ```json { - "type": "URL", - "id": "URL:c9962277277393f8895d2a4fa9b7f70b15f3af3e", + "type": "DNS_NAME", + "id": "DNS_NAME:879e47564ff0ed7711b707d3dbecb706ad6af1a3", "scope_description": "in-scope", - "data": "https://blog.blacklanternsecurity.com/", - "host": "blog.blacklanternsecurity.com", + "data": "www.blacklanternsecurity.com", + "host": "www.blacklanternsecurity.com", "resolved_hosts": [ - "104.18.40.87" + "185.199.108.153", + "2606:50c0:8003::153", + "blacklanternsecurity.github.io" ], - "dns_children": { - "A": [ - "104.18.40.87", - "172.64.147.169" - ] - }, + "dns_children": {}, "web_spider_distance": 0, "scope_distance": 0, - "scan": "SCAN:9224b49405e6d1607fd615243577d9ca86c7d206", - "timestamp": 1717260760.157012, - "parent": "OPEN_TCP_PORT:ebe3d6c10b41f60e3590ce6436ab62510b91c758", + "scan": "SCAN:477d1e6b94be928bf85c554b0845985189cfc81d", + "timestamp": "2024-08-17T03:49:47.906017+00:00", + "parent": "DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", "tags": [ - "in-scope", - "http-title-black-lantern-security-blsops", - "dir", - "ip-104-18-40-87", - "cdn-cloudflare", - "status-200" + "cdn-github", + "subdomain", + "in-scope" ], - "module": "httpx", - "module_sequence": "httpx", - "discovery_context": "httpx visited blog.blacklanternsecurity.com:443 and got status code 200 at https://blog.blacklanternsecurity.com/", + "module": "otx", + "module_sequence": "otx", + "discovery_context": "otx searched otx API for \"blacklanternsecurity.com\" and found DNS_NAME: www.blacklanternsecurity.com", "discovery_path": [ - "Scan difficult_arthur seeded with DNS_NAME: blacklanternsecurity.com", - "certspotter searched certspotter API for \"blacklanternsecurity.com\" and found DNS_NAME: blog.blacklanternsecurity.com", - "speculated OPEN_TCP_PORT: blog.blacklanternsecurity.com:443", - "httpx visited blog.blacklanternsecurity.com:443 and got status code 200 at https://blog.blacklanternsecurity.com/" + "Scan demonic_jimmy seeded with DNS_NAME: blacklanternsecurity.com", + "otx searched otx API for \"blacklanternsecurity.com\" and found DNS_NAME: www.blacklanternsecurity.com" + ], + "parent_chain": [ + "DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", + "DNS_NAME:879e47564ff0ed7711b707d3dbecb706ad6af1a3" ] } + ``` For a more detailed description of BBOT events, see [Developer Documentation - Event](../../dev/event). From a648b1c7c95cfd44bae0b05f2eb52c1c5d5e2ca6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 17 Aug 2024 00:01:48 -0400 Subject: [PATCH 005/254] blacked --- bbot/test/test_step_2/module_tests/test_module_json.py | 4 +--- docs/scanning/events.md | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 7c5d163e0..3823bb8a3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -37,9 +37,7 @@ def check(self, module_test, events): assert scan_reconstructed.data["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_reconstructed.data == dns_data assert dns_reconstructed.discovery_context == context_data - assert dns_reconstructed.discovery_path == [ - context_data - ] + assert dns_reconstructed.discovery_path == [context_data] assert dns_reconstructed.parent_chain == ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc"] diff --git a/docs/scanning/events.md b/docs/scanning/events.md index 8e1a6c1c9..9c44407fc 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -54,7 +54,6 @@ These attributes allow us to construct a visual graph of events (e.g. in [Neo4j] "DNS_NAME:879e47564ff0ed7711b707d3dbecb706ad6af1a3" ] } - ``` For a more detailed description of BBOT events, see [Developer Documentation - Event](../../dev/event). From 4af877d7bcc69126ee40e2cc93faf1ae2dff8726 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 17 Aug 2024 08:23:36 -0400 Subject: [PATCH 006/254] fix csv --- bbot/modules/output/csv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py index 3b214918b..3141713fa 100644 --- a/bbot/modules/output/csv.py +++ b/bbot/modules/output/csv.py @@ -55,7 +55,6 @@ def writerow(self, row): async def handle_event(self, event): # ["Event type", "Event data", "IP Address", "Source Module", "Scope Distance", "Event Tags"] discovery_path = getattr(event, "discovery_path", []) - discovery_path = [e[-1] for e in discovery_path] self.writerow( { "Event type": getattr(event, "type", ""), From 3d74ac5ac5528e68ae70a38e66c75a47315763ab Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 17 Aug 2024 09:07:46 -0400 Subject: [PATCH 007/254] fix json tests --- bbot/test/test_step_2/module_tests/test_module_json.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 3823bb8a3..7d9e052e7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -26,7 +26,8 @@ def check(self, module_test, events): assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["discovery_context"] == context_data - assert dns_json["discovery_path"] == [["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", context_data]] + assert dns_json["discovery_path"] == [context_data] + assert dns_json["parent_chain"] == ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc"] # event objects reconstructed from json scan_reconstructed = event_from_json(scan_json) From 1a6918bc540366eb73652a323086447465feffa8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 26 Aug 2024 15:53:51 -0400 Subject: [PATCH 008/254] clean commit --- bbot/modules/securitytxt.py | 8 -------- .../test_step_2/module_tests/test_module_securitytxt.py | 8 +++----- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/bbot/modules/securitytxt.py b/bbot/modules/securitytxt.py index 31dd0436a..681519e37 100644 --- a/bbot/modules/securitytxt.py +++ b/bbot/modules/securitytxt.py @@ -65,18 +65,15 @@ class securitytxt(BaseModule): "created_date": "2024-05-26", } options = { - "in_scope_only": True, "emails": True, "urls": True, } options_desc = { - "in_scope_only": "Only emit events related to in-scope domains", "emails": "emit EMAIL_ADDRESS events", "urls": "emit URL_UNVERIFIED events", } async def setup(self): - self.in_scope_only = self.config.get("in_scope_only", True) self._emails = self.config.get("emails", True) self._urls = self.config.get("urls", True) return await super().setup() @@ -89,11 +86,6 @@ def _incoming_dedup_hash(self, event): async def filter_event(self, event): if "_wildcard" in str(event.host).split("."): return False, "event is wildcard" - - # scope filtering - if event.scope_distance > 0 and self.in_scope_only: - return False, "event is not in scope" - return True async def handle_event(self, event): diff --git a/bbot/test/test_step_2/module_tests/test_module_securitytxt.py b/bbot/test/test_step_2/module_tests/test_module_securitytxt.py index 3d64a4eb6..ed1443ea4 100644 --- a/bbot/test/test_step_2/module_tests/test_module_securitytxt.py +++ b/bbot/test/test_step_2/module_tests/test_module_securitytxt.py @@ -31,16 +31,14 @@ def check(self, module_test, events): assert not any(str(e.data) == "vdp@example.com" for e in events) -class TestSecurityTxtInScopeFalse(TestSecurityTxt): +class TestSecurityTxtEmailsFalse(TestSecurityTxt): config_overrides = { "scope": {"report_distance": 1}, - "modules": {"securitytxt": {"in_scope_only": False}}, + "modules": {"securitytxt": {"emails": False}}, } def check(self, module_test, events): - assert any( - e.type == "EMAIL_ADDRESS" and e.data == "vdp@example.com" for e in events - ), "Failed to detect email address" + assert not any(e.type == "EMAIL_ADDRESS" for e in events), "Detected email address when emails=False" assert any( e.type == "URL_UNVERIFIED" and e.data == "https://vdp.example.com/" for e in events ), "Failed to detect URL" From e1d5498737a4e1e025dd1824687a9b1e4a74ab1d Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 27 Aug 2024 10:26:23 -0400 Subject: [PATCH 009/254] add test --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 3279d3c5d..2e0427ced 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -19,7 +19,7 @@ async def setup_before_prep(self, module_test): \\nhttps://www1.test.notreal \\x3dhttps://www2.test.notreal %0ahttps://www3.test.notreal - \\u000ahttps://www4.test.notreal + \\u000ahttps://www4.test.notreal: \nwww5.test.notreal \\x3dwww6.test.notreal %0awww7.test.notreal From 5c5f2f6ef69480bb26fdff1e229bb4253df8cc97 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 28 Aug 2024 17:10:57 -0400 Subject: [PATCH 010/254] WIP DNS optimizations --- bbot/modules/internal/dnsresolve.py | 372 ++++++++++------------------ 1 file changed, 124 insertions(+), 248 deletions(-) diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 42ec8cf94..8d1469a30 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -12,10 +12,13 @@ class DNSResolve(InterceptModule): """ TODO: - - scrap event cache in favor of the parent backtracking method + - scrap event cache in favor of the parent backtracking method (actual event should have all the information) - don't duplicate resolution on the same host - clean up wildcard checking to only happen once, and re-emit/abort if one is detected - - same thing with main_host_event. we should never be processing two events - only one. + - same thing with main_host_event. we should never be processing two events - only one. + - do not emit any hosts/children/raw until after scope checks + - and only emit them if they're inside scope distance + - order: A/AAAA --> scope check --> then rest? """ watched_events = ["*"] @@ -40,6 +43,8 @@ async def setup(self): return None, "DNS resolution is disabled in the config" self.minimal = self.dns_config.get("minimal", False) + self.minimal_rdtypes = ("A", "AAAA", "CNAME") + self.non_minimal_rdtypes = [t for t in all_rdtypes if t not in self.minimal_rdtypes] self.dns_search_distance = max(0, int(self.dns_config.get("search_distance", 1))) self._emit_raw_records = None @@ -51,268 +56,122 @@ async def setup(self): return True - @property - def _dns_search_distance(self): - return max(self.scan.scope_search_distance, self.dns_search_distance) - - @property - def emit_raw_records(self): - if self._emit_raw_records is None: - watching_raw_records = any( - ["RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values()] - ) - omitted_event_types = self.scan.config.get("omit_event_types", []) - omit_raw_records = "RAW_DNS_RECORD" in omitted_event_types - self._emit_raw_records = watching_raw_records or not omit_raw_records - return self._emit_raw_records - async def filter_event(self, event): if (not event.host) or (event.type in ("IP_RANGE",)): return False, "event does not have host attribute" return True async def handle_event(self, event, **kwargs): + raw_records = {} event_is_ip = self.helpers.is_ip(event.host) - event_host = str(event.host) - event_host_hash = hash(event_host) - - async with self._event_cache_locks.lock(event_host_hash): - # first thing we do is check for wildcards - if not event_is_ip: - if event.scope_distance <= self.scan.scope_search_distance: - await self.handle_wildcard_event(event) - - event_host = str(event.host) - event_host_hash = hash(event_host) - - # we do DNS resolution inside a lock to make sure we don't duplicate work - # once the resolution happens, it will be cached so it doesn't need to happen again - async with self._event_cache_locks.lock(event_host_hash): - try: - # try to get from cache - # the "main host event" is the original parent IP_ADDRESS or DNS_NAME - main_host_event, dns_tags, event_whitelisted, event_blacklisted = self._event_cache[event_host_hash] - # dns_tags, dns_children, event_whitelisted, event_blacklisted = self._event_cache[event_host_hash] - except KeyError: - - main_host_event, dns_tags, event_whitelisted, event_blacklisted, raw_record_events = ( - await self.resolve_event(event) - ) - - # if we're not blacklisted and we haven't already done it, emit the main host event and all its raw records - main_host_resolved = getattr(main_host_event, "_resolved", False) - if not event_blacklisted and not main_host_resolved: - if event_whitelisted: - self.debug( - f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)" - ) - main_host_event.scope_distance = 0 - await self.handle_wildcard_event(main_host_event) - - in_dns_scope = -1 < main_host_event.scope_distance < self._dns_search_distance + main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event) - if event != main_host_event: - await self.emit_event(main_host_event) - for raw_record_event in raw_record_events: - await self.emit_event(raw_record_event) - - # kill runaway DNS chains - dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) - if dns_resolve_distance >= self.helpers.dns.runaway_limit: - self.debug( - f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})" - ) - main_host_event.dns_children = {} - - # emit DNS children - if not self.minimal: - in_dns_scope = -1 < event.scope_distance < self._dns_search_distance - for rdtype, records in main_host_event.dns_children.items(): - module = self._make_dummy_module(rdtype) - for record in records: - parents = main_host_event.get_parents() - for e in parents: - e_is_host = e.type in ("DNS_NAME", "IP_ADDRESS") - e_parent_matches = str(e.parent.host) == str(main_host_event.host) - e_host_matches = str(e.data) == str(record) - e_module_matches = str(e.module) == str(module) - if e_is_host and e_parent_matches and e_host_matches and e_module_matches: - self.trace( - f"TRYING TO EMIT ALREADY-EMITTED {record}:{rdtype} CHILD OF {main_host_event}, parents: {parents}" - ) - return - try: - child_event = self.scan.make_event( - record, "DNS_NAME", module=module, parent=main_host_event - ) - child_event.discovery_context = f"{rdtype} record for {event.host} contains {child_event.type}: {child_event.host}" - # if it's a hostname and it's only one hop away, mark it as affiliate - if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: - child_event.add_tag("affiliate") - if in_dns_scope or self.preset.in_scope(child_event): - self.debug(f"Queueing DNS child for {event}: {child_event}") - await self.emit_event(child_event) - except ValidationError as e: - self.warning( - f'Event validation failed for DNS child of {main_host_event}: "{record}" ({rdtype}): {e}' - ) - - # mark the host as resolved - main_host_event._resolved = True - - # store results in cache - self._event_cache[event_host_hash] = main_host_event, dns_tags, event_whitelisted, event_blacklisted + # minimal resolution - first, we resolve A/AAAA records for scope purposes + if new_event or event is main_host_event: + if event_is_ip: + basic_records = await self.resolve_event(main_host_event, types=("PTR",)) + else: + basic_records = await self.resolve_event(main_host_event, types=self.minimal_rdtypes) + # are any of its IPs whitelisted/blacklisted? + whitelisted, blacklisted = self.check_scope(main_host_event) + if whitelisted and event.scope_distance > 0: + self.debug( + f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)" + ) + main_host_event.scope_distance = 0 + raw_records.update(basic_records) # abort if the event resolves to something blacklisted - if event_blacklisted: - return False, f"it has a blacklisted DNS record" - - # if the event resolves to an in-scope IP, set its scope distance to 0 - if event_whitelisted: - self.debug(f"Making {event} in-scope because it resolves to an in-scope resource") - event.scope_distance = 0 - await self.handle_wildcard_event(event) - - # transfer resolved hosts - event._resolved_hosts = main_host_event._resolved_hosts - - # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED - if event.type == "DNS_NAME" and "unresolved" in event.tags: - event.type = "DNS_NAME_UNRESOLVED" - - async def resolve_event(self, event): - dns_tags = set() - event_whitelisted = False - event_blacklisted = False - - main_host_event = self.get_dns_parent(event) - event_host = str(event.host) - event_is_ip = self.helpers.is_ip(event.host) - - rdtypes_to_resolve = () - if event_is_ip: - if not self.minimal: - rdtypes_to_resolve = ("PTR",) + if blacklisted: + return False, "it has a blacklisted DNS record" + + # are we within our dns search distance? + within_dns_distance = main_host_event.scope_distance <= self._dns_search_distance + within_scope_distance = main_host_event.scope_distance <= self.scan.scope_search_distance + + # if so, resolve the rest of our records + if not event_is_ip: + if (not self.minimal) and within_dns_distance: + records = await self.resolve_event(main_host_event, types=self.minimal_rdtypes) + raw_records.update(records) + # check for wildcards if we're within the scan's search distance + if new_event and within_scope_distance: + await self.handle_wildcard_event(main_host_event) + + # kill runaway DNS chains + # TODO: test this + dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) + runaway_dns = dns_resolve_distance >= self.helpers.dns.runaway_limit + if runaway_dns: + self.debug( + f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})" + ) else: - if self.minimal: - rdtypes_to_resolve = ("A", "AAAA", "CNAME") - else: - rdtypes_to_resolve = all_rdtypes - - # if missing from cache, do DNS resolution - queries = [(event_host, rdtype) for rdtype in rdtypes_to_resolve] - error_rdtypes = [] - raw_record_events = [] - async for (query, rdtype), (answer, errors) in self.helpers.dns.resolve_raw_batch(queries): - if self.emit_raw_records and rdtype not in ("A", "AAAA", "CNAME", "PTR"): - raw_record_event = self.make_event( - {"host": str(event_host), "type": rdtype, "answer": answer.to_text()}, - "RAW_DNS_RECORD", - parent=main_host_event, - tags=[f"{rdtype.lower()}-record"], - context=f"{rdtype} lookup on {{event.parent.host}} produced {{event.type}}", - ) - raw_record_events.append(raw_record_event) - if errors: - error_rdtypes.append(rdtype) - dns_tags.add(f"{rdtype.lower()}-record") - for _rdtype, host in extract_targets(answer): - try: - main_host_event.dns_children[_rdtype].add(host) - except KeyError: - main_host_event.dns_children[_rdtype] = {host} - - # if there were dns resolution errors, notify the user with tags - for rdtype in error_rdtypes: - if rdtype not in main_host_event.dns_children: - dns_tags.add(f"{rdtype.lower()}-error") - - # if there weren't any DNS children and it's not an IP address, tag as unresolved - if not main_host_event.dns_children and not event_is_ip: - dns_tags.add("unresolved") - - # check DNS children against whitelists and blacklists - for rdtype, children in main_host_event.dns_children.items(): - if event_blacklisted: - break - for host in children: - # whitelisting / blacklisting based on resolved hosts - if rdtype in ("A", "AAAA", "CNAME"): - # having a CNAME to an in-scope resource doesn't make you in-scope - if (not event_whitelisted) and rdtype != "CNAME": + if within_dns_distance: + pass + # emit dns children + # emit raw records + # emit host event + + # update host event --> event + + def check_scope(self, event): + whitelisted = False + blacklisted = False + dns_children = getattr(event, "dns_children", {}) + for rdtype in ("A", "AAAA", "CNAME"): + hosts = dns_children.get(rdtype, []) + # update resolved hosts + event.resolved_hosts.update(hosts) + for host in hosts: + # having a CNAME to an in-scope resource doesn't make you in-scope + if rdtype != "CNAME": + if not whitelisted: with suppress(ValidationError): if self.scan.whitelisted(host): - event_whitelisted = True - dns_tags.add(f"dns-whitelisted-{rdtype.lower()}") - # CNAME to a blacklisted resource, means you're blacklisted + whitelisted = True + event.add_tag(f"dns-whitelisted-{rdtype}") + if not blacklisted: with suppress(ValidationError): if self.scan.blacklisted(host): - dns_tags.add("blacklisted") - dns_tags.add(f"dns-blacklisted-{rdtype.lower()}") - event_blacklisted = True - event_whitelisted = False - break - - # check for private IPs + blacklisted = True + event.add_tag("blacklisted") + event.add_tag(f"dns-blacklisted-{rdtype}") + if blacklisted: + whitelisted = False + return whitelisted, blacklisted + + async def resolve_event(self, event, types): + raw_records = {} + event_host = str(event.host) + queries = [(event_host, rdtype) for rdtype in types] + dns_errors = {} + async for (query, rdtype), (answer, errors) in self.helpers.dns.resolve_raw_batch(queries): + try: + dns_errors[rdtype].update(errors) + except KeyError: + dns_errors[rdtype] = set(errors) + # raw dnspython objects + try: + raw_records[rdtype].add(answer) + except KeyError: + raw_records[rdtype] = {answer} + # hosts + for _rdtype, host in extract_targets(answer): + event.add_tag(f"{_rdtype}-record") try: - ip = ipaddress.ip_address(host) - if ip.is_private: - dns_tags.add("private-ip") - except ValueError: - continue - - # add DNS tags to main host - for tag in dns_tags: - main_host_event.add_tag(tag) - - # set resolved_hosts attribute - for rdtype, children in main_host_event.dns_children.items(): - if rdtype in ("A", "AAAA", "CNAME"): - for host in children: - main_host_event._resolved_hosts.add(host) - - return main_host_event, dns_tags, event_whitelisted, event_blacklisted, raw_record_events + event.dns_children[_rdtype].add(host) + except KeyError: + event.dns_children[_rdtype] = {host} + # tag event with errors + for rdtype, errors in dns_errors.items(): + # only consider it an error if there weren't any results for that rdtype + if errors and not rdtype in event.dns_children: + event.add_tag(f"{rdtype}-error") + return raw_records async def handle_wildcard_event(self, event): - self.debug(f"Entering handle_wildcard_event({event})") - try: - event_host = str(event.host) - # check if the dns name itself is a wildcard entry - wildcard_rdtypes = await self.helpers.is_wildcard(event_host) - for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items(): - if is_wildcard == False: - continue - elif is_wildcard == True: - event.add_tag("wildcard") - wildcard_tag = "wildcard" - elif is_wildcard == None: - wildcard_tag = "error" - - event.add_tag(f"{rdtype.lower()}-{wildcard_tag}") - - # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) - if wildcard_rdtypes and not "target" in event.tags: - # these are the rdtypes that have wildcards - wildcard_rdtypes_set = set(wildcard_rdtypes) - # consider the event a full wildcard if all its records are wildcards - event_is_wildcard = False - if wildcard_rdtypes_set: - event_is_wildcard = all(r[0] == True for r in wildcard_rdtypes.values()) - - if event_is_wildcard: - if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): - wildcard_parent = self.helpers.parent_domain(event_host) - for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): - if _is_wildcard: - wildcard_parent = _parent_domain - break - wildcard_data = f"_wildcard.{wildcard_parent}" - if wildcard_data != event.data: - self.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"') - event.data = wildcard_data - - finally: - self.debug(f"Finished handle_wildcard_event({event})") + pass def get_dns_parent(self, event): """ @@ -320,7 +179,9 @@ def get_dns_parent(self, event): """ for parent in event.get_parents(include_self=True): if parent.host == event.host and parent.type in ("IP_ADDRESS", "DNS_NAME", "DNS_NAME_UNRESOLVED"): - return parent + blacklisted = any(t.startswith("dns-blacklisted-") for t in parent.tags) + whitelisted = any(t.startswith("dns-whitelisted-") for t in parent.tags) + return parent, whitelisted, blacklisted, False tags = set() if "target" in event.tags: tags.add("target") @@ -331,7 +192,22 @@ def get_dns_parent(self, event): parent=event, context="{event.parent.type} has host {event.type}: {event.host}", tags=tags, - ) + ), None, None, True + + @property + def emit_raw_records(self): + if self._emit_raw_records is None: + watching_raw_records = any( + ["RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values()] + ) + omitted_event_types = self.scan.config.get("omit_event_types", []) + omit_raw_records = "RAW_DNS_RECORD" in omitted_event_types + self._emit_raw_records = watching_raw_records or not omit_raw_records + return self._emit_raw_records + + @property + def _dns_search_distance(self): + return max(self.scan.scope_search_distance, self.dns_search_distance) def _make_dummy_module(self, name): try: From d3ad8fe1d4d3a445cc09e6ac525818d9b6bddb18 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 29 Aug 2024 17:06:21 -0400 Subject: [PATCH 011/254] more DNS WIP --- bbot/core/event/base.py | 1 + bbot/core/helpers/dns/dns.py | 20 ++- bbot/core/helpers/dns/engine.py | 252 ++++++++++++++------------- bbot/defaults.yml | 2 +- bbot/errors.py | 8 - bbot/modules/internal/dnsresolve.py | 257 ++++++++++++++++++---------- bbot/test/bbot_fixtures.py | 12 +- bbot/test/test_step_1/test_dns.py | 23 +-- 8 files changed, 332 insertions(+), 243 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 3eb10625f..94df01cf5 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -171,6 +171,7 @@ def __init__( self._module_priority = None self._resolved_hosts = set() self.dns_children = dict() + self.raw_dns_records = dict() self._discovery_context = "" self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}") self.web_spider_distance = 0 diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 07f562132..757474015 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -117,8 +117,11 @@ def brute(self): self._brute = DNSBrute(self.parent_helper) return self._brute - @async_cachedmethod(lambda self: self._is_wildcard_cache) - async def is_wildcard(self, query, ips=None, rdtype=None): + @async_cachedmethod( + lambda self: self._is_wildcard_cache, + key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes))), + ) + async def is_wildcard(self, query, rdtypes, raw_dns_records=None): """ Use this method to check whether a *host* is a wildcard entry @@ -150,9 +153,6 @@ async def is_wildcard(self, query, ips=None, rdtype=None): Note: `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive) """ - if [ips, rdtype].count(None) == 1: - raise ValueError("Both ips and rdtype must be specified") - query = self._wildcard_prevalidation(query) if not query: return {} @@ -161,15 +161,17 @@ async def is_wildcard(self, query, ips=None, rdtype=None): if is_domain(query): return {} - return await self.run_and_return("is_wildcard", query=query, ips=ips, rdtype=rdtype) + return await self.run_and_return("is_wildcard", query=query, rdtypes=rdtypes, raw_dns_records=raw_dns_records) - @async_cachedmethod(lambda self: self._is_wildcard_domain_cache) - async def is_wildcard_domain(self, domain, log_info=False): + @async_cachedmethod( + lambda self: self._is_wildcard_domain_cache, key=lambda domain, rdtypes: (domain, tuple(sorted(rdtypes))) + ) + async def is_wildcard_domain(self, domain, rdtypes): domain = self._wildcard_prevalidation(domain) if not domain: return {} - return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False) + return await self.run_and_return("is_wildcard_domain", domain=domain, rdtypes=rdtypes) def _wildcard_prevalidation(self, host): if self.wildcard_disable: diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 8a41c7c8e..e72b05b80 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -7,7 +7,6 @@ from cachetools import LRUCache from contextlib import suppress -from bbot.errors import DNSWildcardBreak from bbot.core.engine import EngineServer from bbot.core.helpers.async_helpers import NamedLock from bbot.core.helpers.dns.helpers import extract_targets @@ -16,7 +15,6 @@ rand_string, parent_domain, domain_parents, - clean_dns_record, ) @@ -361,8 +359,7 @@ async def resolve_raw_batch(self, queries, threads=10, **kwargs): ): query = args[0] rdtype = kwargs["type"] - for answer in answers: - yield ((query, rdtype), (answer, errors)) + yield ((query, rdtype), (answers, errors)) async def _catch(self, callback, *args, **kwargs): """ @@ -397,7 +394,7 @@ async def _catch(self, callback, *args, **kwargs): self.log.trace(traceback.format_exc()) return [] - async def is_wildcard(self, query, ips=None, rdtype=None): + async def is_wildcard(self, query, rdtypes, raw_dns_records=None): """ Use this method to check whether a *host* is a wildcard entry @@ -407,8 +404,8 @@ async def is_wildcard(self, query, ips=None, rdtype=None): Args: query (str): The hostname to check for a wildcard entry. - ips (list, optional): List of IPs to compare against, typically obtained from a previous DNS resolution of the query. - rdtype (str, optional): The DNS record type (e.g., "A", "AAAA") to consider during the check. + rdtypes (list): The DNS record type (e.g., "A", "AAAA") to consider during the check. + raw_dns_records (dict, optional): Dictionary of {rdtype: [answer1, answer2, ...], ...} containing raw dnspython answers for the query. Returns: dict: A dictionary indicating if the query is a wildcard for each checked DNS record type. @@ -420,104 +417,104 @@ async def is_wildcard(self, query, ips=None, rdtype=None): ValueError: If only one of `ips` or `rdtype` is specified or if no valid IPs are specified. Examples: - >>> is_wildcard("www.github.io") - {"A": (True, "github.io"), "AAAA": (True, "github.io")} + >>> is_wildcard("www.github.io", rdtypes=["A", "AAAA", "MX"]) + {"A": (True, "github.io"), "AAAA": (True, "github.io"), "MX": (False, "github.io")} - >>> is_wildcard("www.evilcorp.com", ips=["93.184.216.34"], rdtype="A") + >>> is_wildcard("www.evilcorp.com", rdtypes=["A"]) {"A": (False, "evilcorp.com")} Note: `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive) """ - result = {} + if isinstance(rdtypes, str): + rdtypes = [rdtypes] - parent = parent_domain(query) - parents = list(domain_parents(query)) + result = {} - if rdtype is not None: - if isinstance(rdtype, str): - rdtype = [rdtype] - rdtypes_to_check = rdtype - else: - rdtypes_to_check = all_rdtypes - - query_baseline = dict() - # if the caller hasn't already done the work of resolving the IPs - if ips is None: - # then resolve the query for all rdtypes - queries = [(query, t) for t in rdtypes_to_check] - async for (query, _rdtype), (answers, errors) in self.resolve_raw_batch(queries): - answers = extract_targets(answers) + # if the work of resolving hasn't been done yet, do it + if raw_dns_records is None: + raw_dns_records = {} + queries = [(query, rdtype) for rdtype in rdtypes] + async for (_, rdtype), (answers, errors) in self.resolve_raw_batch(queries): if answers: - query_baseline[_rdtype] = set([a[1] for a in answers]) + for answer in answers: + try: + raw_dns_records[rdtype].add(answer) + except KeyError: + raw_dns_records[rdtype] = {answer} else: if errors: - self.debug(f"Failed to resolve {query} ({_rdtype}) during wildcard detection") - result[_rdtype] = (None, parent) - continue - else: - # otherwise, we can skip all that - cleaned_ips = set([clean_dns_record(ip) for ip in ips]) - if not cleaned_ips: - raise ValueError("Valid IPs must be specified") - query_baseline[rdtype] = cleaned_ips - if not query_baseline: + self.debug(f"Failed to resolve {query} ({rdtype}) during wildcard detection") + result[rdtype] = (None, query) + + # clean + process the raw records into a baseline + baseline = {} + baseline_raw = {} + for rdtype, answers in raw_dns_records.items(): + for answer in answers: + text_answer = answer.to_text() + try: + baseline_raw[rdtype].add(text_answer) + except KeyError: + baseline_raw[rdtype] = {text_answer} + for _, host in extract_targets(answer): + try: + baseline[rdtype].add(host) + except KeyError: + baseline[rdtype] = {host} + + # if it's unresolved, it's a big nope + if not raw_dns_records: return result # once we've resolved the base query and have IP addresses to work with # we can compare the IPs to the ones we have on file for wildcards # for every parent domain, starting with the shortest - try: - for host in parents[::-1]: - # make sure we've checked that domain for wildcards - await self.is_wildcard_domain(host) - - # for every rdtype - for _rdtype in list(query_baseline): - # get the IPs from above - query_ips = query_baseline.get(_rdtype, set()) - host_hash = hash(host) - - if host_hash in self._wildcard_cache: - # then get its IPs from our wildcard cache - wildcard_rdtypes = self._wildcard_cache[host_hash] - - # then check to see if our IPs match the wildcard ones - if _rdtype in wildcard_rdtypes: - wildcard_ips = wildcard_rdtypes[_rdtype] - # if our IPs match the wildcard ones, then ladies and gentlemen we have a wildcard - is_wildcard = any(r in wildcard_ips for r in query_ips) - - if is_wildcard and not result.get(_rdtype, (None, None))[0] is True: - result[_rdtype] = (True, host) - - # if we've reached a point where the dns name is a complete wildcard, class can be dismissed early - base_query_rdtypes = set(query_baseline) - wildcard_rdtypes_set = set([k for k, v in result.items() if v[0] is True]) - if base_query_rdtypes and wildcard_rdtypes_set and base_query_rdtypes == wildcard_rdtypes_set: - self.log.debug( - f"Breaking from wildcard detection for {query} at {host} because base query rdtypes ({base_query_rdtypes}) == wildcard rdtypes ({wildcard_rdtypes_set})" - ) - raise DNSWildcardBreak() - - except DNSWildcardBreak: - pass - - for _rdtype, answers in query_baseline.items(): - if answers and _rdtype not in result: - result[_rdtype] = (False, query) + parents = list(domain_parents(query)) + for parent in parents[::-1]: + + # check if the parent domain is set up with wildcards + wildcard_results = await self.is_wildcard_domain(parent, rdtypes) + + # for every rdtype + for rdtype in list(baseline_raw): + # skip if we already found a wildcard for this rdtype + if rdtype in result: + continue + + # get our baseline IPs from above + _baseline = baseline.get(rdtype, set()) + _baseline_raw = baseline_raw.get(rdtype, set()) + + wildcards = wildcard_results.get(rdtype, None) + if wildcards is None: + continue + wildcards, wildcard_raw = wildcards + + # check if any of our baseline IPs are in the wildcard results + is_wildcard = any(r in wildcards for r in _baseline) + is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw) + + # if there are any matches, we have a wildcard + if is_wildcard or is_wildcard_raw: + result[rdtype] = (True, query) + + # any rdtype that wasn't a wildcard, mark it as False + for rdtype, answers in baseline_raw.items(): + if answers and rdtype not in result: + result[rdtype] = (False, query) return result - async def is_wildcard_domain(self, domain, log_info=False): + async def is_wildcard_domain(self, domain, rdtypes): """ Check whether a given host or its children make use of wildcard DNS entries. Wildcard DNS can have various implications, particularly in subdomain enumeration and subdomain takeovers. Args: domain (str): The domain to check for wildcard DNS entries. - log_info (bool, optional): Whether to log the result of the check. Defaults to False. + rdtypes (list): Which DNS record types to check. Returns: dict: A dictionary where the keys are the parent domains that have wildcard DNS entries, @@ -531,60 +528,75 @@ async def is_wildcard_domain(self, domain, log_info=False): >>> is_wildcard_domain("example.com") {} """ - wildcard_domain_results = {} - - rdtypes_to_check = set(all_rdtypes) - + if isinstance(rdtypes, str): + rdtypes = [rdtypes] + rdtypes = set(rdtypes) + wildcard_results = {} # make a list of its parents parents = list(domain_parents(domain, include_self=True)) # and check each of them, beginning with the highest parent (i.e. the root domain) for i, host in enumerate(parents[::-1]): - # have we checked this host before? - host_hash = hash(host) - async with self._wildcard_lock.lock(host_hash): - # if we've seen this host before - if host_hash in self._wildcard_cache: - wildcard_domain_results[host] = self._wildcard_cache[host_hash] - continue + queries = [((host, rdtype), {}) for rdtype in rdtypes] + async for ((_, rdtype), _, _), (results, results_raw) in self.task_pool( + self._is_wildcard_zone, args_kwargs=queries + ): + # if we hit a wildcard, we can skip this rdtype from now on + if results_raw: + rdtypes.remove(rdtype) + wildcard_results[rdtype] = results, results_raw + + if wildcard_results: + wildcard_rdtypes_str = ",".join(sorted(wildcard_results)) + self.log.info(f"Encountered domain with wildcard DNS ({wildcard_rdtypes_str}): {host}") + else: + self.log.verbose(f"Finished checking {host}, it is not a wildcard") - self.log.verbose(f"Checking if {host} is a wildcard") + return wildcard_results - # determine if this is a wildcard domain + async def _is_wildcard_zone(self, host, rdtype): + """ + Check whether a specific DNS zone+rdtype has a wildcard configuration + """ + rdtype = rdtype.upper() - # resolve a bunch of random subdomains of the same parent - is_wildcard = False - wildcard_results = dict() + # have we checked this host before? + host_hash = hash(f"{host}:{rdtype}") + async with self._wildcard_lock.lock(host_hash): + # if we've seen this host before + try: + wildcard_results, wildcard_results_raw = self._wildcard_cache[host_hash] + self.debug(f"Got {host}:{rdtype} from cache") + except KeyError: + wildcard_results = set() + wildcard_results_raw = set() + self.debug(f"Checking if {host}:{rdtype} is a wildcard") + # determine if this is a wildcard domain + # resolve a bunch of random subdomains of the same parent rand_queries = [] - for rdtype in rdtypes_to_check: - for _ in range(self.wildcard_tests): - rand_query = f"{rand_string(digits=False, length=10)}.{host}" - rand_queries.append((rand_query, rdtype)) + for _ in range(self.wildcard_tests): + rand_query = f"{rand_string(digits=False, length=10)}.{host}" + rand_queries.append((rand_query, rdtype)) async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(rand_queries, use_cache=False): - answers = extract_targets(answers) - if answers: - is_wildcard = True - if not rdtype in wildcard_results: - wildcard_results[rdtype] = set() - wildcard_results[rdtype].update(set(a[1] for a in answers)) - # we know this rdtype is a wildcard - # so we don't need to check it anymore - with suppress(KeyError): - rdtypes_to_check.remove(rdtype) - - self._wildcard_cache.update({host_hash: wildcard_results}) - wildcard_domain_results.update({host: wildcard_results}) - if is_wildcard: - wildcard_rdtypes_str = ",".join(sorted([t.upper() for t, r in wildcard_results.items() if r])) - log_fn = self.log.verbose - if log_info: - log_fn = self.log.info - log_fn(f"Encountered domain with wildcard DNS ({wildcard_rdtypes_str}): {host}") + for answer in answers: + # consider both the raw record + wildcard_results_raw.add(answer.to_text()) + # and all the extracted hosts + for _, t in extract_targets(answer): + wildcard_results.add(t) + + if wildcard_results: + self.debug(f"Finished checking {host}:{rdtype}, it is a wildcard") else: - self.log.verbose(f"Finished checking {host}, it is not a wildcard") + self.debug(f"Finished checking {host}:{rdtype}, it is not a wildcard") + self._wildcard_cache[host_hash] = wildcard_results, wildcard_results_raw + + return wildcard_results, wildcard_results_raw - return wildcard_domain_results + async def _is_wildcard(self, query, rdtypes, dns_children): + if isinstance(rdtypes, str): + rdtypes = [rdtypes] @property def dns_connectivity_lock(self): diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 2ce8d4208..f57796672 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -30,7 +30,7 @@ dns: # Speed up scan by not creating any new DNS events, and only resolving A and AAAA records minimal: false # How many instances of the dns module to run concurrently - threads: 20 + threads: 25 # How many concurrent DNS resolvers to use when brute-forcing # (under the hood this is passed through directly to massdns -s) brute_threads: 1000 diff --git a/bbot/errors.py b/bbot/errors.py index 53bdee48c..a5ab9619b 100644 --- a/bbot/errors.py +++ b/bbot/errors.py @@ -38,14 +38,6 @@ class WordlistError(BBOTError): pass -class DNSError(BBOTError): - pass - - -class DNSWildcardBreak(DNSError): - pass - - class CurlError(BBOTError): pass diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 8d1469a30..8ffbeb60b 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -1,26 +1,13 @@ import ipaddress from contextlib import suppress -from cachetools import LFUCache from bbot.errors import ValidationError from bbot.core.helpers.dns.engine import all_rdtypes -from bbot.core.helpers.async_helpers import NamedLock from bbot.modules.base import InterceptModule, BaseModule from bbot.core.helpers.dns.helpers import extract_targets class DNSResolve(InterceptModule): - """ - TODO: - - scrap event cache in favor of the parent backtracking method (actual event should have all the information) - - don't duplicate resolution on the same host - - clean up wildcard checking to only happen once, and re-emit/abort if one is detected - - same thing with main_host_event. we should never be processing two events - only one. - - do not emit any hosts/children/raw until after scope checks - - and only emit them if they're inside scope distance - - order: A/AAAA --> scope check --> then rest? - """ - watched_events = ["*"] _priority = 1 scope_distance_modifier = None @@ -44,15 +31,16 @@ async def setup(self): self.minimal = self.dns_config.get("minimal", False) self.minimal_rdtypes = ("A", "AAAA", "CNAME") - self.non_minimal_rdtypes = [t for t in all_rdtypes if t not in self.minimal_rdtypes] + if self.minimal: + self.non_minimal_rdtypes = () + else: + self.non_minimal_rdtypes = tuple([t for t in all_rdtypes if t not in self.minimal_rdtypes]) self.dns_search_distance = max(0, int(self.dns_config.get("search_distance", 1))) self._emit_raw_records = None - # event resolution cache - self._event_cache = LFUCache(maxsize=10000) - self._event_cache_locks = NamedLock() - self.host_module = self.HostModule(self.scan) + self.children_emitted = set() + self.children_emitted_raw = set() return True @@ -62,58 +50,146 @@ async def filter_event(self, event): return True async def handle_event(self, event, **kwargs): - raw_records = {} event_is_ip = self.helpers.is_ip(event.host) + if event_is_ip: + minimal_rdtypes = ("PTR",) + non_minimal_rdtypes = () + else: + minimal_rdtypes = self.minimal_rdtypes + non_minimal_rdtypes = self.non_minimal_rdtypes + + # first, we find or create the main DNS_NAME or IP_ADDRESS associated with this event main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event) # minimal resolution - first, we resolve A/AAAA records for scope purposes if new_event or event is main_host_event: - if event_is_ip: - basic_records = await self.resolve_event(main_host_event, types=("PTR",)) - else: - basic_records = await self.resolve_event(main_host_event, types=self.minimal_rdtypes) - # are any of its IPs whitelisted/blacklisted? - whitelisted, blacklisted = self.check_scope(main_host_event) - if whitelisted and event.scope_distance > 0: - self.debug( - f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)" - ) - main_host_event.scope_distance = 0 - raw_records.update(basic_records) + await self.resolve_event(main_host_event, types=minimal_rdtypes) + # are any of its IPs whitelisted/blacklisted? + whitelisted, blacklisted = self.check_scope(main_host_event) + if whitelisted and event.scope_distance > 0: + self.debug(f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)") + main_host_event.scope_distance = 0 # abort if the event resolves to something blacklisted if blacklisted: return False, "it has a blacklisted DNS record" - # are we within our dns search distance? - within_dns_distance = main_host_event.scope_distance <= self._dns_search_distance - within_scope_distance = main_host_event.scope_distance <= self.scan.scope_search_distance - - # if so, resolve the rest of our records if not event_is_ip: - if (not self.minimal) and within_dns_distance: - records = await self.resolve_event(main_host_event, types=self.minimal_rdtypes) - raw_records.update(records) - # check for wildcards if we're within the scan's search distance - if new_event and within_scope_distance: - await self.handle_wildcard_event(main_host_event) - - # kill runaway DNS chains - # TODO: test this - dns_resolve_distance = getattr(event, "dns_resolve_distance", 0) + # if we're within our dns search distance, resolve the rest of our records + if main_host_event.scope_distance < self._dns_search_distance: + await self.resolve_event(main_host_event, types=non_minimal_rdtypes) + # check for wildcards if we're within the scan's search distance + if ( + new_event + and main_host_event.scope_distance <= self.scan.scope_search_distance + and not "domain" in main_host_event.tags + ): + await self.handle_wildcard_event(main_host_event) + + dns_resolve_distance = getattr(main_host_event, "dns_resolve_distance", 0) runaway_dns = dns_resolve_distance >= self.helpers.dns.runaway_limit if runaway_dns: + # kill runaway DNS chains + # TODO: test this self.debug( f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})" ) else: - if within_dns_distance: - pass - # emit dns children - # emit raw records - # emit host event + # emit dns children + await self.emit_dns_children(main_host_event) + await self.emit_dns_children_raw(main_host_event) + + # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED + if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags: + main_host_event.type = "DNS_NAME_UNRESOLVED" + + # emit the main DNS_NAME or IP_ADDRESS + if new_event and event is not main_host_event and main_host_event.scope_distance <= self._dns_search_distance: + await self.emit_event(main_host_event) + + # transfer scope distance to event + event.scope_distance = main_host_event.scope_distance + event._resolved_hosts = main_host_event.resolved_hosts + + async def handle_wildcard_event(self, event): + rdtypes = tuple(event.raw_dns_records) + wildcard_rdtypes = await self.helpers.is_wildcard( + event.host, rdtypes=rdtypes, raw_dns_records=event.raw_dns_records + ) + for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items(): + if is_wildcard == False: + continue + elif is_wildcard == True: + event.add_tag("wildcard") + wildcard_tag = "wildcard" + elif is_wildcard == None: + wildcard_tag = "error" + event.add_tag(f"{rdtype}-{wildcard_tag}") - # update host event --> event + # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) + if wildcard_rdtypes and not "target" in event.tags: + # these are the rdtypes that have wildcards + wildcard_rdtypes_set = set(wildcard_rdtypes) + # consider the event a full wildcard if all its records are wildcards + event_is_wildcard = False + if wildcard_rdtypes_set: + event_is_wildcard = all(r[0] == True for r in wildcard_rdtypes.values()) + + if event_is_wildcard: + if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."): + wildcard_parent = self.helpers.parent_domain(event.host) + for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items(): + if _is_wildcard: + wildcard_parent = _parent_domain + break + wildcard_data = f"_wildcard.{wildcard_parent}" + if wildcard_data != event.data: + self.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"') + event.data = wildcard_data + + async def emit_dns_children(self, event): + for rdtype, children in event.dns_children.items(): + module = self._make_dummy_module(rdtype) + for child_host in children: + try: + child_event = self.scan.make_event( + child_host, + "DNS_NAME", + module=module, + parent=event, + context=f"{rdtype} record for {event.host} contains {{event.type}}: {{event.host}}", + ) + except ValidationError as e: + self.warning(f'Event validation failed for DNS child of {event}: "{child_host}" ({rdtype}): {e}') + continue + + child_hash = hash(f"{event.host}:{module}:{child_host}") + # if we haven't emitted this one before + if child_hash not in self.children_emitted: + # and it's either in-scope or inside our dns search distance + if self.preset.in_scope(child_host) or child_event.scope_distance <= self._dns_search_distance: + self.children_emitted.add(child_hash) + # if it's a hostname and it's only one hop away, mark it as affiliate + if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: + child_event.add_tag("affiliate") + self.debug(f"Queueing DNS child for {event}: {child_event}") + await self.emit_event(child_event) + + async def emit_dns_children_raw(self, event): + for rdtype, answers in event.raw_dns_records.items(): + if self.emit_raw_records and rdtype not in ("A", "AAAA", "CNAME", "PTR"): + for answer in answers: + text_answer = answer.to_text() + child_hash = hash(f"{event.host}:{rdtype}:{text_answer}") + if child_hash not in self.children_emitted_raw: + self.children_emitted_raw.add(child_hash) + await self.emit_event( + {"host": str(event.host), "type": rdtype, "answer": text_answer}, + "RAW_DNS_RECORD", + parent=event, + tags=[f"{rdtype.lower()}-record"], + context=f"{rdtype} lookup on {{event.parent.host}} produced {{event.type}}", + ) def check_scope(self, event): whitelisted = False @@ -124,13 +200,14 @@ def check_scope(self, event): # update resolved hosts event.resolved_hosts.update(hosts) for host in hosts: - # having a CNAME to an in-scope resource doesn't make you in-scope + # having a CNAME to an in-scope host doesn't make you in-scope if rdtype != "CNAME": if not whitelisted: with suppress(ValidationError): if self.scan.whitelisted(host): whitelisted = True event.add_tag(f"dns-whitelisted-{rdtype}") + # but a CNAME to a blacklisted host means you're blacklisted if not blacklisted: with suppress(ValidationError): if self.scan.blacklisted(host): @@ -142,36 +219,43 @@ def check_scope(self, event): return whitelisted, blacklisted async def resolve_event(self, event, types): - raw_records = {} + if not types: + return event_host = str(event.host) queries = [(event_host, rdtype) for rdtype in types] dns_errors = {} - async for (query, rdtype), (answer, errors) in self.helpers.dns.resolve_raw_batch(queries): + async for (query, rdtype), (answers, errors) in self.helpers.dns.resolve_raw_batch(queries): + # errors try: dns_errors[rdtype].update(errors) except KeyError: dns_errors[rdtype] = set(errors) - # raw dnspython objects - try: - raw_records[rdtype].add(answer) - except KeyError: - raw_records[rdtype] = {answer} - # hosts - for _rdtype, host in extract_targets(answer): - event.add_tag(f"{_rdtype}-record") + for answer in answers: + # raw dnspython answers try: - event.dns_children[_rdtype].add(host) + event.raw_dns_records[rdtype].add(answer) except KeyError: - event.dns_children[_rdtype] = {host} + event.raw_dns_records[rdtype] = {answer} + # hosts + for _rdtype, host in extract_targets(answer): + event.add_tag(f"{_rdtype}-record") + try: + event.dns_children[_rdtype].add(host) + except KeyError: + event.dns_children[_rdtype] = {host} + # check for private IPs + try: + ip = ipaddress.ip_address(host) + if ip.is_private: + event.add_tag("private-ip") + except ValueError: + continue + # tag event with errors for rdtype, errors in dns_errors.items(): # only consider it an error if there weren't any results for that rdtype if errors and not rdtype in event.dns_children: event.add_tag(f"{rdtype}-error") - return raw_records - - async def handle_wildcard_event(self, event): - pass def get_dns_parent(self, event): """ @@ -181,18 +265,24 @@ def get_dns_parent(self, event): if parent.host == event.host and parent.type in ("IP_ADDRESS", "DNS_NAME", "DNS_NAME_UNRESOLVED"): blacklisted = any(t.startswith("dns-blacklisted-") for t in parent.tags) whitelisted = any(t.startswith("dns-whitelisted-") for t in parent.tags) - return parent, whitelisted, blacklisted, False + new_event = parent is event + return parent, whitelisted, blacklisted, new_event tags = set() if "target" in event.tags: tags.add("target") - return self.scan.make_event( - event.host, - "DNS_NAME", - module=self.host_module, - parent=event, - context="{event.parent.type} has host {event.type}: {event.host}", - tags=tags, - ), None, None, True + return ( + self.scan.make_event( + event.host, + "DNS_NAME", + module=self.host_module, + parent=event, + context="{event.parent.type} has host {event.type}: {event.host}", + tags=tags, + ), + None, + None, + True, + ) @property def emit_raw_records(self): @@ -218,14 +308,3 @@ def _make_dummy_module(self, name): dummy_module.suppress_dupes = False self.scan.dummy_modules[name] = dummy_module return dummy_module - - def _dns_child_dedup_hash(self, parent_host, host, rdtype): - # we deduplicate NS records by their parent domain - # because otherwise every DNS_NAME has one, and it gets super messy - if rdtype == "NS": - _, parent_domain = self.helpers.split_domain(parent_host) - return hash(f"{parent_domain}:{host}") - return hash(f"{parent_host}:{host}:{rdtype}") - - def _main_outgoing_dedup_hash(self, event): - return hash(f"{event.host}") diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 1c9631fac..86110a6cb 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -208,9 +208,9 @@ class bbot_events: return bbot_events -@pytest.fixture(scope="session", autouse=True) -def install_all_python_deps(): - deps_pip = set() - for module in DEFAULT_PRESET.module_loader.preloaded().values(): - deps_pip.update(set(module.get("deps", {}).get("pip", []))) - subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) +# @pytest.fixture(scope="session", autouse=True) +# def install_all_python_deps(): +# deps_pip = set() +# for module in DEFAULT_PRESET.module_loader.preloaded().values(): +# deps_pip.update(set(module.get("deps", {}).get("pip", []))) +# subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index b10fcc544..6d7a83b66 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -38,12 +38,14 @@ async def test_dns_engine(bbot_scanner): results = [_ async for _ in scan.helpers.resolve_raw_batch((("one.one.one.one", "A"), ("1.1.1.1", "PTR")))] pass_1 = False pass_2 = False - for (query, rdtype), (result, errors) in results: - result = extract_targets(result) - _results = [r[1] for r in result] - if query == "one.one.one.one" and "1.1.1.1" in _results: + for (query, rdtype), (answers, errors) in results: + results = [] + for answer in answers: + for t in extract_targets(answer): + results.append(t[1]) + if query == "one.one.one.one" and "1.1.1.1" in query: pass_1 = True - elif query == "1.1.1.1" and "one.one.one.one" in _results: + elif query == "1.1.1.1" and "one.one.one.one" in query: pass_2 = True assert pass_1 and pass_2 @@ -115,7 +117,10 @@ async def test_dns_resolution(bbot_scanner): # custom batch resolution batch_results = [r async for r in dnsengine.resolve_raw_batch([("1.1.1.1", "PTR"), ("one.one.one.one", "A")])] - batch_results = [(response.to_text(), response.rdtype.name) for query, (response, errors) in batch_results] + batch_results = [] + for query, (answers, errors) in batch_results: + for answer in answers: + batch_results.append((answer.to_text(), answer.rdtype.name)) assert len(batch_results) == 3 assert any(answer == "1.0.0.1" and rdtype == "A" for answer, rdtype in batch_results) assert any(answer == "one.one.one.one." and rdtype == "PTR" for answer, rdtype in batch_results) @@ -148,11 +153,7 @@ async def test_dns_resolution(bbot_scanner): resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", parent=scan.root_event) resolved_hosts_event2 = scan.make_event("http://one.one.one.one/", "URL_UNVERIFIED", parent=scan.root_event) dnsresolve = scan.modules["dnsresolve"] - assert hash(resolved_hosts_event1.host) not in dnsresolve._event_cache - assert hash(resolved_hosts_event2.host) not in dnsresolve._event_cache await dnsresolve.handle_event(resolved_hosts_event1) - assert hash(resolved_hosts_event1.host) in dnsresolve._event_cache - assert hash(resolved_hosts_event2.host) in dnsresolve._event_cache await dnsresolve.handle_event(resolved_hosts_event2) assert "1.1.1.1" in resolved_hosts_event2.resolved_hosts # URL event should not have dns_children @@ -160,6 +161,8 @@ async def test_dns_resolution(bbot_scanner): assert resolved_hosts_event1.resolved_hosts == resolved_hosts_event2.resolved_hosts # DNS_NAME event should have dns_children assert "1.1.1.1" in resolved_hosts_event1.dns_children["A"] + assert "A" in resolved_hosts_event1.raw_dns_records + assert "AAAA" in resolved_hosts_event1.raw_dns_records assert "a-record" in resolved_hosts_event1.tags assert not "a-record" in resolved_hosts_event2.tags From be70ae388e0b5496e271307799f1ab82ba7362ad Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 29 Aug 2024 17:22:48 -0400 Subject: [PATCH 012/254] bugfixing --- bbot/core/helpers/dns/brute.py | 14 +++++--------- bbot/modules/templates/subdomain_enum.py | 6 +++--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py index 9b5f55f51..d541f9a67 100644 --- a/bbot/core/helpers/dns/brute.py +++ b/bbot/core/helpers/dns/brute.py @@ -39,18 +39,14 @@ async def dnsbrute(self, module, domain, subdomains, type=None): type = "A" type = str(type).strip().upper() - domain_wildcard_rdtypes = set() - for _domain, rdtypes in (await self.parent_helper.dns.is_wildcard_domain(domain)).items(): - for rdtype, results in rdtypes.items(): - if results: - domain_wildcard_rdtypes.add(rdtype) - if any([r in domain_wildcard_rdtypes for r in (type, "CNAME")]): - self.log.info( - f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})" + wildcard_rdtypes = await self.parent_helper.dns.is_wildcard_domain(domain, (type, "CNAME")) + if wildcard_rdtypes: + self.log.hugewarning( + f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(wildcard_rdtypes)})" ) return [] else: - self.log.debug(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}") + self.log.hugeinfo(f"{domain}: A is not in domain_wildcard_rdtypes:{wildcard_rdtypes}") canaries = self.gen_random_subdomains(self.num_canaries) canaries_list = list(canaries) diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index 28f775d2a..9b9a72ebe 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -119,9 +119,9 @@ async def query(self, query, parse_fn=None, request_fn=None): async def _is_wildcard(self, query): if self.helpers.is_dns_name(query): - for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query)).items(): - if any(t in wildcard_rdtypes for t in ("A", "AAAA", "CNAME")): - return True + wildcard_rdtypes = await self.helpers.is_wildcard_domain(query, rdtypes=("A", "AAAA", "CNAME")) + if wildcard_rdtypes: + return True return False async def filter_event(self, event): From ead400ba9eff0500b5482d0ccc503c0b0a83aecb Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 30 Aug 2024 17:30:41 -0400 Subject: [PATCH 013/254] more WIP dns rework --- bbot/core/event/base.py | 20 +- bbot/core/helpers/dns/dns.py | 10 +- bbot/core/helpers/dns/engine.py | 59 +++-- bbot/core/helpers/dns/mock.py | 33 ++- bbot/modules/internal/dnsresolve.py | 45 ++-- bbot/modules/internal/speculate.py | 2 +- bbot/test/bbot_fixtures.py | 12 +- bbot/test/test_step_1/test_dns.py | 339 ++++++++++++++++++++++++--- bbot/test/test_step_1/test_events.py | 10 + 9 files changed, 431 insertions(+), 99 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 94df01cf5..fde835636 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1054,6 +1054,17 @@ def __init__(self, *args, **kwargs): if parent_module_type == "DNS": self.dns_resolve_distance += 1 # self.add_tag(f"resolve-distance-{self.dns_resolve_distance}") + # tag subdomain / domain + if is_subdomain(self.host): + self.add_tag("subdomain") + elif is_domain(self.host): + self.add_tag("domain") + # tag private IP + try: + if self.host.is_private: + self.add_tag("private-ip") + except AttributeError: + pass class IP_RANGE(DnsEvent): @@ -1070,13 +1081,6 @@ def _host(self): class DNS_NAME(DnsEvent): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if is_subdomain(self.data): - self.add_tag("subdomain") - elif is_domain(self.data): - self.add_tag("domain") - def sanitize_data(self, data): return validators.validate_host(data) @@ -1499,7 +1503,7 @@ class FILESYSTEM(DictPathEvent): pass -class RAW_DNS_RECORD(DictHostEvent): +class RAW_DNS_RECORD(DictHostEvent, DnsEvent): # don't emit raw DNS records for affiliates _always_emit_tags = ["target"] diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index 757474015..43380b746 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -66,7 +66,7 @@ def __init__(self, parent_helper): self.resolver.timeout = self.timeout self.resolver.lifetime = self.timeout - self.runaway_limit = self.config.get("runaway_limit", 5) + self.runaway_limit = self.dns_config.get("runaway_limit", 5) # wildcard handling self.wildcard_disable = self.dns_config.get("wildcard_disable", False) @@ -119,7 +119,7 @@ def brute(self): @async_cachedmethod( lambda self: self._is_wildcard_cache, - key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes))), + key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes)), bool(raw_dns_records)), ) async def is_wildcard(self, query, rdtypes, raw_dns_records=None): """ @@ -194,8 +194,8 @@ def _wildcard_prevalidation(self, host): return host - async def _mock_dns(self, mock_data): + async def _mock_dns(self, mock_data, custom_lookup_fn=None): from .mock import MockResolver - self.resolver = MockResolver(mock_data) - await self.run_and_return("_mock_dns", mock_data=mock_data) + self.resolver = MockResolver(mock_data, custom_lookup_fn=custom_lookup_fn) + await self.run_and_return("_mock_dns", mock_data=mock_data, custom_lookup_fn=custom_lookup_fn) diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index e72b05b80..481e6b076 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -400,6 +400,9 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None): This can reliably tell the difference between a valid DNS record and a wildcard within a wildcard domain. + It works by making a bunch of random DNS queries to the parent domain, compiling a list of wildcard IPs, + then comparing those to the IPs of the host in question. If the host's IP matches the wildcard ones, it's a wildcard. + If you want to know whether a domain is using wildcard DNS, use `is_wildcard_domain()` instead. Args: @@ -413,9 +416,6 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None): Values are tuples where the first element is a boolean indicating if the query is a wildcard, and the second element is the wildcard parent if it's a wildcard. - Raises: - ValueError: If only one of `ips` or `rdtype` is specified or if no valid IPs are specified. - Examples: >>> is_wildcard("www.github.io", rdtypes=["A", "AAAA", "MX"]) {"A": (True, "github.io"), "AAAA": (True, "github.io"), "MX": (False, "github.io")} @@ -445,7 +445,7 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None): else: if errors: self.debug(f"Failed to resolve {query} ({rdtype}) during wildcard detection") - result[rdtype] = (None, query) + result[rdtype] = ("ERROR", query) # clean + process the raw records into a baseline baseline = {} @@ -470,12 +470,15 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None): # once we've resolved the base query and have IP addresses to work with # we can compare the IPs to the ones we have on file for wildcards + # only bother to check the rdypes that actually resolve + rdtypes_to_check = set(raw_dns_records) + # for every parent domain, starting with the shortest parents = list(domain_parents(query)) for parent in parents[::-1]: # check if the parent domain is set up with wildcards - wildcard_results = await self.is_wildcard_domain(parent, rdtypes) + wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check) # for every rdtype for rdtype in list(baseline_raw): @@ -487,18 +490,26 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None): _baseline = baseline.get(rdtype, set()) _baseline_raw = baseline_raw.get(rdtype, set()) - wildcards = wildcard_results.get(rdtype, None) + wildcard_rdtypes = wildcard_results.get(parent, {}) + wildcards = wildcard_rdtypes.get(rdtype, None) if wildcards is None: continue wildcards, wildcard_raw = wildcards - # check if any of our baseline IPs are in the wildcard results - is_wildcard = any(r in wildcards for r in _baseline) - is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw) + if wildcard_raw: + # skip this rdtype from now on + rdtypes_to_check.remove(rdtype) - # if there are any matches, we have a wildcard - if is_wildcard or is_wildcard_raw: - result[rdtype] = (True, query) + # check if any of our baseline IPs are in the wildcard results + is_wildcard = any(r in wildcards for r in _baseline) + is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw) + + # if there are any matches, we have a wildcard + if is_wildcard or is_wildcard_raw: + result[rdtype] = (True, parent) + else: + # otherwise, it's still suspicious, because we had random stuff resolve at this level + result[rdtype] = ("POSSIBLE", parent) # any rdtype that wasn't a wildcard, mark it as False for rdtype, answers in baseline_raw.items(): @@ -531,11 +542,13 @@ async def is_wildcard_domain(self, domain, rdtypes): if isinstance(rdtypes, str): rdtypes = [rdtypes] rdtypes = set(rdtypes) + wildcard_results = {} # make a list of its parents parents = list(domain_parents(domain, include_self=True)) # and check each of them, beginning with the highest parent (i.e. the root domain) for i, host in enumerate(parents[::-1]): + host_results = {} queries = [((host, rdtype), {}) for rdtype in rdtypes] async for ((_, rdtype), _, _), (results, results_raw) in self.task_pool( self._is_wildcard_zone, args_kwargs=queries @@ -543,13 +556,10 @@ async def is_wildcard_domain(self, domain, rdtypes): # if we hit a wildcard, we can skip this rdtype from now on if results_raw: rdtypes.remove(rdtype) - wildcard_results[rdtype] = results, results_raw + host_results[rdtype] = results, results_raw - if wildcard_results: - wildcard_rdtypes_str = ",".join(sorted(wildcard_results)) - self.log.info(f"Encountered domain with wildcard DNS ({wildcard_rdtypes_str}): {host}") - else: - self.log.verbose(f"Finished checking {host}, it is not a wildcard") + if host_results: + wildcard_results[host] = host_results return wildcard_results @@ -587,7 +597,7 @@ async def _is_wildcard_zone(self, host, rdtype): wildcard_results.add(t) if wildcard_results: - self.debug(f"Finished checking {host}:{rdtype}, it is a wildcard") + self.log.info(f"Encountered domain with wildcard DNS ({rdtype}): *.{host}") else: self.debug(f"Finished checking {host}:{rdtype}, it is not a wildcard") self._wildcard_cache[host_hash] = wildcard_results, wildcard_results_raw @@ -643,7 +653,14 @@ def debug(self, *args, **kwargs): def in_tests(self): return os.getenv("BBOT_TESTING", "") == "True" - async def _mock_dns(self, mock_data): + async def _mock_dns(self, mock_data, custom_lookup_fn=None): from .mock import MockResolver - self.resolver = MockResolver(mock_data) + def deserialize_function(func_source): + assert self.in_tests, "Can only mock when BBOT_TESTING=True" + if func_source is None: + return None + exec(func_source) + return locals()["custom_lookup"] + + self.resolver = MockResolver(mock_data, custom_lookup_fn=deserialize_function(custom_lookup_fn)) diff --git a/bbot/core/helpers/dns/mock.py b/bbot/core/helpers/dns/mock.py index 70d978aff..02fdbaa63 100644 --- a/bbot/core/helpers/dns/mock.py +++ b/bbot/core/helpers/dns/mock.py @@ -1,10 +1,14 @@ import dns +import logging + +log = logging.getLogger("bbot.core.helpers.dns.mock") class MockResolver: - def __init__(self, mock_data=None): + def __init__(self, mock_data=None, custom_lookup_fn=None): self.mock_data = mock_data if mock_data else {} + self._custom_lookup_fn = custom_lookup_fn self.nameservers = ["127.0.0.1"] async def resolve_address(self, ipaddr, *args, **kwargs): @@ -13,12 +17,22 @@ async def resolve_address(self, ipaddr, *args, **kwargs): modified_kwargs["rdtype"] = "PTR" return await self.resolve(str(dns.reversename.from_address(ipaddr)), *args, **modified_kwargs) - def create_dns_response(self, query_name, rdtype): - query_name = query_name.strip(".") - answers = self.mock_data.get(query_name, {}).get(rdtype, []) - if not answers: - raise dns.resolver.NXDOMAIN(f"No answer found for {query_name} {rdtype}") + def _lookup(self, query, rdtype): + query = query.strip(".") + ret = [] + if self._custom_lookup_fn is not None: + answers = self._custom_lookup_fn(query, rdtype) + if answers is not None: + ret.extend(list(answers)) + answers = self.mock_data.get(query, {}).get(rdtype, []) + if answers: + ret.extend(list(answers)) + if not ret: + raise dns.resolver.NXDOMAIN(f"No answer found for {query} {rdtype}") + return ret + def create_dns_response(self, query_name, answers, rdtype): + query_name = query_name.strip(".") message_text = f"""id 1234 opcode QUERY rcode NOERROR @@ -27,10 +41,13 @@ def create_dns_response(self, query_name, rdtype): {query_name}. IN {rdtype} ;ANSWER""" for answer in answers: + if answer == "": + answer = '""' message_text += f"\n{query_name}. 1 IN {rdtype} {answer}" message_text += "\n;AUTHORITY\n;ADDITIONAL\n" message = dns.message.from_text(message_text) + log.verbose(message_text) return message async def resolve(self, query_name, rdtype=None): @@ -49,7 +66,9 @@ async def resolve(self, query_name, rdtype=None): raise dns.resolver.NXDOMAIN try: - response = self.create_dns_response(query_name, rdtype) + answers = self._lookup(query_name, rdtype) + log.verbose(f"Answers for {query_name}:{rdtype}: {answers}") + response = self.create_dns_response(query_name, answers, rdtype) answer = dns.resolver.Answer(domain_name, rdtype_obj, dns.rdataclass.IN, response) return answer except dns.resolver.NXDOMAIN: diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 8ffbeb60b..b73a49fba 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -41,6 +41,7 @@ async def setup(self): self.host_module = self.HostModule(self.scan) self.children_emitted = set() self.children_emitted_raw = set() + self.hosts_resolved = set() return True @@ -60,6 +61,7 @@ async def handle_event(self, event, **kwargs): # first, we find or create the main DNS_NAME or IP_ADDRESS associated with this event main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event) + original_tags = set(event.tags) # minimal resolution - first, we resolve A/AAAA records for scope purposes if new_event or event is main_host_event: @@ -79,33 +81,37 @@ async def handle_event(self, event, **kwargs): if main_host_event.scope_distance < self._dns_search_distance: await self.resolve_event(main_host_event, types=non_minimal_rdtypes) # check for wildcards if we're within the scan's search distance - if ( - new_event - and main_host_event.scope_distance <= self.scan.scope_search_distance - and not "domain" in main_host_event.tags - ): + if new_event and main_host_event.scope_distance <= self.scan.scope_search_distance: await self.handle_wildcard_event(main_host_event) + # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}") + + dns_tags = main_host_event.tags.difference(original_tags) + dns_resolve_distance = getattr(main_host_event, "dns_resolve_distance", 0) runaway_dns = dns_resolve_distance >= self.helpers.dns.runaway_limit if runaway_dns: # kill runaway DNS chains - # TODO: test this self.debug( f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})" ) + main_host_event.add_tag(f"runaway-dns-{dns_resolve_distance}") else: # emit dns children await self.emit_dns_children(main_host_event) - await self.emit_dns_children_raw(main_host_event) + await self.emit_dns_children_raw(main_host_event, dns_tags) - # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED - if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags: - main_host_event.type = "DNS_NAME_UNRESOLVED" + # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED + if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags: + main_host_event.type = "DNS_NAME_UNRESOLVED" - # emit the main DNS_NAME or IP_ADDRESS - if new_event and event is not main_host_event and main_host_event.scope_distance <= self._dns_search_distance: - await self.emit_event(main_host_event) + # emit the main DNS_NAME or IP_ADDRESS + if ( + new_event + and event is not main_host_event + and main_host_event.scope_distance <= self._dns_search_distance + ): + await self.emit_event(main_host_event) # transfer scope distance to event event.scope_distance = main_host_event.scope_distance @@ -122,8 +128,9 @@ async def handle_wildcard_event(self, event): elif is_wildcard == True: event.add_tag("wildcard") wildcard_tag = "wildcard" - elif is_wildcard == None: - wildcard_tag = "error" + else: + event.add_tag(f"wildcard-{is_wildcard}") + wildcard_tag = f"wildcard-{is_wildcard}" event.add_tag(f"{rdtype}-{wildcard_tag}") # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) @@ -175,8 +182,10 @@ async def emit_dns_children(self, event): self.debug(f"Queueing DNS child for {event}: {child_event}") await self.emit_event(child_event) - async def emit_dns_children_raw(self, event): + async def emit_dns_children_raw(self, event, dns_tags): for rdtype, answers in event.raw_dns_records.items(): + rdtype_lower = rdtype.lower() + tags = {t for t in dns_tags if rdtype_lower in t.split("-")} if self.emit_raw_records and rdtype not in ("A", "AAAA", "CNAME", "PTR"): for answer in answers: text_answer = answer.to_text() @@ -187,7 +196,7 @@ async def emit_dns_children_raw(self, event): {"host": str(event.host), "type": rdtype, "answer": text_answer}, "RAW_DNS_RECORD", parent=event, - tags=[f"{rdtype.lower()}-record"], + tags=tags, context=f"{rdtype} lookup on {{event.parent.host}} produced {{event.type}}", ) @@ -231,6 +240,7 @@ async def resolve_event(self, event, types): except KeyError: dns_errors[rdtype] = set(errors) for answer in answers: + event.add_tag(f"{rdtype}-record") # raw dnspython answers try: event.raw_dns_records[rdtype].add(answer) @@ -238,7 +248,6 @@ async def resolve_event(self, event, types): event.raw_dns_records[rdtype] = {answer} # hosts for _rdtype, host in extract_targets(answer): - event.add_tag(f"{_rdtype}-record") try: event.dns_children[_rdtype].add(host) except KeyError: diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index bb73094ff..622c0bfe0 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -91,7 +91,7 @@ async def handle_event(self, event): # parent domains if event.type.startswith("DNS_NAME"): - parent = self.helpers.parent_domain(event.data) + parent = self.helpers.parent_domain(event.host_original) if parent != event.data: await self.emit_event( parent, "DNS_NAME", parent=event, context=f"speculated parent {{event.type}}: {{event.data}}" diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 86110a6cb..1c9631fac 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -208,9 +208,9 @@ class bbot_events: return bbot_events -# @pytest.fixture(scope="session", autouse=True) -# def install_all_python_deps(): -# deps_pip = set() -# for module in DEFAULT_PRESET.module_loader.preloaded().values(): -# deps_pip.update(set(module.get("deps", {}).get("pip", []))) -# subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) +@pytest.fixture(scope="session", autouse=True) +def install_all_python_deps(): + deps_pip = set() + for module in DEFAULT_PRESET.module_loader.preloaded().values(): + deps_pip.update(set(module.get("deps", {}).get("pip", []))) + subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 6d7a83b66..517e42dc6 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -43,9 +43,9 @@ async def test_dns_engine(bbot_scanner): for answer in answers: for t in extract_targets(answer): results.append(t[1]) - if query == "one.one.one.one" and "1.1.1.1" in query: + if query == "one.one.one.one" and "1.1.1.1" in results: pass_1 = True - elif query == "1.1.1.1" and "one.one.one.one" in query: + elif query == "1.1.1.1" and "one.one.one.one" in results: pass_2 = True assert pass_1 and pass_2 @@ -117,13 +117,13 @@ async def test_dns_resolution(bbot_scanner): # custom batch resolution batch_results = [r async for r in dnsengine.resolve_raw_batch([("1.1.1.1", "PTR"), ("one.one.one.one", "A")])] - batch_results = [] + batch_results_new = [] for query, (answers, errors) in batch_results: for answer in answers: - batch_results.append((answer.to_text(), answer.rdtype.name)) - assert len(batch_results) == 3 - assert any(answer == "1.0.0.1" and rdtype == "A" for answer, rdtype in batch_results) - assert any(answer == "one.one.one.one." and rdtype == "PTR" for answer, rdtype in batch_results) + batch_results_new.append((answer.to_text(), answer.rdtype.name)) + assert len(batch_results_new) == 3 + assert any(answer == "1.0.0.1" and rdtype == "A" for answer, rdtype in batch_results_new) + assert any(answer == "one.one.one.one." and rdtype == "PTR" for answer, rdtype in batch_results_new) # dns cache dnsengine._dns_cache.clear() @@ -184,40 +184,316 @@ async def test_dns_resolution(bbot_scanner): @pytest.mark.asyncio async def test_wildcards(bbot_scanner): + scan = bbot_scanner("1.1.1.1") helpers = scan.helpers - from bbot.core.helpers.dns.engine import DNSEngine + from bbot.core.helpers.dns.engine import DNSEngine, all_rdtypes - dnsengine = DNSEngine(None) + dnsengine = DNSEngine(None, debug=True) - # wildcards - wildcard_domains = await dnsengine.is_wildcard_domain("asdf.github.io") - assert hash("github.io") in dnsengine._wildcard_cache - assert hash("asdf.github.io") in dnsengine._wildcard_cache + # is_wildcard_domain + wildcard_domains = await dnsengine.is_wildcard_domain("asdf.github.io", all_rdtypes) + assert len(dnsengine._wildcard_cache) == len(all_rdtypes) + (len(all_rdtypes) - 2) + for rdtype in all_rdtypes: + assert hash(f"github.io:{rdtype}") in dnsengine._wildcard_cache + if not rdtype in ("A", "AAAA"): + assert hash(f"asdf.github.io:{rdtype}") in dnsengine._wildcard_cache assert "github.io" in wildcard_domains assert "A" in wildcard_domains["github.io"] assert "SRV" not in wildcard_domains["github.io"] - assert wildcard_domains["github.io"]["A"] and all(helpers.is_ip(r) for r in wildcard_domains["github.io"]["A"]) + assert wildcard_domains["github.io"]["A"] and all(helpers.is_ip(r) for r in wildcard_domains["github.io"]["A"][0]) dnsengine._wildcard_cache.clear() - wildcard_rdtypes = await dnsengine.is_wildcard("blacklanternsecurity.github.io") - assert "A" in wildcard_rdtypes - assert "SRV" not in wildcard_rdtypes - assert wildcard_rdtypes["A"] == (True, "github.io") - assert hash("github.io") in dnsengine._wildcard_cache - assert len(dnsengine._wildcard_cache[hash("github.io")]) > 0 - dnsengine._wildcard_cache.clear() + # is_wildcard + for test_domain in ("blacklanternsecurity.github.io", "asdf.asdf.asdf.github.io"): + wildcard_rdtypes = await dnsengine.is_wildcard(test_domain, all_rdtypes) + assert "A" in wildcard_rdtypes + assert "SRV" not in wildcard_rdtypes + assert wildcard_rdtypes["A"] == (True, "github.io") + assert wildcard_rdtypes["AAAA"] == (True, "github.io") + assert len(dnsengine._wildcard_cache) == 2 + for rdtype in ("A", "AAAA"): + assert hash(f"github.io:{rdtype}") in dnsengine._wildcard_cache + assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")]) == 2 + assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")][0]) > 0 + assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")][1]) > 0 + dnsengine._wildcard_cache.clear() + + ### wildcard TXT record ### + + custom_lookup = """ +def custom_lookup(query, rdtype): + if rdtype == "TXT" and query.strip(".").endswith("test.evilcorp.com"): + return {""} +""" + + mock_data = { + "evilcorp.com": {"A": ["127.0.0.1"]}, + "test.evilcorp.com": {"A": ["127.0.0.2"]}, + "www.test.evilcorp.com": {"AAAA": ["dead::beef"]}, + } + + # basic sanity checks + + await dnsengine._mock_dns(mock_data, custom_lookup_fn=custom_lookup) + + a_result = await dnsengine.resolve("evilcorp.com") + assert a_result == {"127.0.0.1"} + aaaa_result = await dnsengine.resolve("www.test.evilcorp.com", type="AAAA") + assert aaaa_result == {"dead::beef"} + txt_result = await dnsengine.resolve("asdf.www.test.evilcorp.com", type="TXT") + assert txt_result == set() + txt_result_raw, errors = await dnsengine.resolve_raw("asdf.www.test.evilcorp.com", type="TXT") + txt_result_raw = list(txt_result_raw) + assert txt_result_raw + + await dnsengine._shutdown() + + # first, we check with wildcard detection disabled + + scan = bbot_scanner( + "bbot.fdsa.www.test.evilcorp.com", + whitelist=["evilcorp.com"], + config={ + "dns": {"minimal": False, "disable": False, "search_distance": 5, "wildcard_ignore": ["evilcorp.com"]}, + "speculate": True, + }, + ) + await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) - wildcard_rdtypes = await dnsengine.is_wildcard("asdf.asdf.asdf.github.io") - assert "A" in wildcard_rdtypes - assert "SRV" not in wildcard_rdtypes - assert wildcard_rdtypes["A"] == (True, "github.io") - assert hash("github.io") in dnsengine._wildcard_cache - assert not hash("asdf.github.io") in dnsengine._wildcard_cache - assert not hash("asdf.asdf.github.io") in dnsengine._wildcard_cache - assert not hash("asdf.asdf.asdf.github.io") in dnsengine._wildcard_cache - assert len(dnsengine._wildcard_cache[hash("github.io")]) > 0 + events = [e async for e in scan.async_start()] + assert len(events) == 11 + assert len([e for e in events if e.type == "DNS_NAME"]) == 5 + assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 + assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ + "bbot.fdsa.www.test.evilcorp.com", + "evilcorp.com", + "fdsa.www.test.evilcorp.com", + "test.evilcorp.com", + "www.test.evilcorp.com", + ] + + dns_names_by_host = {e.host: e for e in events if e.type == "DNS_NAME"} + assert dns_names_by_host["evilcorp.com"].tags == {"domain", "private-ip", "in-scope", "a-record"} + assert dns_names_by_host["evilcorp.com"].resolved_hosts == {"127.0.0.1"} + assert dns_names_by_host["test.evilcorp.com"].tags == { + "subdomain", + "private-ip", + "in-scope", + "a-record", + "txt-record", + } + assert dns_names_by_host["test.evilcorp.com"].resolved_hosts == {"127.0.0.2"} + assert dns_names_by_host["www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "aaaa-record", "txt-record"} + assert dns_names_by_host["www.test.evilcorp.com"].resolved_hosts == {"dead::beef"} + assert dns_names_by_host["fdsa.www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"} + assert dns_names_by_host["fdsa.www.test.evilcorp.com"].resolved_hosts == set() + assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].tags == { + "target", + "subdomain", + "in-scope", + "txt-record", + } + assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set() + + raw_records_by_host = {e.host: e for e in events if e.type == "RAW_DNS_RECORD"} + assert raw_records_by_host["test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"} + assert raw_records_by_host["test.evilcorp.com"].resolved_hosts == {"127.0.0.2"} + assert raw_records_by_host["www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"} + assert raw_records_by_host["www.test.evilcorp.com"].resolved_hosts == {"dead::beef"} + assert raw_records_by_host["fdsa.www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"} + assert raw_records_by_host["fdsa.www.test.evilcorp.com"].resolved_hosts == set() + assert raw_records_by_host["bbot.fdsa.www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"} + assert raw_records_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set() + + # then we run it again with wildcard detection enabled + + scan = bbot_scanner( + "bbot.fdsa.www.test.evilcorp.com", + whitelist=["evilcorp.com"], + config={ + "dns": {"minimal": False, "disable": False, "search_distance": 5, "wildcard_ignore": []}, + "speculate": True, + }, + ) + await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) + + events = [e async for e in scan.async_start()] + assert len(events) == 11 + assert len([e for e in events if e.type == "DNS_NAME"]) == 5 + assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 + assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ + "_wildcard.test.evilcorp.com", + "bbot.fdsa.www.test.evilcorp.com", + "evilcorp.com", + "test.evilcorp.com", + "www.test.evilcorp.com", + ] + + dns_names_by_host = {e.host: e for e in events if e.type == "DNS_NAME"} + assert dns_names_by_host["evilcorp.com"].tags == {"domain", "private-ip", "in-scope", "a-record"} + assert dns_names_by_host["evilcorp.com"].resolved_hosts == {"127.0.0.1"} + assert dns_names_by_host["test.evilcorp.com"].tags == { + "subdomain", + "private-ip", + "in-scope", + "a-record", + "txt-record", + } + assert dns_names_by_host["test.evilcorp.com"].resolved_hosts == {"127.0.0.2"} + assert dns_names_by_host["_wildcard.test.evilcorp.com"].tags == { + "subdomain", + "in-scope", + "txt-record", + "txt-wildcard", + "wildcard", + } + assert dns_names_by_host["_wildcard.test.evilcorp.com"].resolved_hosts == set() + assert dns_names_by_host["www.test.evilcorp.com"].tags == { + "subdomain", + "in-scope", + "aaaa-record", + "txt-record", + "txt-wildcard", + "wildcard", + } + assert dns_names_by_host["www.test.evilcorp.com"].resolved_hosts == {"dead::beef"} + assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].tags == { + "target", + "subdomain", + "in-scope", + "txt-record", + "txt-wildcard", + "wildcard", + } + assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set() + + raw_records_by_host = {e.host: e for e in events if e.type == "RAW_DNS_RECORD"} + assert raw_records_by_host["test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"} + assert raw_records_by_host["test.evilcorp.com"].resolved_hosts == {"127.0.0.2"} + assert raw_records_by_host["www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record", "txt-wildcard"} + assert raw_records_by_host["www.test.evilcorp.com"].resolved_hosts == {"dead::beef"} + assert raw_records_by_host["_wildcard.test.evilcorp.com"].tags == { + "subdomain", + "in-scope", + "txt-record", + "txt-wildcard", + } + assert raw_records_by_host["_wildcard.test.evilcorp.com"].resolved_hosts == set() + assert raw_records_by_host["bbot.fdsa.www.test.evilcorp.com"].tags == { + "subdomain", + "in-scope", + "txt-record", + "txt-wildcard", + } + assert raw_records_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set() + + ### runaway SRV wildcard ### + + custom_lookup = """ +def custom_lookup(query, rdtype): + if rdtype == "SRV" and query.strip(".").endswith("evilcorp.com"): + return {f"0 100 389 test.{query}"} +""" + + mock_data = { + "evilcorp.com": {"A": ["127.0.0.1"]}, + "test.evilcorp.com": {"AAAA": ["dead::beef"]}, + } + + scan = bbot_scanner( + "evilcorp.com", + config={ + "dns": { + "minimal": False, + "disable": False, + "search_distance": 5, + "wildcard_ignore": [], + "runaway_limit": 3, + }, + }, + ) + await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) + + events = [e async for e in scan.async_start()] + + assert len(events) == 10 + assert len([e for e in events if e.type == "DNS_NAME"]) == 5 + assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 + assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ + "evilcorp.com", + "test.evilcorp.com", + "test.test.evilcorp.com", + "test.test.test.evilcorp.com", + "test.test.test.test.evilcorp.com", + ] + + dns_names_by_host = {e.host: e for e in events if e.type == "DNS_NAME"} + assert dns_names_by_host["evilcorp.com"].tags == { + "target", + "a-record", + "in-scope", + "domain", + "srv-record", + "private-ip", + } + assert dns_names_by_host["test.evilcorp.com"].tags == { + "in-scope", + "srv-record", + "aaaa-record", + "srv-wildcard-possible", + "wildcard-possible", + "subdomain", + } + assert dns_names_by_host["test.test.evilcorp.com"].tags == { + "in-scope", + "srv-record", + "srv-wildcard-possible", + "wildcard-possible", + "subdomain", + } + assert dns_names_by_host["test.test.test.evilcorp.com"].tags == { + "in-scope", + "srv-record", + "srv-wildcard-possible", + "wildcard-possible", + "subdomain", + } + assert dns_names_by_host["test.test.test.test.evilcorp.com"].tags == { + "in-scope", + "srv-record", + "srv-wildcard-possible", + "wildcard-possible", + "subdomain", + "runaway-dns-3", + } + + raw_records_by_host = {e.host: e for e in events if e.type == "RAW_DNS_RECORD"} + assert raw_records_by_host["evilcorp.com"].tags == {"in-scope", "srv-record", "domain"} + assert raw_records_by_host["test.evilcorp.com"].tags == { + "in-scope", + "srv-record", + "srv-wildcard-possible", + "subdomain", + } + assert raw_records_by_host["test.test.evilcorp.com"].tags == { + "in-scope", + "srv-record", + "srv-wildcard-possible", + "subdomain", + } + assert raw_records_by_host["test.test.test.evilcorp.com"].tags == { + "in-scope", + "srv-record", + "srv-wildcard-possible", + "subdomain", + } + + scan = bbot_scanner("1.1.1.1") + helpers = scan.helpers + + # event resolution wildcard_event1 = scan.make_event("wat.asdf.fdsa.github.io", "DNS_NAME", parent=scan.root_event) wildcard_event1.scope_distance = 0 wildcard_event2 = scan.make_event("wats.asd.fdsa.github.io", "DNS_NAME", parent=scan.root_event) @@ -225,9 +501,6 @@ async def test_wildcards(bbot_scanner): wildcard_event3 = scan.make_event("github.io", "DNS_NAME", parent=scan.root_event) wildcard_event3.scope_distance = 0 - await dnsengine._shutdown() - - # event resolution await scan._prep() dnsresolve = scan.modules["dnsresolve"] await dnsresolve.handle_event(wildcard_event1) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 913035d66..94446e71c 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -311,6 +311,16 @@ async def test_events(events, helpers): {"host": "evilcorp.com", "severity": "WACK", "description": "asdf"}, "VULNERABILITY", dummy=True ) + # test tagging + ip_event_1 = scan.make_event("8.8.8.8", dummy=True) + assert "private-ip" not in ip_event_1.tags + ip_event_2 = scan.make_event("192.168.0.1", dummy=True) + assert "private-ip" in ip_event_2.tags + dns_event_1 = scan.make_event("evilcorp.com", dummy=True) + assert "domain" in dns_event_1.tags + dns_event_2 = scan.make_event("www.evilcorp.com", dummy=True) + assert "subdomain" in dns_event_2.tags + # punycode - event type detection # japanese From 5047debe579ee348a94945c56b066701e2d13a2a Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 31 Aug 2024 18:35:11 -0400 Subject: [PATCH 014/254] fix tests --- bbot/core/helpers/dns/engine.py | 8 ++-- bbot/core/helpers/dns/mock.py | 2 +- bbot/modules/internal/dnsresolve.py | 8 +++- bbot/test/test_step_1/test_dns.py | 34 ++++++++--------- .../test_manager_scope_accuracy.py | 38 +++++++++++-------- 5 files changed, 51 insertions(+), 39 deletions(-) diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py index 481e6b076..219339c30 100644 --- a/bbot/core/helpers/dns/engine.py +++ b/bbot/core/helpers/dns/engine.py @@ -206,8 +206,8 @@ async def _resolve_hostname(self, query, **kwargs): retries = kwargs.pop("retries", self.retries) use_cache = kwargs.pop("use_cache", True) tries_left = int(retries) + 1 - parent_hash = hash(f"{parent}:{rdtype}") - dns_cache_hash = hash(f"{query}:{rdtype}") + parent_hash = hash((parent, rdtype)) + dns_cache_hash = hash((query, rdtype)) while tries_left > 0: try: if use_cache: @@ -295,7 +295,7 @@ async def _resolve_ip(self, query, **kwargs): tries_left = int(retries) + 1 results = [] errors = [] - dns_cache_hash = hash(f"{query}:PTR") + dns_cache_hash = hash((query, "PTR")) while tries_left > 0: try: if use_cache: @@ -570,7 +570,7 @@ async def _is_wildcard_zone(self, host, rdtype): rdtype = rdtype.upper() # have we checked this host before? - host_hash = hash(f"{host}:{rdtype}") + host_hash = hash((host, rdtype)) async with self._wildcard_lock.lock(host_hash): # if we've seen this host before try: diff --git a/bbot/core/helpers/dns/mock.py b/bbot/core/helpers/dns/mock.py index 02fdbaa63..17ee2759a 100644 --- a/bbot/core/helpers/dns/mock.py +++ b/bbot/core/helpers/dns/mock.py @@ -47,7 +47,7 @@ def create_dns_response(self, query_name, answers, rdtype): message_text += "\n;AUTHORITY\n;ADDITIONAL\n" message = dns.message.from_text(message_text) - log.verbose(message_text) + # log.verbose(message_text) return message async def resolve(self, query_name, rdtype=None): diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index b73a49fba..b1c10ab2f 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -77,13 +77,17 @@ async def handle_event(self, event, **kwargs): return False, "it has a blacklisted DNS record" if not event_is_ip: - # if we're within our dns search distance, resolve the rest of our records + # if the event is within our dns search distance, resolve the rest of our records if main_host_event.scope_distance < self._dns_search_distance: await self.resolve_event(main_host_event, types=non_minimal_rdtypes) - # check for wildcards if we're within the scan's search distance + # check for wildcards if the event is within the scan's search distance if new_event and main_host_event.scope_distance <= self.scan.scope_search_distance: await self.handle_wildcard_event(main_host_event) + # if there weren't any DNS children and it's not an IP address, tag as unresolved + if not main_host_event.raw_dns_records and not event_is_ip: + main_host_event.add_tag("unresolved") + # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}") dns_tags = main_host_event.tags.difference(original_tags) diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 517e42dc6..648c5445b 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -127,24 +127,24 @@ async def test_dns_resolution(bbot_scanner): # dns cache dnsengine._dns_cache.clear() - assert hash(f"1.1.1.1:PTR") not in dnsengine._dns_cache - assert hash(f"one.one.one.one:A") not in dnsengine._dns_cache - assert hash(f"one.one.one.one:AAAA") not in dnsengine._dns_cache + assert hash(("1.1.1.1", "PTR")) not in dnsengine._dns_cache + assert hash(("one.one.one.one", "A")) not in dnsengine._dns_cache + assert hash(("one.one.one.one", "AAAA")) not in dnsengine._dns_cache await dnsengine.resolve("1.1.1.1", use_cache=False) await dnsengine.resolve("one.one.one.one", use_cache=False) - assert hash(f"1.1.1.1:PTR") not in dnsengine._dns_cache - assert hash(f"one.one.one.one:A") not in dnsengine._dns_cache - assert hash(f"one.one.one.one:AAAA") not in dnsengine._dns_cache + assert hash(("1.1.1.1", "PTR")) not in dnsengine._dns_cache + assert hash(("one.one.one.one", "A")) not in dnsengine._dns_cache + assert hash(("one.one.one.one", "AAAA")) not in dnsengine._dns_cache await dnsengine.resolve("1.1.1.1") - assert hash(f"1.1.1.1:PTR") in dnsengine._dns_cache + assert hash(("1.1.1.1", "PTR")) in dnsengine._dns_cache await dnsengine.resolve("one.one.one.one", type="A") - assert hash(f"one.one.one.one:A") in dnsengine._dns_cache - assert not hash(f"one.one.one.one:AAAA") in dnsengine._dns_cache + assert hash(("one.one.one.one", "A")) in dnsengine._dns_cache + assert not hash(("one.one.one.one", "AAAA")) in dnsengine._dns_cache dnsengine._dns_cache.clear() await dnsengine.resolve("one.one.one.one", type="AAAA") - assert hash(f"one.one.one.one:AAAA") in dnsengine._dns_cache - assert not hash(f"one.one.one.one:A") in dnsengine._dns_cache + assert hash(("one.one.one.one", "AAAA")) in dnsengine._dns_cache + assert not hash(("one.one.one.one", "A")) in dnsengine._dns_cache await dnsengine._shutdown() @@ -196,9 +196,9 @@ async def test_wildcards(bbot_scanner): wildcard_domains = await dnsengine.is_wildcard_domain("asdf.github.io", all_rdtypes) assert len(dnsengine._wildcard_cache) == len(all_rdtypes) + (len(all_rdtypes) - 2) for rdtype in all_rdtypes: - assert hash(f"github.io:{rdtype}") in dnsengine._wildcard_cache + assert hash(("github.io", rdtype)) in dnsengine._wildcard_cache if not rdtype in ("A", "AAAA"): - assert hash(f"asdf.github.io:{rdtype}") in dnsengine._wildcard_cache + assert hash(("asdf.github.io", rdtype)) in dnsengine._wildcard_cache assert "github.io" in wildcard_domains assert "A" in wildcard_domains["github.io"] assert "SRV" not in wildcard_domains["github.io"] @@ -214,10 +214,10 @@ async def test_wildcards(bbot_scanner): assert wildcard_rdtypes["AAAA"] == (True, "github.io") assert len(dnsengine._wildcard_cache) == 2 for rdtype in ("A", "AAAA"): - assert hash(f"github.io:{rdtype}") in dnsengine._wildcard_cache - assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")]) == 2 - assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")][0]) > 0 - assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")][1]) > 0 + assert hash(("github.io", rdtype)) in dnsengine._wildcard_cache + assert len(dnsengine._wildcard_cache[hash(("github.io", rdtype))]) == 2 + assert len(dnsengine._wildcard_cache[hash(("github.io", rdtype))][0]) > 0 + assert len(dnsengine._wildcard_cache[hash(("github.io", rdtype))][1]) > 0 dnsengine._wildcard_cache.clear() ### wildcard TXT record ### diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index bef9b13e6..5af147082 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -383,7 +383,6 @@ def custom_setup(scan): events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( "127.0.0.1/31", modules=["httpx"], - output_modules=["neo4j"], _config={ "dns": {"minimal": False, "search_distance": 2}, "scope": {"search_distance": 0, "report_distance": 1}, @@ -499,7 +498,7 @@ def custom_setup(scan): assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"]) - assert len(all_events) == 23 + assert len(all_events) == 22 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0]) assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -519,9 +518,8 @@ def custom_setup(scan): assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["url"] == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.99:8888/" and e.internal == True and e.scope_distance == 3]) - assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.99" and e.internal == True and e.scope_distance == 3]) - assert len(all_events_nodups) == 21 + assert len(all_events_nodups) == 20 assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0]) assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -541,7 +539,6 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["url"] == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.99:8888/" and e.internal == True and e.scope_distance == 3]) - assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.99" and e.internal == True and e.scope_distance == 3]) for _graph_output_events in (graph_output_events, graph_output_batch_events): assert len(_graph_output_events) == 7 @@ -700,10 +697,11 @@ def custom_setup(scan): _dns_mock={"www.bbottest.notreal": {"A": ["127.0.1.0"]}, "test.notreal": {"A": ["127.0.0.1"]}}, ) - assert len(events) == 6 + assert len(events) == 7 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) - assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) + assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "speculate"]) + assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) @@ -711,10 +709,11 @@ def custom_setup(scan): assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) assert 0 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"]) - assert len(all_events) == 13 + assert len(all_events) == 14 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0]) - assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) + assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "speculate"]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 0]) assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) @@ -736,10 +735,13 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) - assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) + assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "speculate"]) + assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A"]) + assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "speculate"]) + assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) @@ -757,44 +759,50 @@ def custom_setup(scan): _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}}, ) - assert len(events) == 3 + assert len(events) == 4 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) + assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) - assert len(all_events) == 11 + assert len(all_events) == 13 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2]) assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2]) assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) + assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.1.0:9999" and e.internal == True and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) - assert len(all_events_nodups) == 9 + assert len(all_events_nodups) == 11 assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) + assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) + assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.1.0:9999" and e.internal == True and e.scope_distance == 0]) assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"]) assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 5 + assert len(_graph_output_events) == 6 assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) + assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"]) assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) From 485d8279c58fb9d7cb30f800e508eb176097d587 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 31 Aug 2024 18:44:20 -0400 Subject: [PATCH 015/254] remove hugeinfo --- bbot/core/helpers/dns/brute.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py index d541f9a67..168815bf6 100644 --- a/bbot/core/helpers/dns/brute.py +++ b/bbot/core/helpers/dns/brute.py @@ -45,8 +45,6 @@ async def dnsbrute(self, module, domain, subdomains, type=None): f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(wildcard_rdtypes)})" ) return [] - else: - self.log.hugeinfo(f"{domain}: A is not in domain_wildcard_rdtypes:{wildcard_rdtypes}") canaries = self.gen_random_subdomains(self.num_canaries) canaries_list = list(canaries) From 910980e4d547b8a6986104bdc4c3606827b03dad Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 31 Aug 2024 19:12:41 -0400 Subject: [PATCH 016/254] fix stats tests --- bbot/test/test_step_1/test_modules_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 76c2373db..29335482e 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -440,8 +440,8 @@ async def handle_event(self, event): speculate_stats = scan.stats.module_stats["speculate"] assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1} assert speculate_stats.produced_total == 3 - assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 2, "URL_UNVERIFIED": 1} - assert speculate_stats.consumed_total == 4 + assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 2, "URL_UNVERIFIED": 1, "IP_ADDRESS": 2} + assert speculate_stats.consumed_total == 6 @pytest.mark.asyncio From 05e3918263264e6cdd0d33108ab519806c9e72bf Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 31 Aug 2024 20:48:37 -0400 Subject: [PATCH 017/254] more work on tests --- bbot/modules/internal/dnsresolve.py | 3 +- .../test_manager_scope_accuracy.py | 31 ++++++------------- bbot/test/test_step_1/test_modules_basic.py | 2 +- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index b1c10ab2f..f07b833f1 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -102,8 +102,9 @@ async def handle_event(self, event, **kwargs): main_host_event.add_tag(f"runaway-dns-{dns_resolve_distance}") else: # emit dns children - await self.emit_dns_children(main_host_event) await self.emit_dns_children_raw(main_host_event, dns_tags) + if not self.minimal: + await self.emit_dns_children(main_host_event) # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags: diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index 5af147082..e6eaac7ac 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -697,11 +697,10 @@ def custom_setup(scan): _dns_mock={"www.bbottest.notreal": {"A": ["127.0.1.0"]}, "test.notreal": {"A": ["127.0.0.1"]}}, ) - assert len(events) == 7 + assert len(events) == 6 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) - assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "speculate"]) - assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A"]) + assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) @@ -709,11 +708,10 @@ def custom_setup(scan): assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) assert 0 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"]) - assert len(all_events) == 14 + assert len(all_events) == 13 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0]) - assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "speculate"]) - assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) + assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 0]) assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) @@ -735,13 +733,10 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 7 + assert len(_graph_output_events) == 6 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) - assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "speculate"]) - assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A"]) - assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "speculate"]) - assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) + assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) @@ -759,50 +754,44 @@ def custom_setup(scan): _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}}, ) - assert len(events) == 4 + assert len(events) == 3 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) - assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) - assert len(all_events) == 13 + assert len(all_events) == 11 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2]) assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2]) assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) - assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) - assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.1.0:9999" and e.internal == True and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) - assert len(all_events_nodups) == 11 + assert len(all_events_nodups) == 9 assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2]) assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) - assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) - assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.1.0:9999" and e.internal == True and e.scope_distance == 0]) assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"]) assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 5 assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"]) assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"]) - assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"]) assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"]) assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"]) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 29335482e..10cfa5bb6 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -365,7 +365,7 @@ async def handle_event(self, event): scan = bbot_scanner( "evilcorp.com", - config={"speculate": True}, + config={"speculate": True, "dns": {"minimal": False}}, output_modules=["python"], force_start=True, ) From 0c3587a19e0bf984aac5309e78e161e89d4bf95e Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 31 Aug 2024 21:43:45 -0400 Subject: [PATCH 018/254] more work on tests --- bbot/modules/internal/dnsresolve.py | 3 ++ bbot/modules/internal/excavate.py | 35 +++++++++---------- .../test_manager_scope_accuracy.py | 2 -- bbot/test/test_step_1/test_scope.py | 4 +-- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index f07b833f1..f1f215afd 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -17,6 +17,9 @@ class HostModule(BaseModule): _type = "internal" def _outgoing_dedup_hash(self, event): + # this exists to ensure a second, more interesting host isn't passed up + # because its ugly cousin spent its one dedup token before it arrived + # by removing those race conditions, this makes for more consistent results return hash((event, self.name, event.always_emit)) @property diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 794adae25..edb2a766d 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -313,29 +313,28 @@ class excavateTestRule(ExcavateRule): _module_threads = 8 - parameter_blacklist = [ - "__VIEWSTATE", - "__EVENTARGUMENT", - "__EVENTVALIDATION", - "__EVENTTARGET", - "__EVENTARGUMENT", - "__VIEWSTATEGENERATOR", - "__SCROLLPOSITIONY", - "__SCROLLPOSITIONX", - "ASP.NET_SessionId", - "JSESSIONID", - "PHPSESSID", - ] + parameter_blacklist = set( + p.lower() + for p in [ + "__VIEWSTATE", + "__EVENTARGUMENT", + "__EVENTVALIDATION", + "__EVENTTARGET", + "__EVENTARGUMENT", + "__VIEWSTATEGENERATOR", + "__SCROLLPOSITIONY", + "__SCROLLPOSITIONX", + "ASP.NET_SessionId", + "JSESSIONID", + "PHPSESSID", + ] + ) yara_rule_name_regex = re.compile(r"rule\s(\w+)\s{") yara_rule_regex = re.compile(r"(?s)((?:rule\s+\w+\s*{[^{}]*(?:{[^{}]*}[^{}]*)*[^{}]*(?:/\S*?}[^/]*?/)*)*})") def in_bl(self, value): - in_bl = False - for bl_param in self.parameter_blacklist: - if bl_param.lower() == value.lower(): - in_bl = True - return in_bl + return value.lower() in self.parameter_blacklist def url_unparse(self, param_type, parsed_url): if param_type == "GETPARAM": diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index e6eaac7ac..ef254f38c 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -603,8 +603,6 @@ def custom_setup(scan): assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888"]) assert len(all_events) == 29 - for e in all_events: - log.critical(e) assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110" and e.internal == True and e.scope_distance == 0]) assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0]) diff --git a/bbot/test/test_step_1/test_scope.py b/bbot/test/test_step_1/test_scope.py index ebd94333f..7435b82af 100644 --- a/bbot/test/test_step_1/test_scope.py +++ b/bbot/test/test_step_1/test_scope.py @@ -12,7 +12,7 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert len(events) == 5 + assert len(events) == 6 assert 1 == len( [ e @@ -24,7 +24,7 @@ def check(self, module_test, events): ] ) # we have two of these because the host module considers "always_emit" in its outgoing deduplication - assert 1 == len( + assert 2 == len( [ e for e in events From 02b65dd6e75e0fc4217f0744893cecf7fc7a8796 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 31 Aug 2024 22:18:40 -0400 Subject: [PATCH 019/254] fix download error --- bbot/core/helpers/web/engine.py | 28 ++++++++++++++---------- bbot/core/helpers/web/web.py | 23 ++++++++++++++++++-- bbot/test/test_step_1/test_web.py | 36 +++++++++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py index 60e7038aa..6d13d775c 100644 --- a/bbot/core/helpers/web/engine.py +++ b/bbot/core/helpers/web/engine.py @@ -81,15 +81,20 @@ async def request(self, *args, **kwargs): if client_kwargs: client = self.AsyncClient(**client_kwargs) - async with self._acatch(url, raise_error): - if self.http_debug: - log.trace(f"Web request: {str(args)}, {str(kwargs)}") - response = await client.request(*args, **kwargs) - if self.http_debug: - log.trace( - f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}" - ) - return response + try: + async with self._acatch(url, raise_error): + if self.http_debug: + log.trace(f"Web request: {str(args)}, {str(kwargs)}") + response = await client.request(*args, **kwargs) + if self.http_debug: + log.trace( + f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}" + ) + return response + except httpx.HTTPError as e: + if raise_error: + _response = getattr(e, "response", None) + return {"_request_error": str(e), "_response": _response} async def request_batch(self, urls, threads=10, **kwargs): async for (args, _, _), response in self.task_pool( @@ -105,8 +110,8 @@ async def request_custom_batch(self, urls_and_kwargs, threads=10, **kwargs): async def download(self, url, **kwargs): warn = kwargs.pop("warn", True) + raise_error = kwargs.pop("raise_error", False) filename = kwargs.pop("filename") - raise_error = kwargs.get("raise_error", False) try: result = await self.stream_request(url, **kwargs) if result is None: @@ -123,7 +128,8 @@ async def download(self, url, **kwargs): log_fn = log.warning log_fn(f"Failed to download {url}: {e}") if raise_error: - raise + _response = getattr(e, "response", None) + return {"_download_error": str(e), "_response": _response} async def stream_request(self, url, **kwargs): follow_redirects = kwargs.pop("follow_redirects", True) diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py index 66be930c1..5ed7e781e 100644 --- a/bbot/core/helpers/web/web.py +++ b/bbot/core/helpers/web/web.py @@ -121,7 +121,16 @@ async def request(self, *args, **kwargs): Note: If the web request fails, it will return None unless `raise_error` is `True`. """ - return await self.run_and_return("request", *args, **kwargs) + raise_error = kwargs.get("raise_error", False) + result = await self.run_and_return("request", *args, **kwargs) + if isinstance(result, dict) and "_request_error" in result: + if raise_error: + error_msg = result["_request_error"] + response = result["_response"] + error = self.ERROR_CLASS(error_msg) + error.response = response + raise error + return result async def request_batch(self, urls, *args, **kwargs): """ @@ -199,6 +208,7 @@ async def download(self, url, **kwargs): >>> filepath = await self.helpers.download("https://www.evilcorp.com/passwords.docx", cache_hrs=24) """ success = False + raise_error = kwargs.get("raise_error", False) filename = kwargs.pop("filename", self.parent_helper.cache_filename(url)) filename = truncate_filename(Path(filename).resolve()) kwargs["filename"] = filename @@ -211,7 +221,16 @@ async def download(self, url, **kwargs): log.debug(f"{url} is cached at {self.parent_helper.cache_filename(url)}") success = True else: - success = await self.run_and_return("download", url, **kwargs) + result = await self.run_and_return("download", url, **kwargs) + if isinstance(result, dict) and "_download_error" in result: + if raise_error: + error_msg = result["_download_error"] + response = result["_response"] + error = self.ERROR_CLASS(error_msg) + error.response = response + raise error + elif result: + success = True if success: return filename diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index a181306a1..f282b1d62 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -1,4 +1,5 @@ import re +import httpx from ..bbot_fixtures import * @@ -13,6 +14,7 @@ def server_handler(request): base_url = bbot_httpserver.url_for("/test/") bbot_httpserver.expect_request(uri=re.compile(r"/test/\d+")).respond_with_handler(server_handler) + bbot_httpserver.expect_request(uri=re.compile(r"/nope")).respond_with_data("nope", status=500) scan = bbot_scanner() @@ -49,15 +51,45 @@ def server_handler(request): assert response.text.startswith(f"{url}: ") assert f"H{custom_tracker}: v{custom_tracker}" in response.text + # request with raise_error=True + with pytest.raises(WebError): + await scan.helpers.request("http://www.example.com/", raise_error=True) + try: + await scan.helpers.request("http://www.example.com/", raise_error=True) + except WebError as e: + assert hasattr(e, "response") + assert e.response is None + with pytest.raises(httpx.HTTPStatusError): + response = await scan.helpers.request(bbot_httpserver.url_for("/nope"), raise_error=True) + response.raise_for_status() + try: + response = await scan.helpers.request(bbot_httpserver.url_for("/nope"), raise_error=True) + response.raise_for_status() + except httpx.HTTPStatusError as e: + assert hasattr(e, "response") + assert e.response.status_code == 500 + # download url = f"{base_url}999" filename = await scan.helpers.download(url) file_content = open(filename).read() assert file_content.startswith(f"{url}: ") - # raise_error=True + # download with raise_error=True with pytest.raises(WebError): - await scan.helpers.request("http://www.example.com/", raise_error=True) + await scan.helpers.download("http://www.example.com/", raise_error=True) + try: + await scan.helpers.download("http://www.example.com/", raise_error=True) + except WebError as e: + assert hasattr(e, "response") + assert e.response is None + with pytest.raises(WebError): + await scan.helpers.download(bbot_httpserver.url_for("/nope"), raise_error=True) + try: + await scan.helpers.download(bbot_httpserver.url_for("/nope"), raise_error=True) + except WebError as e: + assert hasattr(e, "response") + assert e.response.status_code == 500 await scan._cleanup() From 98b1253d6035683607ec9ac61a66f3443fec0ec0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 1 Sep 2024 23:39:52 -0400 Subject: [PATCH 020/254] prevent subdomain enum modules from touching wildcard-suspicious domains --- bbot/modules/templates/subdomain_enum.py | 7 +- bbot/test/test_step_2/module_tests/base.py | 4 +- .../test_template_subdomain_enum.py | 135 ++++++++++++++++++ 3 files changed, 141 insertions(+), 5 deletions(-) diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index 9b9a72ebe..4f0608fb9 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -118,10 +118,11 @@ async def query(self, query, parse_fn=None, request_fn=None): self.info(f"Error retrieving results for {query}: {e}", trace=True) async def _is_wildcard(self, query): + rdtypes = ("A", "AAAA", "CNAME") if self.helpers.is_dns_name(query): - wildcard_rdtypes = await self.helpers.is_wildcard_domain(query, rdtypes=("A", "AAAA", "CNAME")) - if wildcard_rdtypes: - return True + for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query, rdtypes=rdtypes)).items(): + if any(t in wildcard_rdtypes for t in rdtypes): + return True return False async def filter_event(self, event): diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index 7b8a3b941..bbc9701d8 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -82,10 +82,10 @@ def set_expect_requests(self, expect_args={}, respond_args={}): def set_expect_requests_handler(self, expect_args=None, request_handler=None): self.httpserver.expect_request(expect_args).respond_with_handler(request_handler) - async def mock_dns(self, mock_data, scan=None): + async def mock_dns(self, mock_data, custom_lookup_fn=None, scan=None): if scan is None: scan = self.scan - await scan.helpers.dns._mock_dns(mock_data) + await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup_fn) def mock_interactsh(self, name): from ...conftest import Interactsh_mock diff --git a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py index f6cb3b740..c0bcb25a5 100644 --- a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py +++ b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py @@ -84,3 +84,138 @@ def check(self, module_test, events): "asdf.www.blacklanternsecurity.com", "www.blacklanternsecurity.com", } + + +class TestSubdomainEnumWildcardBaseline(ModuleTestBase): + # oh walmart.cn why are you like this + targets = ["www.walmart.cn"] + whitelist = ["walmart.cn"] + modules_overrides = [] + config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 10}, "omit_event_types": []} + dedup_strategy = "highest_parent" + + dns_mock_data = { + "walmart.cn": {"A": ["127.0.0.1"]}, + "www.walmart.cn": {"A": ["127.0.0.1"]}, + "test.walmart.cn": {"A": ["127.0.0.1"]}, + } + + async def setup_before_prep(self, module_test): + await module_test.mock_dns(self.dns_mock_data) + self.queries = [] + + async def mock_query(query): + self.queries.append(query) + return ["walmart.cn", "www.walmart.cn", "test.walmart.cn", "asdf.walmart.cn"] + + # load subdomain enum template as module + from bbot.modules.templates.subdomain_enum import subdomain_enum + + subdomain_enum_module = subdomain_enum(module_test.scan) + + subdomain_enum_module.query = mock_query + subdomain_enum_module._name = "subdomain_enum" + subdomain_enum_module.dedup_strategy = self.dedup_strategy + module_test.scan.modules["subdomain_enum"] = subdomain_enum_module + + def check(self, module_test, events): + assert self.queries == ["walmart.cn"] + assert len(events) == 6 + assert 2 == len( + [ + e + for e in events + if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A" and e.scope_distance == 1 + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" + and e.data == "www.walmart.cn" + and str(e.module) == "TARGET" + and e.scope_distance == 0 + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" + and e.data == "test.walmart.cn" + and str(e.module) == "subdomain_enum" + and e.scope_distance == 0 + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME_UNRESOLVED" + and e.data == "asdf.walmart.cn" + and str(e.module) == "subdomain_enum" + and e.scope_distance == 0 + ] + ) + + +class TestSubdomainEnumWildcardDefense(TestSubdomainEnumWildcardBaseline): + # oh walmart.cn why are you like this + targets = ["walmart.cn"] + modules_overrides = [] + config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 10}} + dedup_strategy = "highest_parent" + + dns_mock_data = { + "walmart.cn": {"A": ["127.0.0.2"], "TXT": ["asdf.walmart.cn"]}, + } + + async def setup_after_prep(self, module_test): + # simulate wildcard + custom_lookup = """ +def custom_lookup(query, rdtype): + import random + if rdtype == "A" and query.endswith(".walmart.cn"): + ip = ".".join([str(random.randint(0,256)) for _ in range(4)]) + return {ip} +""" + await module_test.mock_dns(self.dns_mock_data, custom_lookup_fn=custom_lookup) + + def check(self, module_test, events): + # no subdomain enum should happen on this domain! + assert self.queries == [] + assert len(events) == 6 + assert 2 == len( + [e for e in events if e.type == "IP_ADDRESS" and str(e.module) == "A" and e.scope_distance == 1] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" + and e.data == "walmart.cn" + and str(e.module) == "TARGET" + and e.scope_distance == 0 + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" + and e.data == "asdf.walmart.cn" + and str(e.module) == "TXT" + and e.scope_distance == 0 + and "wildcard-possible" in e.tags + and "a-wildcard-possible" in e.tags + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "RAW_DNS_RECORD" + and e.data == {"host": "walmart.cn", "type": "TXT", "answer": '"asdf.walmart.cn"'} + ] + ) From dd70fa10355f46b3188a1dac88f97b4e7f46bd34 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Sep 2024 00:10:26 -0400 Subject: [PATCH 021/254] make sure system nameservers are excluded from use by DNS brute force --- bbot/core/helpers/dns/brute.py | 13 ++++++++++--- bbot/defaults.yml | 3 +++ bbot/test/bbot_fixtures.py | 11 ++++++++++- bbot/test/test_step_1/test_dns.py | 11 ++++++++++- bbot/test/test_step_2/module_tests/base.py | 11 ----------- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py index 168815bf6..0c3799ca5 100644 --- a/bbot/core/helpers/dns/brute.py +++ b/bbot/core/helpers/dns/brute.py @@ -15,15 +15,17 @@ class DNSBrute: >>> results = await self.helpers.dns.brute(self, domain, subdomains) """ - nameservers_url = ( + _nameservers_url = ( "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt" ) def __init__(self, parent_helper): self.parent_helper = parent_helper self.log = logging.getLogger("bbot.helper.dns.brute") + self.dns_config = self.parent_helper.config.get("dns", {}) self.num_canaries = 100 - self.max_resolvers = self.parent_helper.config.get("dns", {}).get("brute_threads", 1000) + self.max_resolvers = self.dns_config.get("brute_threads", 1000) + self.nameservers_url = self.dns_config.get("brute_nameservers", self._nameservers_url) self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations) self.digit_regex = self.parent_helper.re.compile(r"\d+") self._resolver_file = None @@ -142,10 +144,15 @@ async def gen_subdomains(self, prefixes, domain): async def resolver_file(self): if self._resolver_file is None: - self._resolver_file = await self.parent_helper.wordlist( + self._resolver_file_original = await self.parent_helper.wordlist( self.nameservers_url, cache_hrs=24 * 7, ) + nameservers = set(self.parent_helper.read_file(self._resolver_file_original)) + nameservers.difference_update(self.parent_helper.dns.system_resolvers) + # exclude system nameservers from brute-force + # this helps prevent rate-limiting which might cause BBOT's main dns queries to fail + self._resolver_file = self.parent_helper.tempfile(nameservers, pipe=False) return self._resolver_file def gen_random_subdomains(self, n=50): diff --git a/bbot/defaults.yml b/bbot/defaults.yml index f57796672..18591bfda 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -34,6 +34,9 @@ dns: # How many concurrent DNS resolvers to use when brute-forcing # (under the hood this is passed through directly to massdns -s) brute_threads: 1000 + # nameservers to use for DNS brute-forcing + # default is updated weekly and contains ~10K high-quality public servers + brute_nameservers: https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt # How far away from the main target to explore via DNS resolution (independent of scope.search_distance) # This is safe to change search_distance: 1 diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 1c9631fac..abad144d1 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -15,7 +15,7 @@ from bbot.errors import * # noqa: F401 from bbot.core import CORE from bbot.scanner import Preset -from bbot.core.helpers.misc import mkdir +from bbot.core.helpers.misc import mkdir, rand_string from bbot.core.helpers.async_helpers import get_event_loop @@ -33,6 +33,15 @@ available_internal_modules = list(DEFAULT_PRESET.module_loader.configs(type="internal")) +def tempwordlist(content): + filename = bbot_test_dir / f"{rand_string(8)}" + with open(filename, "w", errors="ignore") as f: + for c in content: + line = f"{c}\n" + f.write(line) + return filename + + @pytest.fixture def clean_default_config(monkeypatch): clean_config = OmegaConf.merge( diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index 648c5445b..f4bfb218a 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -742,7 +742,8 @@ async def test_dns_graph_structure(bbot_scanner): assert str(events_by_data["evilcorp.com"].module) == "host" -def test_dns_helpers(): +@pytest.mark.asyncio +async def test_dns_helpers(bbot_scanner): assert service_record("") == False assert service_record("localhost") == False assert service_record("www.example.com") == False @@ -753,3 +754,11 @@ def test_dns_helpers(): for srv_record in common_srvs[:100]: hostname = f"{srv_record}.example.com" assert service_record(hostname) == True + + # make sure system nameservers are excluded from use by DNS brute force + brute_nameservers = tempwordlist(["1.2.3.4", "8.8.4.4", "4.3.2.1", "8.8.8.8"]) + scan = bbot_scanner(config={"dns": {"brute_nameservers": brute_nameservers}}) + scan.helpers.dns.system_resolvers = ["8.8.8.8", "8.8.4.4"] + resolver_file = await scan.helpers.dns.brute.resolver_file() + resolvers = set(scan.helpers.read_file(resolver_file)) + assert resolvers == {"1.2.3.4", "4.3.2.1"} diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index bbc9701d8..f5d4255f6 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -11,17 +11,6 @@ log = logging.getLogger("bbot.test.modules") -def tempwordlist(content): - from bbot.core.helpers.misc import rand_string - - filename = bbot_test_dir / f"{rand_string(8)}" - with open(filename, "w", errors="ignore") as f: - for c in content: - line = f"{c}\n" - f.write(line) - return filename - - class ModuleTestBase: targets = ["blacklanternsecurity.com"] scan_name = None From e389d40dbb7d2fd4a15137561d33d65ab676d343 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 05:03:12 +0000 Subject: [PATCH 022/254] Bump mkdocstrings from 0.25.1 to 0.26.0 Bumps [mkdocstrings](https://github.com/mkdocstrings/mkdocstrings) from 0.25.1 to 0.26.0. - [Release notes](https://github.com/mkdocstrings/mkdocstrings/releases) - [Changelog](https://github.com/mkdocstrings/mkdocstrings/blob/main/CHANGELOG.md) - [Commits](https://github.com/mkdocstrings/mkdocstrings/compare/0.25.1...0.26.0) --- updated-dependencies: - dependency-name: mkdocstrings dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 20 ++++++++++---------- pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5eb03255b..447113c02 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1190,13 +1190,13 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp [[package]] name = "mkdocs-autorefs" -version = "1.0.1" +version = "1.2.0" description = "Automatically link across pages in MkDocs." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_autorefs-1.0.1-py3-none-any.whl", hash = "sha256:aacdfae1ab197780fb7a2dac92ad8a3d8f7ca8049a9cbe56a4218cd52e8da570"}, - {file = "mkdocs_autorefs-1.0.1.tar.gz", hash = "sha256:f684edf847eced40b570b57846b15f0bf57fb93ac2c510450775dcf16accb971"}, + {file = "mkdocs_autorefs-1.2.0-py3-none-any.whl", hash = "sha256:d588754ae89bd0ced0c70c06f58566a4ee43471eeeee5202427da7de9ef85a2f"}, + {file = "mkdocs_autorefs-1.2.0.tar.gz", hash = "sha256:a86b93abff653521bda71cf3fc5596342b7a23982093915cb74273f67522190f"}, ] [package.dependencies] @@ -1279,24 +1279,24 @@ files = [ [[package]] name = "mkdocstrings" -version = "0.25.1" +version = "0.26.0" description = "Automatic documentation from sources, for MkDocs." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocstrings-0.25.1-py3-none-any.whl", hash = "sha256:da01fcc2670ad61888e8fe5b60afe9fee5781017d67431996832d63e887c2e51"}, - {file = "mkdocstrings-0.25.1.tar.gz", hash = "sha256:c3a2515f31577f311a9ee58d089e4c51fc6046dbd9e9b4c3de4c3194667fe9bf"}, + {file = "mkdocstrings-0.26.0-py3-none-any.whl", hash = "sha256:1aa227fe94f88e80737d37514523aacd473fc4b50a7f6852ce41447ab23f2654"}, + {file = "mkdocstrings-0.26.0.tar.gz", hash = "sha256:ff9d0de28c8fa877ed9b29a42fe407cfe6736d70a1c48177aa84fcc3dc8518cd"}, ] [package.dependencies] click = ">=7.0" importlib-metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} Jinja2 = ">=2.11.1" -Markdown = ">=3.3" +Markdown = ">=3.6" MarkupSafe = ">=1.1" mkdocs = ">=1.4" -mkdocs-autorefs = ">=0.3.1" -platformdirs = ">=2.2.0" +mkdocs-autorefs = ">=1.2" +platformdirs = ">=2.2" pymdown-extensions = ">=6.3" typing-extensions = {version = ">=4.1", markers = "python_version < \"3.10\""} @@ -2973,4 +2973,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "d5a58b845248d60d5cfc5111d6e611486e9137479952f180e2e01d719e440746" +content-hash = "c4d16176c75de710966e2d5dadfa8f0c181d90f9ad3ca4fd250702dc23909b98" diff --git a/pyproject.toml b/pyproject.toml index 1bcfe4540..4006d67ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ mkdocs = "^1.5.2" mkdocs-extra-sass-plugin = "^0.1.0" mkdocs-material = "^9.2.5" mkdocs-material-extensions = "^1.1.1" -mkdocstrings = ">=0.22,<0.26" +mkdocstrings = ">=0.22,<0.27" mkdocstrings-python = "^1.6.0" livereload = "^2.6.3" mike = "^2.1.2" From f33c2e77e7d19679a80d0b3082e9a7393cbca723 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 05:03:31 +0000 Subject: [PATCH 023/254] Bump cloudcheck from 5.0.1.515 to 5.0.1.537 Bumps [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck) from 5.0.1.515 to 5.0.1.537. - [Commits](https://github.com/blacklanternsecurity/cloudcheck/commits) --- updated-dependencies: - dependency-name: cloudcheck dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5eb03255b..ed05c530c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -387,13 +387,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cloudcheck" -version = "5.0.1.515" +version = "5.0.1.537" description = "Check whether an IP address belongs to a cloud provider" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "cloudcheck-5.0.1.515-py3-none-any.whl", hash = "sha256:427ee423b9abca9f742f21300c3968dde8784cbfdd99ba69b336a0a6723fe677"}, - {file = "cloudcheck-5.0.1.515.tar.gz", hash = "sha256:64c7c22567a3ae14731b4826c631585a9714858cc9dd70fafbfa40b5cef37049"}, + {file = "cloudcheck-5.0.1.537-py3-none-any.whl", hash = "sha256:ef5c3326bdacfd7048ecc4b1849e728ca705163e7e4713ba4546493af491024a"}, + {file = "cloudcheck-5.0.1.537.tar.gz", hash = "sha256:8884dd9c3de0f634a191559255f3370222bfde5929b10d356415b58291f5d639"}, ] [package.dependencies] From 8c801d675cc04f71e691403a4b33a05342fb84c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 05:04:09 +0000 Subject: [PATCH 024/254] Bump httpx from 0.27.0 to 0.27.2 Bumps [httpx](https://github.com/encode/httpx) from 0.27.0 to 0.27.2. - [Release notes](https://github.com/encode/httpx/releases) - [Changelog](https://github.com/encode/httpx/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/httpx/compare/0.27.0...0.27.2) --- updated-dependencies: - dependency-name: httpx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5eb03255b..6abd3c2e6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -719,13 +719,13 @@ trio = ["trio (>=0.22.0,<0.26.0)"] [[package]] name = "httpx" -version = "0.27.0" +version = "0.27.2" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"}, - {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"}, + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, ] [package.dependencies] @@ -740,6 +740,7 @@ brotli = ["brotli", "brotlicffi"] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "identify" From 575a4f391a33124f3f26b431705c7fcc2be66039 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 05:04:27 +0000 Subject: [PATCH 025/254] Bump mkdocs-material from 9.5.33 to 9.5.34 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.33 to 9.5.34. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.33...9.5.34) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5eb03255b..0efb4206b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1239,13 +1239,13 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.33" +version = "9.5.34" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.33-py3-none-any.whl", hash = "sha256:dbc79cf0fdc6e2c366aa987de8b0c9d4e2bb9f156e7466786ba2fd0f9bf7ffca"}, - {file = "mkdocs_material-9.5.33.tar.gz", hash = "sha256:d23a8b5e3243c9b2f29cdfe83051104a8024b767312dc8fde05ebe91ad55d89d"}, + {file = "mkdocs_material-9.5.34-py3-none-any.whl", hash = "sha256:54caa8be708de2b75167fd4d3b9f3d949579294f49cb242515d4653dbee9227e"}, + {file = "mkdocs_material-9.5.34.tar.gz", hash = "sha256:1e60ddf716cfb5679dfd65900b8a25d277064ed82d9a53cd5190e3f894df7840"}, ] [package.dependencies] From 402b37283a003cb8219c62f98b37cfd1b8d614ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 05:05:03 +0000 Subject: [PATCH 026/254] Bump mkdocstrings-python from 1.10.8 to 1.10.9 Bumps [mkdocstrings-python](https://github.com/mkdocstrings/python) from 1.10.8 to 1.10.9. - [Release notes](https://github.com/mkdocstrings/python/releases) - [Changelog](https://github.com/mkdocstrings/python/blob/main/CHANGELOG.md) - [Commits](https://github.com/mkdocstrings/python/compare/1.10.8...1.10.9) --- updated-dependencies: - dependency-name: mkdocstrings-python dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5eb03255b..1acdf04db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1307,17 +1307,18 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] [[package]] name = "mkdocstrings-python" -version = "1.10.8" +version = "1.10.9" description = "A Python handler for mkdocstrings." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocstrings_python-1.10.8-py3-none-any.whl", hash = "sha256:bb12e76c8b071686617f824029cb1dfe0e9afe89f27fb3ad9a27f95f054dcd89"}, - {file = "mkdocstrings_python-1.10.8.tar.gz", hash = "sha256:5856a59cbebbb8deb133224a540de1ff60bded25e54d8beacc375bb133d39016"}, + {file = "mkdocstrings_python-1.10.9-py3-none-any.whl", hash = "sha256:cbe98710a6757dfd4dff79bf36cb9731908fb4c69dd2736b15270ae7a488243d"}, + {file = "mkdocstrings_python-1.10.9.tar.gz", hash = "sha256:f344aaa47e727d8a2dc911e063025e58e2b7fb31a41110ccc3902aa6be7ca196"}, ] [package.dependencies] griffe = ">=0.49" +mkdocs-autorefs = ">=1.0" mkdocstrings = ">=0.25" [[package]] From b04b047478b9afb7f5b7581c20f7a9acb52f9dd5 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 2 Sep 2024 17:38:49 +0100 Subject: [PATCH 027/254] Changed webhook to POST an adaptive card --- bbot/modules/output/teams.py | 107 ++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index 64991f46d..b7731c2c4 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -1,3 +1,5 @@ +import yaml + from bbot.modules.templates.webhook import WebhookOutputModule @@ -10,13 +12,114 @@ class Teams(WebhookOutputModule): } options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"} options_desc = { - "webhook_url": "Discord webhook URL", + "webhook_url": "Teams webhook URL", "event_types": "Types of events to send", "min_severity": "Only allow VULNERABILITY events of this severity or higher", } _module_threads = 5 good_status_code = 200 - content_key = "text" + adaptive_card = { + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "contentUrl": None, + "content": { + "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.2", + "body": [], + }, + } + ], + } + + async def handle_event(self, event): + while 1: + data = self.format_message(self.adaptive_card.copy(), event) + + response = await self.helpers.request( + url=self.webhook_url, + method="POST", + json=data, + ) + status_code = getattr(response, "status_code", 0) + if self.evaluate_response(response): + break + else: + response_data = getattr(response, "text", "") + try: + retry_after = response.json().get("retry_after", 1) + except Exception: + retry_after = 1 + self.verbose( + f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds" + ) + await self.helpers.sleep(retry_after) + + def format_message_str(self, event): + items = [] + msg = event.data + if len(msg) > self.message_size_limit: + msg = msg[: self.message_size_limit - 3] + "..." + items.append({"type": "TextBlock", "text": f"{msg}", "wrap": True}) + items.append({"type": "FactSet", "facts": [{"title": "Tags:", "value": ", ".join(event.tags)}]}) + return items + + def format_message_other(self, event): + event_yaml = yaml.dump(event.data) + msg = event_yaml + if len(msg) > self.message_size_limit: + msg = msg[: self.message_size_limit - 3] + "..." + return [{"type": "TextBlock", "text": f"{msg}", "wrap": True}] + + def get_severity_color(self, event): + color = "Accent" + if event.type == "VULNERABILITY": + severity = event.data.get("severity", "UNKNOWN") + if severity == "CRITICAL": + color = "Attention" + elif severity == "HIGH": + color = "Attention" + elif severity == "MEDIUM": + color = "Warning" + elif severity == "LOW": + color = "Good" + return color + + def format_message(self, adaptive_card, event): + heading = {"type": "TextBlock", "text": f"{event.type}", "wrap": True, "style": "heading"} + body = adaptive_card["attachments"][0]["content"]["body"] + body.append(heading) + if event.type in ("VULNERABILITY", "FINDING"): + subheading = { + "type": "TextBlock", + "text": event.data.get("severity", "UNKNOWN"), + "spacing": "None", + "size": "Large", + "wrap": True, + } + subheading["color"] = self.get_severity_color(self, event) + main_text = { + "type": "ColumnSet", + "separator": True, + "spacing": "Medium", + "columns": [ + { + "type": "Column", + "width": "stretch", + "items": [], + } + ], + } + if isinstance(event.data, str): + items = self.format_message_str(event) + else: + items = self.format_message_other(event) + for item in items: + main_text["columns"][0]["items"].append(item) + body.append(main_text) + return adaptive_card def evaluate_response(self, response): text = getattr(response, "text", "") From 8807c168f97eeef8e4d45a68f07b75d9aefcd134 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 2 Sep 2024 20:01:23 +0100 Subject: [PATCH 028/254] Change `format_message_other()` to return a `FactSet` so keys are bold. Message is still trimmed using `trim_message` --- bbot/modules/output/teams.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index b7731c2c4..4ec9f1c1e 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -56,22 +56,25 @@ async def handle_event(self, event): f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds" ) await self.helpers.sleep(retry_after) + + def trim_message(self, message): + if len(message) > self.message_size_limit: + message = message[: self.message_size_limit - 3] + "..." + return message def format_message_str(self, event): items = [] - msg = event.data - if len(msg) > self.message_size_limit: - msg = msg[: self.message_size_limit - 3] + "..." + msg = self.trim_message(event.data) items.append({"type": "TextBlock", "text": f"{msg}", "wrap": True}) items.append({"type": "FactSet", "facts": [{"title": "Tags:", "value": ", ".join(event.tags)}]}) return items def format_message_other(self, event): - event_yaml = yaml.dump(event.data) - msg = event_yaml - if len(msg) > self.message_size_limit: - msg = msg[: self.message_size_limit - 3] + "..." - return [{"type": "TextBlock", "text": f"{msg}", "wrap": True}] + items = [{"type": "FactSet", "facts": []}] + for key, value in event.data.items(): + msg = self.trim_message(str(value)) + items[0]["facts"].append({"title": f"{key}:", "value": msg}) + return items def get_severity_color(self, event): color = "Accent" @@ -116,8 +119,7 @@ def format_message(self, adaptive_card, event): items = self.format_message_str(event) else: items = self.format_message_other(event) - for item in items: - main_text["columns"][0]["items"].append(item) + main_text["columns"][0]["items"] = items body.append(main_text) return adaptive_card From 9ad1e8fb383aa4c1263871e419a1acb835a584ff Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Sep 2024 15:05:26 -0400 Subject: [PATCH 029/254] encrypt sudo password --- bbot/core/core.py | 2 - bbot/core/helpers/command.py | 2 +- bbot/core/helpers/depsinstaller/installer.py | 61 +++++++++++++++---- .../helpers/depsinstaller/sudo_askpass.py | 38 +++++++++++- 4 files changed, 86 insertions(+), 17 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index e7eacf18d..47831af25 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -34,8 +34,6 @@ def __init__(self): self._logger = None self._files_config = None - self.bbot_sudo_pass = None - self._config = None self._custom_config = None diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py index 6f43a401d..16f9c9131 100644 --- a/bbot/core/helpers/command.py +++ b/bbot/core/helpers/command.py @@ -295,7 +295,7 @@ def _prepare_command_kwargs(self, command, kwargs): if sudo and os.geteuid() != 0: self.depsinstaller.ensure_root() env["SUDO_ASKPASS"] = str((self.tools_dir / self.depsinstaller.askpass_filename).resolve()) - env["BBOT_SUDO_PASS"] = self.depsinstaller._sudo_password + env["BBOT_SUDO_PASS"] = self.depsinstaller.encrypted_sudo_pw kwargs["env"] = env PATH = os.environ.get("PATH", "") diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index f17c96499..4416d9738 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -10,10 +10,11 @@ from threading import Lock from itertools import chain from contextlib import suppress +from secrets import token_bytes from ansible_runner.interface import run from subprocess import CalledProcessError -from ..misc import can_sudo_without_password, os_platform +from ..misc import can_sudo_without_password, os_platform, rm_at_exit log = logging.getLogger("bbot.core.helpers.depsinstaller") @@ -29,14 +30,13 @@ def __init__(self, parent_helper): http_timeout = self.web_config.get("http_timeout", 30) os.environ["ANSIBLE_TIMEOUT"] = str(http_timeout) + # cache encrypted sudo pass self.askpass_filename = "sudo_askpass.py" + self._sudo_password = None + self._sudo_cache_setup = False + self._setup_sudo_cache() self._installed_sudo_askpass = False - self._sudo_password = os.environ.get("BBOT_SUDO_PASS", None) - if self._sudo_password is None: - if self.core.bbot_sudo_pass is not None: - self._sudo_password = self.core.bbot_sudo_pass - elif can_sudo_without_password(): - self._sudo_password = "" + self.data_dir = self.parent_helper.cache_dir / "depsinstaller" self.parent_helper.mkdir(self.data_dir) self.setup_status_cache = self.data_dir / "setup_status.json" @@ -314,18 +314,25 @@ def write_setup_status(self): def ensure_root(self, message=""): self._install_sudo_askpass() + # skip if we've already done this + if self._sudo_password is not None: + return with self.ensure_root_lock: - if os.geteuid() != 0 and self._sudo_password is None: + # first check if the environment variable is set + _sudo_password = os.environ.get("BBOT_SUDO_PASS", None) + if _sudo_password is not None or can_sudo_without_password(): + # if we can sudo without a password, there's no need to prompt + return + if os.geteuid() != 0: if message: log.warning(message) while not self._sudo_password: # sleep for a split second to flush previous log messages sleep(0.1) - password = getpass.getpass(prompt="[USER] Please enter sudo password: ") - if self.parent_helper.verify_sudo_password(password): + _sudo_password = getpass.getpass(prompt="[USER] Please enter sudo password: ") + if self.parent_helper.verify_sudo_password(_sudo_password): log.success("Authentication successful") - self._sudo_password = password - self.core.bbot_sudo_pass = password + self._sudo_password = _sudo_password else: log.warning("Incorrect password") @@ -343,6 +350,36 @@ def install_core_deps(self): self.ensure_root() self.apt_install(list(to_install)) + def _setup_sudo_cache(self): + if not self._sudo_cache_setup: + self._sudo_cache_setup = True + # write temporary encryption key, to be deleted upon scan completion + self._sudo_temp_keyfile = self.parent_helper.temp_filename() + # remove it at exit + rm_at_exit(self._sudo_temp_keyfile) + # generate random 32-byte key + random_key = token_bytes(32) + # write key to file and set secure permissions + self._sudo_temp_keyfile.write_bytes(random_key) + self._sudo_temp_keyfile.chmod(0o600) + # export path to environment variable, for use in askpass script + os.environ["BBOT_SUDO_KEYFILE"] = str(self._sudo_temp_keyfile.resolve()) + + @property + def encrypted_sudo_pw(self): + return self._encrypt_sudo_pw(self._sudo_password) + + def _encrypt_sudo_pw(self, pw): + from Crypto.Cipher import AES + from Crypto.Util.Padding import pad + + key = self._sudo_temp_keyfile.read_bytes() + cipher = AES.new(key, AES.MODE_CBC) + ct_bytes = cipher.encrypt(pad(pw.encode(), AES.block_size)) + iv = cipher.iv.hex() + ct = ct_bytes.hex() + return f"{iv}:{ct}" + def _install_sudo_askpass(self): if not self._installed_sudo_askpass: self._installed_sudo_askpass = True diff --git a/bbot/core/helpers/depsinstaller/sudo_askpass.py b/bbot/core/helpers/depsinstaller/sudo_askpass.py index 42eccc167..cb00338e6 100644 --- a/bbot/core/helpers/depsinstaller/sudo_askpass.py +++ b/bbot/core/helpers/depsinstaller/sudo_askpass.py @@ -1,5 +1,39 @@ #!/usr/bin/env python3 - import os +import sys +from pathlib import Path +from Crypto.Cipher import AES +from Crypto.Util.Padding import unpad + +ENV_VAR_NAME = "BBOT_SUDO_PASS" +KEY_ENV_VAR_PATH = "BBOT_SUDO_KEYFILE" + + +def decrypt_password(encrypted_data, key): + iv, ciphertext = encrypted_data.split(":") + iv = bytes.fromhex(iv) + ct = bytes.fromhex(ciphertext) + cipher = AES.new(key, AES.MODE_CBC, iv) + pt = unpad(cipher.decrypt(ct), AES.block_size) + return pt.decode("utf-8") + + +def main(): + encrypted_password = os.environ.get(ENV_VAR_NAME, "") + encryption_keypath = Path(os.environ.get(KEY_ENV_VAR_PATH, "")) + + if not encrypted_password or not encryption_keypath.is_file(): + print("Error: Encrypted password or encryption key not found in environment variables.", file=sys.stderr) + sys.exit(1) + + try: + key = encryption_keypath.read_bytes() + decrypted_password = decrypt_password(encrypted_password, key) + print(decrypted_password, end="") + except Exception as e: + print(f'Error decrypting password "{encrypted_password}": {str(e)}', file=sys.stderr) + sys.exit(1) + -print(os.environ.get("BBOT_SUDO_PASS", ""), end="") +if __name__ == "__main__": + main() From 81bb2438db52f6e6df94b396a252386e54a16995 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 2 Sep 2024 20:08:13 +0100 Subject: [PATCH 030/254] Lint and fix syntax error --- bbot/modules/output/teams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index 4ec9f1c1e..ec549c2e5 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -56,7 +56,7 @@ async def handle_event(self, event): f"Error sending {event}: status code {status_code}, response: {response_data}, retrying in {retry_after} seconds" ) await self.helpers.sleep(retry_after) - + def trim_message(self, message): if len(message) > self.message_size_limit: message = message[: self.message_size_limit - 3] + "..." @@ -102,7 +102,7 @@ def format_message(self, adaptive_card, event): "size": "Large", "wrap": True, } - subheading["color"] = self.get_severity_color(self, event) + subheading["color"] = self.get_severity_color(event) main_text = { "type": "ColumnSet", "separator": True, From 3448b47a5219db4c44e7be317ef0a0829e904581 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 2 Sep 2024 20:16:06 +0100 Subject: [PATCH 031/254] Remove unused import --- bbot/modules/output/teams.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index ec549c2e5..b095abc4b 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -1,5 +1,3 @@ -import yaml - from bbot.modules.templates.webhook import WebhookOutputModule From 7d5f3a5459b5e6f3b98fe6b51e1fcd04a89c241f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 21:06:49 +0000 Subject: [PATCH 032/254] Bump mkdocs from 1.6.0 to 1.6.1 Bumps [mkdocs](https://github.com/mkdocs/mkdocs) from 1.6.0 to 1.6.1. - [Release notes](https://github.com/mkdocs/mkdocs/releases) - [Commits](https://github.com/mkdocs/mkdocs/compare/1.6.0...1.6.1) --- updated-dependencies: - dependency-name: mkdocs dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4bfa8d64f..ef8391cec 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1160,13 +1160,13 @@ test = ["coverage", "flake8 (>=3.0)", "flake8-quotes", "shtab"] [[package]] name = "mkdocs" -version = "1.6.0" +version = "1.6.1" description = "Project documentation with Markdown." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs-1.6.0-py3-none-any.whl", hash = "sha256:1eb5cb7676b7d89323e62b56235010216319217d4af5ddc543a91beb8d125ea7"}, - {file = "mkdocs-1.6.0.tar.gz", hash = "sha256:a73f735824ef83a4f3bcb7a231dcab23f5a838f88b7efc54a0eef5fbdbc3c512"}, + {file = "mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e"}, + {file = "mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2"}, ] [package.dependencies] From 2b30d1b3a7f895b188ae936d5ed0399fcaf06982 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Sep 2024 18:02:25 -0400 Subject: [PATCH 033/254] fix command tests --- bbot/core/helpers/depsinstaller/installer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 4416d9738..e20022d85 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -367,6 +367,8 @@ def _setup_sudo_cache(self): @property def encrypted_sudo_pw(self): + if self._sudo_password is None: + return "" return self._encrypt_sudo_pw(self._sudo_password) def _encrypt_sudo_pw(self, pw): From e04e681d270f8f0c5208597170eead888392759b Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Sep 2024 19:58:48 -0400 Subject: [PATCH 034/254] remove var from env once we're finished with it --- bbot/core/helpers/depsinstaller/sudo_askpass.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbot/core/helpers/depsinstaller/sudo_askpass.py b/bbot/core/helpers/depsinstaller/sudo_askpass.py index cb00338e6..ccd8cd01b 100644 --- a/bbot/core/helpers/depsinstaller/sudo_askpass.py +++ b/bbot/core/helpers/depsinstaller/sudo_askpass.py @@ -20,6 +20,8 @@ def decrypt_password(encrypted_data, key): def main(): encrypted_password = os.environ.get(ENV_VAR_NAME, "") + # remove variable from environment once we've got it + os.environ.pop(ENV_VAR_NAME, None) encryption_keypath = Path(os.environ.get(KEY_ENV_VAR_PATH, "")) if not encrypted_password or not encryption_keypath.is_file(): From ecac174ec7633d3c7f2b063ed92dbb75e4cd5e59 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Sep 2024 20:16:06 -0400 Subject: [PATCH 035/254] cache yara regexes --- bbot/scanner/scanner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index ba550f217..111b6bf80 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -231,6 +231,7 @@ def __init__( self._dns_strings = None self._dns_regexes = None + self._dns_regexes_yara = None self.__log_handlers = None self._log_handler_backup = [] @@ -1000,7 +1001,7 @@ def dns_regexes(self): ... for match in regex.finditer(response.text): ... hostname = match.group().lower() """ - if not self._dns_regexes: + if self._dns_regexes is None: self._dns_regexes = self._generate_dns_regexes(r"((?:(?:[\w-]+)\.)+") return self._dns_regexes @@ -1009,7 +1010,9 @@ def dns_regexes_yara(self): """ Returns a list of DNS hostname regexes formatted specifically for compatibility with YARA rules. """ - return self._generate_dns_regexes(r"(([a-z0-9-]+\.)+") + if self._dns_regexes_yara is None: + self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)+") + return self._dns_regexes_yara @property def json(self): From b9917242ac0fae0b113eae0f22e392da22c24d5e Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 2 Sep 2024 22:37:28 -0400 Subject: [PATCH 036/254] clean up code --- bbot/core/helpers/depsinstaller/installer.py | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index e20022d85..8df5f05cf 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -320,21 +320,21 @@ def ensure_root(self, message=""): with self.ensure_root_lock: # first check if the environment variable is set _sudo_password = os.environ.get("BBOT_SUDO_PASS", None) - if _sudo_password is not None or can_sudo_without_password(): - # if we can sudo without a password, there's no need to prompt + if _sudo_password is not None or os.geteuid() == 0 or can_sudo_without_password(): + # if we're already root or we can sudo without a password, there's no need to prompt return - if os.geteuid() != 0: - if message: - log.warning(message) - while not self._sudo_password: - # sleep for a split second to flush previous log messages - sleep(0.1) - _sudo_password = getpass.getpass(prompt="[USER] Please enter sudo password: ") - if self.parent_helper.verify_sudo_password(_sudo_password): - log.success("Authentication successful") - self._sudo_password = _sudo_password - else: - log.warning("Incorrect password") + + if message: + log.warning(message) + while not self._sudo_password: + # sleep for a split second to flush previous log messages + sleep(0.1) + _sudo_password = getpass.getpass(prompt="[USER] Please enter sudo password: ") + if self.parent_helper.verify_sudo_password(_sudo_password): + log.success("Authentication successful") + self._sudo_password = _sudo_password + else: + log.warning("Incorrect password") def install_core_deps(self): to_install = set() From 262a93852f5036ebd64fde904df7be749cd74412 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Tue, 3 Sep 2024 09:42:10 +0100 Subject: [PATCH 037/254] Corrected valid response and test and tweaked teams card --- bbot/modules/output/teams.py | 15 +++++++-------- .../test_step_2/module_tests/test_module_teams.py | 5 +---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index b095abc4b..25cf24001 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -15,7 +15,7 @@ class Teams(WebhookOutputModule): "min_severity": "Only allow VULNERABILITY events of this severity or higher", } _module_threads = 5 - good_status_code = 200 + good_status_code = 202 adaptive_card = { "type": "message", "attachments": [ @@ -26,6 +26,7 @@ class Teams(WebhookOutputModule): "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", "type": "AdaptiveCard", "version": "1.2", + "msteams": {"width": "full"}, "body": [], }, } @@ -70,8 +71,9 @@ def format_message_str(self, event): def format_message_other(self, event): items = [{"type": "FactSet", "facts": []}] for key, value in event.data.items(): - msg = self.trim_message(str(value)) - items[0]["facts"].append({"title": f"{key}:", "value": msg}) + if key != "severity": + msg = self.trim_message(str(value)) + items[0]["facts"].append({"title": f"{key}:", "value": msg}) return items def get_severity_color(self, event): @@ -89,7 +91,7 @@ def get_severity_color(self, event): return color def format_message(self, adaptive_card, event): - heading = {"type": "TextBlock", "text": f"{event.type}", "wrap": True, "style": "heading"} + heading = {"type": "TextBlock", "text": f"{event.type}", "wrap": True, "size": "Large", "style": "heading"} body = adaptive_card["attachments"][0]["content"]["body"] body.append(heading) if event.type in ("VULNERABILITY", "FINDING"): @@ -101,6 +103,7 @@ def format_message(self, adaptive_card, event): "wrap": True, } subheading["color"] = self.get_severity_color(event) + body.append(subheading) main_text = { "type": "ColumnSet", "separator": True, @@ -120,7 +123,3 @@ def format_message(self, adaptive_card, event): main_text["columns"][0]["items"] = items body.append(main_text) return adaptive_card - - def evaluate_response(self, response): - text = getattr(response, "text", "") - return text == "1" diff --git a/bbot/test/test_step_2/module_tests/test_module_teams.py b/bbot/test/test_step_2/module_tests/test_module_teams.py index c53c5c75d..56245b89d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_teams.py +++ b/bbot/test/test_step_2/module_tests/test_module_teams.py @@ -20,9 +20,6 @@ def custom_response(request: httpx.Request): text="Webhook message delivery failed with error: Microsoft Teams endpoint returned HTTP error 429 with ContextId tcid=0,server=msgapi-production-eus-azsc2-4-170,cv=deadbeef=2..", ) else: - return httpx.Response( - status_code=200, - text="1", - ) + return httpx.Response(status_code=202) module_test.httpx_mock.add_callback(custom_response, url=self.webhook_url) From 842c7b550e215906295990d8072369b4e22de9ed Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 3 Sep 2024 16:17:45 -0400 Subject: [PATCH 038/254] try things --- .github/workflows/version_updater.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/version_updater.yaml b/.github/workflows/version_updater.yaml index d2911bcba..21ff30c5e 100644 --- a/.github/workflows/version_updater.yaml +++ b/.github/workflows/version_updater.yaml @@ -1,5 +1,8 @@ name: Version Updater on: + push: + branches: + - troubleshoot-version-updater schedule: # Runs at 00:00 every day - cron: '0 0 * * *' @@ -9,10 +12,11 @@ jobs: update-nuclei-version: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: dev fetch-depth: 0 + token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} - name: Set up Python uses: actions/setup-python@v4 with: @@ -54,16 +58,15 @@ jobs: # Release notes: ${{ env.release_notes }} branch: "update-nuclei" - committer: GitHub - author: GitHub assignees: "TheTechromancer" update-trufflehog-version: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: dev fetch-depth: 0 + token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} - name: Set up Python uses: actions/setup-python@v4 with: @@ -105,6 +108,4 @@ jobs: # Release notes: ${{ env.release_notes }} branch: "update-trufflehog" - committer: GitHub - author: GitHub assignees: "TheTechromancer" \ No newline at end of file From e5097f9acc3504a96f45ea88c5e35aabea598f4d Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 3 Sep 2024 16:36:17 -0400 Subject: [PATCH 039/254] update committer/author --- .github/workflows/version_updater.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/version_updater.yaml b/.github/workflows/version_updater.yaml index 21ff30c5e..c4b7bb362 100644 --- a/.github/workflows/version_updater.yaml +++ b/.github/workflows/version_updater.yaml @@ -58,6 +58,8 @@ jobs: # Release notes: ${{ env.release_notes }} branch: "update-nuclei" + committer: blsaccess + author: blsaccess assignees: "TheTechromancer" update-trufflehog-version: runs-on: ubuntu-latest @@ -108,4 +110,6 @@ jobs: # Release notes: ${{ env.release_notes }} branch: "update-trufflehog" + committer: blsaccess + author: blsaccess assignees: "TheTechromancer" \ No newline at end of file From 4c08dfaebc38169ae35d1fdb4367fbb84b1f55eb Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 3 Sep 2024 16:53:39 -0400 Subject: [PATCH 040/254] succeed dammit --- .github/workflows/version_updater.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/version_updater.yaml b/.github/workflows/version_updater.yaml index c4b7bb362..c3892aac4 100644 --- a/.github/workflows/version_updater.yaml +++ b/.github/workflows/version_updater.yaml @@ -8,6 +8,10 @@ on: - cron: '0 0 * * *' workflow_dispatch: # Adds the ability to manually trigger the workflow +permissions: + contents: write + pull-requests: write + jobs: update-nuclei-version: runs-on: ubuntu-latest From 39edc327cd30a57985936fd7eda413fb3433e64f Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 3 Sep 2024 16:57:08 -0400 Subject: [PATCH 041/254] token on checkout only --- .github/workflows/version_updater.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/version_updater.yaml b/.github/workflows/version_updater.yaml index c3892aac4..76732a2b4 100644 --- a/.github/workflows/version_updater.yaml +++ b/.github/workflows/version_updater.yaml @@ -53,7 +53,7 @@ jobs: if: steps.update-version.outcome == 'success' uses: peter-evans/create-pull-request@v5 with: - token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} + # token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} commit-message: "Update nuclei" title: "Update nuclei to ${{ env.latest_version }}" body: | @@ -105,7 +105,7 @@ jobs: if: steps.update-version.outcome == 'success' uses: peter-evans/create-pull-request@v5 with: - token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} + # token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} commit-message: "Update trufflehog" title: "Update trufflehog to ${{ env.latest_version }}" body: | From 68c8ec05115f3429bcd16dcee527fc1fa6dcebc3 Mon Sep 17 00:00:00 2001 From: blsaccess Date: Tue, 3 Sep 2024 20:57:28 +0000 Subject: [PATCH 042/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 092e105de..04fe5c3fc 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -14,7 +14,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.81.9", + "version": "3.81.10", "config": "", "only_verified": True, "concurrency": 8, From f4bc545c1c4695c22326f2f1126372b5eb362ec4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 3 Sep 2024 17:04:52 -0400 Subject: [PATCH 043/254] succeed dammit --- .github/workflows/version_updater.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/version_updater.yaml b/.github/workflows/version_updater.yaml index 76732a2b4..c3892aac4 100644 --- a/.github/workflows/version_updater.yaml +++ b/.github/workflows/version_updater.yaml @@ -53,7 +53,7 @@ jobs: if: steps.update-version.outcome == 'success' uses: peter-evans/create-pull-request@v5 with: - # token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} + token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} commit-message: "Update nuclei" title: "Update nuclei to ${{ env.latest_version }}" body: | @@ -105,7 +105,7 @@ jobs: if: steps.update-version.outcome == 'success' uses: peter-evans/create-pull-request@v5 with: - # token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} + token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} commit-message: "Update trufflehog" title: "Update trufflehog to ${{ env.latest_version }}" body: | From fa130612ec694768a22f27b8c924fe80b07742f0 Mon Sep 17 00:00:00 2001 From: blsaccess Date: Tue, 3 Sep 2024 21:05:11 +0000 Subject: [PATCH 044/254] Update nuclei --- bbot/modules/deadly/nuclei.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 1af628827..02e36aacf 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -15,7 +15,7 @@ class nuclei(BaseModule): } options = { - "version": "3.3.0", + "version": "3.3.1", "tags": "", "templates": "", "severity": "", From fa34c99a5bdc5adaeb0c847ff19d89117a33a7b1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 4 Sep 2024 10:06:33 -0400 Subject: [PATCH 045/254] clean up PR --- .github/workflows/version_updater.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/version_updater.yaml b/.github/workflows/version_updater.yaml index c3892aac4..bb149820c 100644 --- a/.github/workflows/version_updater.yaml +++ b/.github/workflows/version_updater.yaml @@ -1,17 +1,10 @@ name: Version Updater on: - push: - branches: - - troubleshoot-version-updater schedule: # Runs at 00:00 every day - cron: '0 0 * * *' workflow_dispatch: # Adds the ability to manually trigger the workflow -permissions: - contents: write - pull-requests: write - jobs: update-nuclei-version: runs-on: ubuntu-latest From 16138a5b337a781e865ba5a02be78c381ef09c2e Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:31:15 +0300 Subject: [PATCH 046/254] bump nuclei version --- bbot/modules/deadly/nuclei.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 1af628827..b4750c55b 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -15,7 +15,7 @@ class nuclei(BaseModule): } options = { - "version": "3.3.0", + "version": "3.3.2", "tags": "", "templates": "", "severity": "", From 8e323e0581f255ab5ad25ce60b76729df86587bf Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Wed, 4 Sep 2024 20:26:08 +0100 Subject: [PATCH 047/254] Changed evaluate response to `response.is_success` --- bbot/modules/output/slack.py | 1 - bbot/modules/output/teams.py | 1 - bbot/modules/templates/webhook.py | 4 +--- .../test_step_2/module_tests/test_module_discord.py | 2 +- .../test_step_2/module_tests/test_module_teams.py | 11 ++++++++--- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/bbot/modules/output/slack.py b/bbot/modules/output/slack.py index 438ef4973..5d4769554 100644 --- a/bbot/modules/output/slack.py +++ b/bbot/modules/output/slack.py @@ -16,7 +16,6 @@ class Slack(WebhookOutputModule): "event_types": "Types of events to send", "min_severity": "Only allow VULNERABILITY events of this severity or higher", } - good_status_code = 200 content_key = "text" def format_message_str(self, event): diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index 25cf24001..98ab3e432 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -15,7 +15,6 @@ class Teams(WebhookOutputModule): "min_severity": "Only allow VULNERABILITY events of this severity or higher", } _module_threads = 5 - good_status_code = 202 adaptive_card = { "type": "message", "attachments": [ diff --git a/bbot/modules/templates/webhook.py b/bbot/modules/templates/webhook.py index 23d455764..05b291acc 100644 --- a/bbot/modules/templates/webhook.py +++ b/bbot/modules/templates/webhook.py @@ -9,7 +9,6 @@ class WebhookOutputModule(BaseOutputModule): """ accept_dupes = False - good_status_code = 204 message_size_limit = 2000 content_key = "content" vuln_severities = ["UNKNOWN", "LOW", "MEDIUM", "HIGH", "CRITICAL"] @@ -94,5 +93,4 @@ def format_message(self, event): return msg def evaluate_response(self, response): - status_code = getattr(response, "status_code", 0) - return status_code == self.good_status_code + return response.is_success diff --git a/bbot/test/test_step_2/module_tests/test_module_discord.py b/bbot/test/test_step_2/module_tests/test_module_discord.py index 96cec33a9..d1aeb5c60 100644 --- a/bbot/test/test_step_2/module_tests/test_module_discord.py +++ b/bbot/test/test_step_2/module_tests/test_module_discord.py @@ -29,7 +29,7 @@ def custom_response(request: httpx.Request): if module_test.request_count == 2: return httpx.Response(status_code=429, json={"retry_after": 0.01}) else: - return httpx.Response(status_code=module_test.module.good_status_code) + return httpx.Response(status_code=200) module_test.httpx_mock.add_callback(custom_response, url=self.webhook_url) diff --git a/bbot/test/test_step_2/module_tests/test_module_teams.py b/bbot/test/test_step_2/module_tests/test_module_teams.py index 56245b89d..bd00af650 100644 --- a/bbot/test/test_step_2/module_tests/test_module_teams.py +++ b/bbot/test/test_step_2/module_tests/test_module_teams.py @@ -16,10 +16,15 @@ def custom_response(request: httpx.Request): module_test.request_count += 1 if module_test.request_count == 2: return httpx.Response( - status_code=200, - text="Webhook message delivery failed with error: Microsoft Teams endpoint returned HTTP error 429 with ContextId tcid=0,server=msgapi-production-eus-azsc2-4-170,cv=deadbeef=2..", + status_code=400, + json={ + "error": { + "code": "WorkflowTriggerIsNotEnabled", + "message": "Could not execute workflow 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' trigger 'manual' with state 'Disabled': trigger is not enabled.", + } + }, ) else: - return httpx.Response(status_code=202) + return httpx.Response(status_code=200) module_test.httpx_mock.add_callback(custom_response, url=self.webhook_url) From f4f97ab883c0f0d12e50cea71c1f67445d093fd2 Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:20:17 -0400 Subject: [PATCH 048/254] handle nonetype --- bbot/modules/templates/webhook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/templates/webhook.py b/bbot/modules/templates/webhook.py index 05b291acc..0d3c6aee5 100644 --- a/bbot/modules/templates/webhook.py +++ b/bbot/modules/templates/webhook.py @@ -93,4 +93,4 @@ def format_message(self, event): return msg def evaluate_response(self, response): - return response.is_success + return getattr(response, "is_success", False) From 0c899fcdd9fe6fb90762d56967694ad0163506f1 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sat, 7 Sep 2024 14:31:16 +0100 Subject: [PATCH 049/254] Changed defaults --- bbot/modules/wpscan.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index 10dade438..d9c43905e 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -14,19 +14,19 @@ class wpscan(BaseModule): options = { "api_key": "", - "enumerate": "vp,vt,tt,cb,dbe,u,m", + "enumerate": "vp,vt,cb,dbe", "threads": 5, - "request_timeout": 60, - "connection_timeout": 30, + "request_timeout": 5, + "connection_timeout": 2, "disable_tls_checks": True, "force": False, } options_desc = { "api_key": "WPScan API Key", - "enumerate": "Enumeration Process see wpscan help documentation (default: vp,vt,tt,cb,dbe,u,m)", + "enumerate": "Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)", "threads": "How many wpscan threads to spawn (default is 5)", - "request_timeout": "The request timeout in seconds (default 60)", - "connection_timeout": "The connection timeout in seconds (default 30)", + "request_timeout": "The request timeout in seconds (default 5)", + "connection_timeout": "The connection timeout in seconds (default 2)", "disable_tls_checks": "Disables the SSL/TLS certificate verification (Default True)", "force": "Do not check if the target is running WordPress or returns a 403", } @@ -61,11 +61,11 @@ async def setup(self): self.processed = set() self.ignore_events = ["xmlrpc", "readme"] self.api_key = self.config.get("api_key", "") - self.enumerate = self.config.get("enumerate", "vp,vt,tt,cb,dbe,u,m") + self.enumerate = self.config.get("enumerate", "vp,vt,cb,dbe") self.proxy = self.scan.web_config.get("http_proxy", "") self.threads = self.config.get("threads", 5) - self.request_timeout = self.config.get("request_timeout", 60) - self.connection_timeout = self.config.get("connection_timeout", 30) + self.request_timeout = self.config.get("request_timeout", 5) + self.connection_timeout = self.config.get("connection_timeout", 2) self.disable_tls_checks = self.config.get("disable_tls_checks", True) self.force = self.config.get("force", False) return True From 6fe09554720b20b750296718d6f333805f739424 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 8 Sep 2024 00:47:54 -0400 Subject: [PATCH 050/254] add event UUIDs --- bbot/core/event/base.py | 12 +++++++++--- bbot/core/helpers/names_generator.py | 3 ++- bbot/modules/internal/dnsresolve.py | 5 +---- bbot/test/test_step_1/test_events.py | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index aa2052ae7..b058bc3aa 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1,5 +1,6 @@ import io import re +import uuid import json import base64 import logging @@ -58,7 +59,8 @@ class BaseEvent: Attributes: type (str): Specifies the type of the event, e.g., `IP_ADDRESS`, `DNS_NAME`. - id (str): A unique identifier for the event. + id (str): An identifier for the event (event type + sha1 hash of data). + uuid (UUID): A universally unique identifier for the event. data (str or dict): The main data for the event, e.g., a URL or IP address. data_graph (str): Representation of `self.data` for Neo4j graph nodes. data_human (str): Representation of `self.data` for human output. @@ -154,7 +156,7 @@ def __init__( Raises: ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False. """ - + self.uuid = uuid.uuid4() self._id = None self._hash = None self._data = None @@ -733,8 +735,9 @@ def json(self, mode="json", siem_friendly=False): Returns: dict: JSON-serializable dictionary representation of the event object. """ - # type, ID, scope description j = dict() + j["uuid"] = str(self.uuid) + # type, ID, scope description for i in ("type", "id", "scope_description"): v = getattr(self, i, "") if v: @@ -1691,6 +1694,9 @@ def event_from_json(j, siem_friendly=False): data = j["data"] kwargs["data"] = data event = make_event(**kwargs) + event_uuid = j.get("uuid", None) + if event_uuid is not None: + event.uuid = uuid.UUID(event_uuid) resolved_hosts = j.get("resolved_hosts", []) event._resolved_hosts = set(resolved_hosts) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index c0a9ef4c3..27c983514 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -75,6 +75,7 @@ "ethereal", "euphoric", "evil", + "expired", "exquisite", "extreme", "ferocious", @@ -87,7 +88,6 @@ "foreboding", "frenetic", "frolicking", - "frothy", "furry", "fuzzy", "gay", @@ -149,6 +149,7 @@ "muscular", "mushy", "mysterious", + "nascent", "naughty", "nefarious", "negligent", diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index f1f215afd..5dc4acc83 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -90,6 +90,7 @@ async def handle_event(self, event, **kwargs): # if there weren't any DNS children and it's not an IP address, tag as unresolved if not main_host_event.raw_dns_records and not event_is_ip: main_host_event.add_tag("unresolved") + main_host_event.type = "DNS_NAME_UNRESOLVED" # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}") @@ -109,10 +110,6 @@ async def handle_event(self, event, **kwargs): if not self.minimal: await self.emit_dns_children(main_host_event) - # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED - if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags: - main_host_event.type = "DNS_NAME_UNRESOLVED" - # emit the main DNS_NAME or IP_ADDRESS if ( new_event diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 399cf592b..eacc55c00 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -412,6 +412,20 @@ async def test_events(events, helpers): == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ" ) + # test event uuid + import uuid + + event1 = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context") + assert hasattr(event1, "uuid") + assert isinstance(event1.uuid, uuid.UUID) + event2 = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context") + assert hasattr(event2, "uuid") + assert isinstance(event2.uuid, uuid.UUID) + # ids should match because the event type + data is the same + assert event1.id == event2.id + # but uuids should be unique! + assert event1.uuid != event2.uuid + # test event serialization from bbot.core.event import event_from_json @@ -423,6 +437,8 @@ async def test_events(events, helpers): assert db_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"] timestamp = db_event.timestamp.isoformat() json_event = db_event.json() + assert isinstance(json_event["uuid"], str) + assert json_event["uuid"] == str(db_event.uuid) assert json_event["scope_distance"] == 1 assert json_event["data"] == "evilcorp.com:80" assert json_event["type"] == "OPEN_TCP_PORT" @@ -432,6 +448,9 @@ async def test_events(events, helpers): assert json_event["discovery_path"] == ["test context"] assert json_event["parent_chain"] == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"] reconstituted_event = event_from_json(json_event) + assert isinstance(reconstituted_event.uuid, uuid.UUID) + assert str(reconstituted_event.uuid) == json_event["uuid"] + assert reconstituted_event.uuid == db_event.uuid assert reconstituted_event.scope_distance == 1 assert reconstituted_event.timestamp.isoformat() == timestamp assert reconstituted_event.data == "evilcorp.com:80" From b6732d4b0f194691b6e96279f592ba88e044f045 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sun, 8 Sep 2024 13:58:58 +0100 Subject: [PATCH 051/254] Add a regex to detect postman profiles and code repositorys --- bbot/modules/code_repository.py | 1 + bbot/modules/social.py | 1 + .../module_tests/test_module_code_repository.py | 12 +++++++++++- .../test_step_2/module_tests/test_module_social.py | 12 +++++++++++- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py index 372c73b08..f485579f9 100644 --- a/bbot/modules/code_repository.py +++ b/bbot/modules/code_repository.py @@ -19,6 +19,7 @@ class code_repository(BaseModule): (r"gitlab.(?:com|org)/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False), ], "docker": (r"hub.docker.com/r/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False), + "postman": (r"www.postman.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False), } scope_distance_modifier = 1 diff --git a/bbot/modules/social.py b/bbot/modules/social.py index 0c834cd7f..5833228a0 100644 --- a/bbot/modules/social.py +++ b/bbot/modules/social.py @@ -25,6 +25,7 @@ class social(BaseModule): "discord": (r"discord.gg/([a-zA-Z0-9_-]+)", True), "docker": (r"hub.docker.com/[ru]/([a-zA-Z0-9_-]+)", False), "huggingface": (r"huggingface.co/([a-zA-Z0-9_-]+)", False), + "postman": (r"www.postman.com/([a-zA-Z0-9_-]+)", False), } scope_distance_modifier = 1 diff --git a/bbot/test/test_step_2/module_tests/test_module_code_repository.py b/bbot/test/test_step_2/module_tests/test_module_code_repository.py index 3bda459c8..bfb01ef03 100644 --- a/bbot/test/test_step_2/module_tests/test_module_code_repository.py +++ b/bbot/test/test_step_2/module_tests/test_module_code_repository.py @@ -14,13 +14,14 @@ async def setup_after_prep(self, module_test): + """ } module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert 4 == len([e for e in events if e.type == "CODE_REPOSITORY"]) + assert 5 == len([e for e in events if e.type == "CODE_REPOSITORY"]) assert 1 == len( [ e @@ -57,3 +58,12 @@ def check(self, module_test, events): and e.data["url"] == "https://hub.docker.com/r/blacklanternsecurity/bbot" ] ) + assert 1 == len( + [ + e + for e in events + if e.type == "CODE_REPOSITORY" + and "postman" in e.tags + and e.data["url"] == "https://www.postman.com/blacklanternsecurity/bbot" + ] + ) diff --git a/bbot/test/test_step_2/module_tests/test_module_social.py b/bbot/test/test_step_2/module_tests/test_module_social.py index 627643746..6b03c77ed 100644 --- a/bbot/test/test_step_2/module_tests/test_module_social.py +++ b/bbot/test/test_step_2/module_tests/test_module_social.py @@ -14,13 +14,14 @@ async def setup_after_prep(self, module_test): + """ } module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert 3 == len([e for e in events if e.type == "SOCIAL"]) + assert 4 == len([e for e in events if e.type == "SOCIAL"]) assert 1 == len( [ e @@ -46,3 +47,12 @@ def check(self, module_test, events): and e.data["profile_name"] == "blacklanternsecurity" ] ) + assert 1 == len( + [ + e + for e in events + if e.type == "SOCIAL" + and e.data["platform"] == "postman" + and e.data["profile_name"] == "blacklanternsecurity" + ] + ) From 57e5e101464397f1a360c47186319907d16c35a4 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sun, 8 Sep 2024 14:37:50 +0100 Subject: [PATCH 052/254] Rework postman module --- bbot/modules/postman.py | 170 +++------ bbot/modules/templates/postman.py | 11 + .../module_tests/test_module_postman.py | 339 +++++------------- 3 files changed, 148 insertions(+), 372 deletions(-) create mode 100644 bbot/modules/templates/postman.py diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index e4d8895db..982c9ff96 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -1,18 +1,16 @@ -from bbot.modules.templates.subdomain_enum import subdomain_enum +from bbot.modules.templates.postman import postman -class postman(subdomain_enum): - watched_events = ["DNS_NAME"] - produced_events = ["URL_UNVERIFIED"] +class postman(postman): + watched_events = ["ORG_STUB", "SOCIAL"] + produced_events = ["CODE_REPOSITORY"] flags = ["passive", "subdomain-enum", "safe", "code-enum"] meta = { - "description": "Query Postman's API for related workspaces, collections, requests", - "created_date": "2023-12-23", + "description": "Query Postman's API for related workspaces, collections, requests and download them", + "created_date": "2024-09-07", "author": "@domwhewell-sage", } - base_url = "https://www.postman.com/_api" - headers = { "Content-Type": "application/json", "X-App-Version": "10.18.8-230926-0808", @@ -24,13 +22,51 @@ class postman(subdomain_enum): reject_wildcards = False async def handle_event(self, event): - query = self.make_query(event) - self.verbose(f"Searching for any postman workspaces, collections, requests belonging to {query}") - for url, context in await self.query(query): - await self.emit_event(url, "URL_UNVERIFIED", parent=event, tags="httpx-safe", context=context) + # Handle postman profile + if event.type == "SOCIAL": + await self.handle_profile(event) + elif event.type == "ORG_STUB": + await self.handle_org_stub(event) + + async def handle_profile(self, event): + profile_name = event.data.get("profile_name", "") + self.verbose(f"Searching for postman workspaces, collections, requests belonging to {profile_name}") + for item in await self.query(profile_name): + workspace = item["document"] + name = workspace["slug"] + profile = workspace["publisherHandle"] + if profile_name.lower() == profile.lower(): + self.verbose(f"Got {name}") + workspace_url = f"{self.html_url}/{profile}/{name}" + await self.emit_event( + {"url": workspace_url}, + "CODE_REPOSITORY", + tags="postman", + parent=event, + context=f'{{module}} searched postman.com for workspaces belonging to "{profile_name}" and found "{name}" at {{event.type}}: {workspace_url}', + ) + + async def handle_org_stub(self, event): + org_name = event.data + self.verbose(f"Searching for any postman workspaces, collections, requests for {org_name}") + for item in await self.query(org_name): + workspace = item["document"] + human_readable_name = workspace["name"] + name = workspace["slug"] + profile = workspace["publisherHandle"] + if org_name.lower() in human_readable_name.lower(): + self.verbose(f"Got {name}") + workspace_url = f"{self.html_url}/{profile}/{name}" + await self.emit_event( + {"url": workspace_url}, + "CODE_REPOSITORY", + tags="postman", + parent=event, + context=f'{{module}} searched postman.com for "{org_name}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}', + ) async def query(self, query): - interesting_urls = [] + data = [] url = f"{self.base_url}/ws/proxy" json = { "service": "search", @@ -39,11 +75,6 @@ async def query(self, query): "body": { "queryIndices": [ "collaboration.workspace", - "runtime.collection", - "runtime.request", - "adp.api", - "flow.flow", - "apinetwork.team", ], "queryText": self.helpers.quote(query), "size": 100, @@ -57,108 +88,11 @@ async def query(self, query): } r = await self.helpers.request(url, method="POST", json=json, headers=self.headers) if r is None: - return interesting_urls - status_code = getattr(r, "status_code", 0) - try: - json = r.json() - except Exception as e: - self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") - return interesting_urls - workspaces = [] - for item in json.get("data", {}): - for workspace in item.get("document", {}).get("workspaces", []): - if workspace not in workspaces: - workspaces.append(workspace) - for item in workspaces: - id = item.get("id", "") - name = item.get("name", "") - tldextract = self.helpers.tldextract(query) - if tldextract.domain.lower() in name.lower(): - self.verbose(f"Discovered workspace {name} ({id})") - workspace_url = f"{self.base_url}/workspace/{id}" - interesting_urls.append( - ( - workspace_url, - f'{{module}} searched postman.com for "{query}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}', - ) - ) - environments, collections = await self.search_workspace(id) - globals_url = f"{self.base_url}/workspace/{id}/globals" - interesting_urls.append( - ( - globals_url, - f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, and found globals at {{event.type}}: {globals_url}', - ) - ) - for e_id in environments: - env_url = f"{self.base_url}/environment/{e_id}" - interesting_urls.append( - ( - env_url, - f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated environments, and found {{event.type}}: {env_url}', - ) - ) - for c_id in collections: - collection_url = f"{self.base_url}/collection/{c_id}" - interesting_urls.append( - ( - collection_url, - f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated collections, and found {{event.type}}: {collection_url}', - ) - ) - requests = await self.search_collections(id) - for r_id in requests: - request_url = f"{self.base_url}/request/{r_id}" - interesting_urls.append( - ( - request_url, - f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated requests, and found {{event.type}}: {request_url}', - ) - ) - else: - self.verbose(f"Skipping workspace {name} ({id}) as it does not appear to be in scope") - return interesting_urls - - async def search_workspace(self, id): - url = f"{self.base_url}/workspace/{id}" - r = await self.helpers.request(url) - if r is None: - return [], [] + return data status_code = getattr(r, "status_code", 0) try: json = r.json() - if not isinstance(json, dict): - raise ValueError(f"Got unexpected value for JSON: {json}") except Exception as e: self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") - return [], [] - environments = json.get("data", {}).get("dependencies", {}).get("environments", []) - collections = json.get("data", {}).get("dependencies", {}).get("collections", []) - return environments, collections - - async def search_collections(self, id): - request_ids = [] - url = f"{self.base_url}/list/collection?workspace={id}" - r = await self.helpers.request(url, method="POST") - if r is None: - return request_ids - status_code = getattr(r, "status_code", 0) - try: - json = r.json() - except Exception as e: - self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") - return request_ids - for item in json.get("data", {}): - request_ids.extend(await self.parse_collection(item)) - return request_ids - - async def parse_collection(self, json): - request_ids = [] - folders = json.get("folders", []) - requests = json.get("requests", []) - for folder in folders: - request_ids.extend(await self.parse_collection(folder)) - for request in requests: - r_id = request.get("id", "") - request_ids.append(r_id) - return request_ids + return None + return json.get("data", []) diff --git a/bbot/modules/templates/postman.py b/bbot/modules/templates/postman.py new file mode 100644 index 000000000..ec2f987b0 --- /dev/null +++ b/bbot/modules/templates/postman.py @@ -0,0 +1,11 @@ +from bbot.modules.base import BaseModule + + +class postman(BaseModule): + """ + A template module for use of the GitHub API + Inherited by several other github modules. + """ + + base_url = "https://www.postman.com/_api" + html_url = "https://www.postman.com" diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py index 5fc09b1a1..7b30ac20f 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman.py @@ -2,281 +2,112 @@ class TestPostman(ModuleTestBase): - config_overrides = { - "omit_event_types": [], - "scope": {"report_distance": 1}, - } - - modules_overrides = ["postman", "httpx", "excavate"] + modules_overrides = ["postman", "speculate"] async def setup_after_prep(self, module_test): + await module_test.mock_dns( + {"blacklanternsecurity.com": {"A": ["127.0.0.99"]}, "github.com": {"A": ["127.0.0.99"]}} + ) module_test.httpx_mock.add_response( url="https://www.postman.com/_api/ws/proxy", json={ "data": [ { - "score": 499.22498, - "normalizedScore": 8.43312276976538, + "score": 611.41156, + "normalizedScore": 23, "document": { - "isPublisherVerified": False, - "publisherType": "user", - "curatedInList": [], - "publisherId": "28329861", - "publisherHandle": "", - "publisherLogo": "", - "isPublic": True, - "customHostName": "", - "id": "28329861-28329861-f6ef-4f23-9f3a-8431f3567ac1", - "workspaces": [ - { - "visibilityStatus": "public", - "name": "BlackLanternSecuritySpilledSecrets", - "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f", - "slug": "blacklanternsecurityspilledsecrets", - } - ], - "collectionForkLabel": "", - "method": "POST", - "entityType": "request", - "url": "www.example.com/index", - "isBlacklisted": False, - "warehouse__updated_at_collection": "2023-12-11 02:00:00", - "isPrivateNetworkEntity": False, - "warehouse__updated_at_request": "2023-12-11 02:00:00", - "publisherName": "NA", - "name": "A test post request", - "privateNetworkMeta": "", + "watcherCount": 6, + "apiCount": 0, + "forkCount": 0, + "isblacklisted": "false", + "createdAt": "2021-06-15T14:03:51", + "publishertype": "team", + "publisherHandle": "blacklanternsecurity", + "id": "b50893b0-5e50-441d-b884-1900d7cfe919", + "slug": "bbot-public", + "updatedAt": "2024-07-30T11:00:35", + "entityType": "workspace", + "visibilityStatus": "public", + "forkcount": "0", + "tags": [], + "createdat": "2021-06-15T14:03:51", + "forkLabel": "", + "publisherName": "blacklanternsecurity", + "name": "BlackLanternSecurity BBOT [Public]", + "dependencyCount": 7, + "collectionCount": 6, + "warehouse__updated_at": "2024-07-30 11:00:00", "privateNetworkFolders": [], - "documentType": "request", - "collection": { - "id": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205", - "name": "Secret Collection", - "tags": [], - "forkCount": 0, - "watcherCount": 0, - "views": 31, - "apiId": "", - "apiName": "", - }, - }, - }, - { - "score": 498.22398, - "normalizedScore": 8.43312266976538, - "document": { "isPublisherVerified": False, - "publisherType": "user", + "publisherType": "team", "curatedInList": [], - "publisherId": "28329861", - "publisherHandle": "", - "publisherLogo": "", + "creatorId": "6900157", + "description": "", + "forklabel": "", + "publisherId": "299401", + "publisherLogo": "https://res.cloudinary.com/postman/image/upload/t_team_logo/v1664791014/team/cc590b9554074991d0c0fed3fee3e2a9e6b5bfac3dd6fface5f7ea0e33ad33de.png", + "popularity": 5, "isPublic": True, - "customHostName": "", - "id": "b7fa2137-b7fa2137-23bf-45d1-b176-35359af30ded", - "workspaces": [ - { - "visibilityStatus": "public", - "name": "SpilledSecrets", - "id": "92d0451b-119d-4ef0-b74c-22c400e5ce05", - "slug": "spilledsecrets", - } - ], - "collectionForkLabel": "", - "method": "POST", - "entityType": "request", - "url": "www.example.com/index", + "categories": [], + "universaltags": "", + "views": 5788, + "summary": "BLS public workspaces.", + "memberCount": 2, "isBlacklisted": False, - "warehouse__updated_at_collection": "2023-12-11 02:00:00", + "publisherid": "299401", "isPrivateNetworkEntity": False, - "warehouse__updated_at_request": "2023-12-11 02:00:00", - "publisherName": "NA", - "name": "A test post request", + "isDomainNonTrivial": True, "privateNetworkMeta": "", - "privateNetworkFolders": [], - "documentType": "request", - "collection": { - "id": "007e8d67-007e8d67-932b-46ff-b95c-a2aa216edaf3", - "name": "Secret Collection", - "tags": [], - "forkCount": 0, - "watcherCount": 0, - "views": 31, - "apiId": "", - "apiName": "", - }, + "updatedat": "2021-10-20T16:19:29", + "documentType": "workspace", }, - }, + "highlight": {"summary": "BLS BBOT api test."}, + } ], - }, - ) - module_test.httpx_mock.add_response( - url="https://www.postman.com/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f", - json={ - "model_id": "afa061be-9cb0-4520-9d4d-fe63361daf0f", - "meta": {"model": "workspace", "action": "find"}, - "data": { - "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f", - "name": "SpilledSecrets", - "description": "A Mock workspace environment filled with fake secrets to act as a testing ground for secret scanners", - "summary": "A Public workspace with mock secrets", - "createdBy": "28329861", - "updatedBy": "28329861", - "team": "28329861", - "createdAt": "2023-12-13T19:12:21.000Z", - "updatedAt": "2023-12-13T19:15:07.000Z", - "visibilityStatus": "public", - "profileInfo": { - "slug": "spilledsecrets", - "profileType": "team", - "profileId": "28329861", - "publicHandle": "https://www.postman.com/lunar-pizza-28329861", - "publicImageURL": "https://res.cloudinary.com/postman/image/upload/t_team_logo/v1/team/default-L4", - "publicName": "lunar-pizza-28329861", - "isVerified": False, + "meta": { + "queryText": "blacklanternsecurity", + "total": { + "collection": 0, + "request": 0, + "workspace": 1, + "api": 0, + "team": 0, + "user": 0, + "flow": 0, + "apiDefinition": 0, + "privateNetworkFolder": 0, }, - "user": "28329861", - "type": "team", - "dependencies": { - "collections": ["28129865-d9f8833b-3dd2-4b07-9634-1831206d5205"], - "environments": ["28129865-fa7edca0-2df6-4187-9805-11845912f567"], - "globals": ["28735eff-5ecd-43e6-8473-387f160f220f"], + "state": "AQ4", + "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None}, + "featureFlags": { + "enabledPublicResultCuration": True, + "boostByPopularity": True, + "reRankPostNormalization": True, + "enableUrlBarHostNameSearch": True, }, - "members": {"users": {"28129865": {"id": "28129865"}}}, }, }, ) - module_test.httpx_mock.add_response( - url="https://www.postman.com/_api/list/collection?workspace=afa061be-9cb0-4520-9d4d-fe63361daf0f", - json={ - "data": [ - { - "id": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205", - "name": "Secret Collection", - "folders_order": ["d6dd1092-94d4-462c-be48-4e3cad3b8612"], - "order": ["67c9db4c-d0ed-461c-86d2-9a8c5a5de896"], - "attributes": { - "permissions": { - "userCanUpdate": False, - "userCanDelete": False, - "userCanShare": False, - "userCanCreateMock": False, - "userCanCreateMonitor": False, - "anybodyCanView": True, - "teamCanView": True, - }, - "fork": None, - "parent": {"type": "workspace", "id": "afa061be-9cb0-4520-9d4d-fe63361daf0f"}, - "flags": {"isArchived": False, "isFavorite": False}, - }, - "folders": [ - { - "id": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612", - "name": "Nested folder", - "folder": None, - "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205", - "folders_order": ["73a999c1-4246-4805-91bd-0232cc75958a"], - "order": ["3aa78b71-2c4f-4299-94df-287ed1036409"], - "folders": [ - { - "id": "28129865-73a999c1-4246-4805-91bd-0232cc75958a", - "name": "Another Nested Folder", - "folder": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612", - "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205", - "folders_order": [], - "order": ["987c8ac8-bfa9-4bab-ade9-88ccf0597862"], - "folders": [], - "requests": [ - { - "id": "28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862", - "name": "Delete User", - "method": "DELETE", - "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205", - "folder": "28129865-73a999c1-4246-4805-91bd-0232cc75958a", - "responses_order": [], - "responses": [], - } - ], - } - ], - "requests": [ - { - "id": "28129865-3aa78b71-2c4f-4299-94df-287ed1036409", - "name": "Login", - "method": "POST", - "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205", - "folder": "28129865-d6dd1092-94d4-462c-be48-4e3cad3b8612", - "responses_order": [], - "responses": [], - } - ], - } - ], - "requests": [ - { - "id": "28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896", - "name": "Example Basic HTTP request", - "method": "GET", - "collection": "28129865-d9f8833b-3dd2-4b07-9634-1831206d5205", - "folder": None, - "responses_order": [], - "responses": [], - } - ], - } - ] - }, - ) - - old_emit_event = module_test.module.emit_event - - async def new_emit_event(event_data, event_type, **kwargs): - if event_data.startswith("https://www.postman.com"): - event_data = event_data.replace("https://www.postman.com", "http://127.0.0.1:8888") - await old_emit_event(event_data, event_type, **kwargs) - - module_test.monkeypatch.setattr(module_test.module, "emit_event", new_emit_event) - await module_test.mock_dns( - {"blacklanternsecurity.com": {"A": ["127.0.0.1"]}, "asdf.blacklanternsecurity.com": {"A": ["127.0.0.1"]}} - ) - - request_args = dict(uri="/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862") - respond_args = dict(response_data="https://asdf.blacklanternsecurity.com") - module_test.set_expect_requests(request_args, respond_args) def check(self, module_test, events): - assert any( - e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f" for e in events - ), "Failed to detect workspace" - assert any( - e.data != "http://127.0.0.1:8888/_api/workspace/92d0451b-119d-4ef0-b74c-22c400e5ce05" for e in events - ), "Workspace should not be detected" - assert any( - e.data == "http://127.0.0.1:8888/_api/workspace/afa061be-9cb0-4520-9d4d-fe63361daf0f/globals" - for e in events - ), "Failed to detect workspace globals" - assert any( - e.data == "http://127.0.0.1:8888/_api/environment/28129865-fa7edca0-2df6-4187-9805-11845912f567" - for e in events - ), "Failed to detect workspace environment" - assert any( - e.data == "http://127.0.0.1:8888/_api/collection/28129865-d9f8833b-3dd2-4b07-9634-1831206d5205" - for e in events - ), "Failed to detect collection" - assert any( - e.data == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862" - for e in events - ), "Failed to detect collection request #1" - assert any( - e.data == "http://127.0.0.1:8888/_api/request/28129865-3aa78b71-2c4f-4299-94df-287ed1036409" - for e in events - ), "Failed to detect collection request #2" - assert any( - e.data == "http://127.0.0.1:8888/_api/request/28129865-67c9db4c-d0ed-461c-86d2-9a8c5a5de896" - for e in events - ), "Failed to detect collection request #3" - assert any( - e.type == "HTTP_RESPONSE" - and e.data["url"] == "http://127.0.0.1:8888/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862" - for e in events - ), "Failed to emit HTTP_RESPONSE" - assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert len(events) == 4 + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com" and e.scope_distance == 0 + ] + ), "Failed to emit target DNS_NAME" + assert 1 == len( + [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 0] + ), "Failed to find ORG_STUB" + assert 1 == len( + [ + e + for e in events + if e.type == "CODE_REPOSITORY" + and "postman" in e.tags + and e.data["url"] == "https://www.postman.com/blacklanternsecurity/bbot-public" + and e.scope_distance == 1 + ] + ), "Failed to find blacklanternsecurity postman workspace" From b1dcb6d95fc8b7a7b0de5f3fd8fac82f155ece22 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Sun, 8 Sep 2024 14:43:45 +0100 Subject: [PATCH 053/254] Change UUID in test --- bbot/test/test_step_2/module_tests/test_module_postman.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py index 7b30ac20f..b6f807a42 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman.py @@ -23,7 +23,7 @@ async def setup_after_prep(self, module_test): "createdAt": "2021-06-15T14:03:51", "publishertype": "team", "publisherHandle": "blacklanternsecurity", - "id": "b50893b0-5e50-441d-b884-1900d7cfe919", + "id": "11498add-357d-4bc5-a008-0a2d44fb8829", "slug": "bbot-public", "updatedAt": "2024-07-30T11:00:35", "entityType": "workspace", @@ -45,7 +45,7 @@ async def setup_after_prep(self, module_test): "description": "", "forklabel": "", "publisherId": "299401", - "publisherLogo": "https://res.cloudinary.com/postman/image/upload/t_team_logo/v1664791014/team/cc590b9554074991d0c0fed3fee3e2a9e6b5bfac3dd6fface5f7ea0e33ad33de.png", + "publisherLogo": "", "popularity": 5, "isPublic": True, "categories": [], From 46b355bc65c85fde008e2fbe47f9b690eb1474cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 04:57:07 +0000 Subject: [PATCH 054/254] Bump pydantic from 2.7.3 to 2.9.0 Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.7.3 to 2.9.0. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.7.3...v2.9.0) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 195 +++++++++++++++++++++++++++++----------------------- 1 file changed, 110 insertions(+), 85 deletions(-) diff --git a/poetry.lock b/poetry.lock index 705791091..02ac4491e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1683,109 +1683,123 @@ files = [ [[package]] name = "pydantic" -version = "2.7.3" +version = "2.9.0" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.7.3-py3-none-any.whl", hash = "sha256:ea91b002777bf643bb20dd717c028ec43216b24a6001a280f83877fd2655d0b4"}, - {file = "pydantic-2.7.3.tar.gz", hash = "sha256:c46c76a40bb1296728d7a8b99aa73dd70a48c3510111ff290034f860c99c419e"}, + {file = "pydantic-2.9.0-py3-none-any.whl", hash = "sha256:f66a7073abd93214a20c5f7b32d56843137a7a2e70d02111f3be287035c45370"}, + {file = "pydantic-2.9.0.tar.gz", hash = "sha256:c7a8a9fdf7d100afa49647eae340e2d23efa382466a8d177efcd1381e9be5598"}, ] [package.dependencies] annotated-types = ">=0.4.0" -pydantic-core = "2.18.4" -typing-extensions = ">=4.6.1" +pydantic-core = "2.23.2" +typing-extensions = [ + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, +] +tzdata = {version = "*", markers = "python_version >= \"3.9\""} [package.extras] email = ["email-validator (>=2.0.0)"] [[package]] name = "pydantic-core" -version = "2.18.4" +version = "2.23.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.18.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f76d0ad001edd426b92233d45c746fd08f467d56100fd8f30e9ace4b005266e4"}, - {file = "pydantic_core-2.18.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:59ff3e89f4eaf14050c8022011862df275b552caef8082e37b542b066ce1ff26"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a55b5b16c839df1070bc113c1f7f94a0af4433fcfa1b41799ce7606e5c79ce0a"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4d0dcc59664fcb8974b356fe0a18a672d6d7cf9f54746c05f43275fc48636851"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8951eee36c57cd128f779e641e21eb40bc5073eb28b2d23f33eb0ef14ffb3f5d"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4701b19f7e3a06ea655513f7938de6f108123bf7c86bbebb1196eb9bd35cf724"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e00a3f196329e08e43d99b79b286d60ce46bed10f2280d25a1718399457e06be"}, - {file = "pydantic_core-2.18.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:97736815b9cc893b2b7f663628e63f436018b75f44854c8027040e05230eeddb"}, - {file = "pydantic_core-2.18.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6891a2ae0e8692679c07728819b6e2b822fb30ca7445f67bbf6509b25a96332c"}, - {file = "pydantic_core-2.18.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bc4ff9805858bd54d1a20efff925ccd89c9d2e7cf4986144b30802bf78091c3e"}, - {file = "pydantic_core-2.18.4-cp310-none-win32.whl", hash = "sha256:1b4de2e51bbcb61fdebd0ab86ef28062704f62c82bbf4addc4e37fa4b00b7cbc"}, - {file = "pydantic_core-2.18.4-cp310-none-win_amd64.whl", hash = "sha256:6a750aec7bf431517a9fd78cb93c97b9b0c496090fee84a47a0d23668976b4b0"}, - {file = "pydantic_core-2.18.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:942ba11e7dfb66dc70f9ae66b33452f51ac7bb90676da39a7345e99ffb55402d"}, - {file = "pydantic_core-2.18.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b2ebef0e0b4454320274f5e83a41844c63438fdc874ea40a8b5b4ecb7693f1c4"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a642295cd0c8df1b86fc3dced1d067874c353a188dc8e0f744626d49e9aa51c4"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f09baa656c904807e832cf9cce799c6460c450c4ad80803517032da0cd062e2"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98906207f29bc2c459ff64fa007afd10a8c8ac080f7e4d5beff4c97086a3dabd"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19894b95aacfa98e7cb093cd7881a0c76f55731efad31073db4521e2b6ff5b7d"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fbbdc827fe5e42e4d196c746b890b3d72876bdbf160b0eafe9f0334525119c8"}, - {file = "pydantic_core-2.18.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f85d05aa0918283cf29a30b547b4df2fbb56b45b135f9e35b6807cb28bc47951"}, - {file = "pydantic_core-2.18.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e85637bc8fe81ddb73fda9e56bab24560bdddfa98aa64f87aaa4e4b6730c23d2"}, - {file = "pydantic_core-2.18.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2f5966897e5461f818e136b8451d0551a2e77259eb0f73a837027b47dc95dab9"}, - {file = "pydantic_core-2.18.4-cp311-none-win32.whl", hash = "sha256:44c7486a4228413c317952e9d89598bcdfb06399735e49e0f8df643e1ccd0558"}, - {file = "pydantic_core-2.18.4-cp311-none-win_amd64.whl", hash = "sha256:8a7164fe2005d03c64fd3b85649891cd4953a8de53107940bf272500ba8a788b"}, - {file = "pydantic_core-2.18.4-cp311-none-win_arm64.whl", hash = "sha256:4e99bc050fe65c450344421017f98298a97cefc18c53bb2f7b3531eb39bc7805"}, - {file = "pydantic_core-2.18.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6f5c4d41b2771c730ea1c34e458e781b18cc668d194958e0112455fff4e402b2"}, - {file = "pydantic_core-2.18.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2fdf2156aa3d017fddf8aea5adfba9f777db1d6022d392b682d2a8329e087cef"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4748321b5078216070b151d5271ef3e7cc905ab170bbfd27d5c83ee3ec436695"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847a35c4d58721c5dc3dba599878ebbdfd96784f3fb8bb2c356e123bdcd73f34"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c40d4eaad41f78e3bbda31b89edc46a3f3dc6e171bf0ecf097ff7a0ffff7cb1"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:21a5e440dbe315ab9825fcd459b8814bb92b27c974cbc23c3e8baa2b76890077"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01dd777215e2aa86dfd664daed5957704b769e726626393438f9c87690ce78c3"}, - {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b06beb3b3f1479d32befd1f3079cc47b34fa2da62457cdf6c963393340b56e9"}, - {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:564d7922e4b13a16b98772441879fcdcbe82ff50daa622d681dd682175ea918c"}, - {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0eb2a4f660fcd8e2b1c90ad566db2b98d7f3f4717c64fe0a83e0adb39766d5b8"}, - {file = "pydantic_core-2.18.4-cp312-none-win32.whl", hash = "sha256:8b8bab4c97248095ae0c4455b5a1cd1cdd96e4e4769306ab19dda135ea4cdb07"}, - {file = "pydantic_core-2.18.4-cp312-none-win_amd64.whl", hash = "sha256:14601cdb733d741b8958224030e2bfe21a4a881fb3dd6fbb21f071cabd48fa0a"}, - {file = "pydantic_core-2.18.4-cp312-none-win_arm64.whl", hash = "sha256:c1322d7dd74713dcc157a2b7898a564ab091ca6c58302d5c7b4c07296e3fd00f"}, - {file = "pydantic_core-2.18.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:823be1deb01793da05ecb0484d6c9e20baebb39bd42b5d72636ae9cf8350dbd2"}, - {file = "pydantic_core-2.18.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ebef0dd9bf9b812bf75bda96743f2a6c5734a02092ae7f721c048d156d5fabae"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae1d6df168efb88d7d522664693607b80b4080be6750c913eefb77e34c12c71a"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9899c94762343f2cc2fc64c13e7cae4c3cc65cdfc87dd810a31654c9b7358cc"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99457f184ad90235cfe8461c4d70ab7dd2680e28821c29eca00252ba90308c78"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18f469a3d2a2fdafe99296a87e8a4c37748b5080a26b806a707f25a902c040a8"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cdf28938ac6b8b49ae5e92f2735056a7ba99c9b110a474473fd71185c1af5d"}, - {file = "pydantic_core-2.18.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:938cb21650855054dc54dfd9120a851c974f95450f00683399006aa6e8abb057"}, - {file = "pydantic_core-2.18.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:44cd83ab6a51da80fb5adbd9560e26018e2ac7826f9626bc06ca3dc074cd198b"}, - {file = "pydantic_core-2.18.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:972658f4a72d02b8abfa2581d92d59f59897d2e9f7e708fdabe922f9087773af"}, - {file = "pydantic_core-2.18.4-cp38-none-win32.whl", hash = "sha256:1d886dc848e60cb7666f771e406acae54ab279b9f1e4143babc9c2258213daa2"}, - {file = "pydantic_core-2.18.4-cp38-none-win_amd64.whl", hash = "sha256:bb4462bd43c2460774914b8525f79b00f8f407c945d50881568f294c1d9b4443"}, - {file = "pydantic_core-2.18.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:44a688331d4a4e2129140a8118479443bd6f1905231138971372fcde37e43528"}, - {file = "pydantic_core-2.18.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a2fdd81edd64342c85ac7cf2753ccae0b79bf2dfa063785503cb85a7d3593223"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86110d7e1907ab36691f80b33eb2da87d780f4739ae773e5fc83fb272f88825f"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46387e38bd641b3ee5ce247563b60c5ca098da9c56c75c157a05eaa0933ed154"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:123c3cec203e3f5ac7b000bd82235f1a3eced8665b63d18be751f115588fea30"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dc1803ac5c32ec324c5261c7209e8f8ce88e83254c4e1aebdc8b0a39f9ddb443"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53db086f9f6ab2b4061958d9c276d1dbe3690e8dd727d6abf2321d6cce37fa94"}, - {file = "pydantic_core-2.18.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abc267fa9837245cc28ea6929f19fa335f3dc330a35d2e45509b6566dc18be23"}, - {file = "pydantic_core-2.18.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0d829524aaefdebccb869eed855e2d04c21d2d7479b6cada7ace5448416597b"}, - {file = "pydantic_core-2.18.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:509daade3b8649f80d4e5ff21aa5673e4ebe58590b25fe42fac5f0f52c6f034a"}, - {file = "pydantic_core-2.18.4-cp39-none-win32.whl", hash = "sha256:ca26a1e73c48cfc54c4a76ff78df3727b9d9f4ccc8dbee4ae3f73306a591676d"}, - {file = "pydantic_core-2.18.4-cp39-none-win_amd64.whl", hash = "sha256:c67598100338d5d985db1b3d21f3619ef392e185e71b8d52bceacc4a7771ea7e"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:574d92eac874f7f4db0ca653514d823a0d22e2354359d0759e3f6a406db5d55d"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1f4d26ceb5eb9eed4af91bebeae4b06c3fb28966ca3a8fb765208cf6b51102ab"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77450e6d20016ec41f43ca4a6c63e9fdde03f0ae3fe90e7c27bdbeaece8b1ed4"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d323a01da91851a4f17bf592faf46149c9169d68430b3146dcba2bb5e5719abc"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43d447dd2ae072a0065389092a231283f62d960030ecd27565672bd40746c507"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:578e24f761f3b425834f297b9935e1ce2e30f51400964ce4801002435a1b41ef"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:81b5efb2f126454586d0f40c4d834010979cb80785173d1586df845a632e4e6d"}, - {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ab86ce7c8f9bea87b9d12c7f0af71102acbf5ecbc66c17796cff45dae54ef9a5"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:90afc12421df2b1b4dcc975f814e21bc1754640d502a2fbcc6d41e77af5ec312"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:51991a89639a912c17bef4b45c87bd83593aee0437d8102556af4885811d59f5"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:293afe532740370aba8c060882f7d26cfd00c94cae32fd2e212a3a6e3b7bc15e"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b48ece5bde2e768197a2d0f6e925f9d7e3e826f0ad2271120f8144a9db18d5c8"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eae237477a873ab46e8dd748e515c72c0c804fb380fbe6c85533c7de51f23a8f"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:834b5230b5dfc0c1ec37b2fda433b271cbbc0e507560b5d1588e2cc1148cf1ce"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e858ac0a25074ba4bce653f9b5d0a85b7456eaddadc0ce82d3878c22489fa4ee"}, - {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2fd41f6eff4c20778d717af1cc50eca52f5afe7805ee530a4fbd0bae284f16e9"}, - {file = "pydantic_core-2.18.4.tar.gz", hash = "sha256:ec3beeada09ff865c344ff3bc2f427f5e6c26401cc6113d77e372c3fdac73864"}, + {file = "pydantic_core-2.23.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7d0324a35ab436c9d768753cbc3c47a865a2cbc0757066cb864747baa61f6ece"}, + {file = "pydantic_core-2.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:276ae78153a94b664e700ac362587c73b84399bd1145e135287513442e7dfbc7"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:964c7aa318da542cdcc60d4a648377ffe1a2ef0eb1e996026c7f74507b720a78"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1cf842265a3a820ebc6388b963ead065f5ce8f2068ac4e1c713ef77a67b71f7c"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae90b9e50fe1bd115b24785e962b51130340408156d34d67b5f8f3fa6540938e"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ae65fdfb8a841556b52935dfd4c3f79132dc5253b12c0061b96415208f4d622"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c8aa40f6ca803f95b1c1c5aeaee6237b9e879e4dfb46ad713229a63651a95fb"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c53100c8ee5a1e102766abde2158077d8c374bee0639201f11d3032e3555dfbc"}, + {file = "pydantic_core-2.23.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6b9dd6aa03c812017411734e496c44fef29b43dba1e3dd1fa7361bbacfc1354"}, + {file = "pydantic_core-2.23.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b18cf68255a476b927910c6873d9ed00da692bb293c5b10b282bd48a0afe3ae2"}, + {file = "pydantic_core-2.23.2-cp310-none-win32.whl", hash = "sha256:e460475719721d59cd54a350c1f71c797c763212c836bf48585478c5514d2854"}, + {file = "pydantic_core-2.23.2-cp310-none-win_amd64.whl", hash = "sha256:5f3cf3721eaf8741cffaf092487f1ca80831202ce91672776b02b875580e174a"}, + {file = "pydantic_core-2.23.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7ce8e26b86a91e305858e018afc7a6e932f17428b1eaa60154bd1f7ee888b5f8"}, + {file = "pydantic_core-2.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7e9b24cca4037a561422bf5dc52b38d390fb61f7bfff64053ce1b72f6938e6b2"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753294d42fb072aa1775bfe1a2ba1012427376718fa4c72de52005a3d2a22178"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:257d6a410a0d8aeb50b4283dea39bb79b14303e0fab0f2b9d617701331ed1515"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c8319e0bd6a7b45ad76166cc3d5d6a36c97d0c82a196f478c3ee5346566eebfd"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a05c0240f6c711eb381ac392de987ee974fa9336071fb697768dfdb151345ce"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d5b0ff3218858859910295df6953d7bafac3a48d5cd18f4e3ed9999efd2245f"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:96ef39add33ff58cd4c112cbac076726b96b98bb8f1e7f7595288dcfb2f10b57"}, + {file = "pydantic_core-2.23.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0102e49ac7d2df3379ef8d658d3bc59d3d769b0bdb17da189b75efa861fc07b4"}, + {file = "pydantic_core-2.23.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a6612c2a844043e4d10a8324c54cdff0042c558eef30bd705770793d70b224aa"}, + {file = "pydantic_core-2.23.2-cp311-none-win32.whl", hash = "sha256:caffda619099cfd4f63d48462f6aadbecee3ad9603b4b88b60cb821c1b258576"}, + {file = "pydantic_core-2.23.2-cp311-none-win_amd64.whl", hash = "sha256:6f80fba4af0cb1d2344869d56430e304a51396b70d46b91a55ed4959993c0589"}, + {file = "pydantic_core-2.23.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:4c83c64d05ffbbe12d4e8498ab72bdb05bcc1026340a4a597dc647a13c1605ec"}, + {file = "pydantic_core-2.23.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6294907eaaccf71c076abdd1c7954e272efa39bb043161b4b8aa1cd76a16ce43"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a801c5e1e13272e0909c520708122496647d1279d252c9e6e07dac216accc41"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc0c316fba3ce72ac3ab7902a888b9dc4979162d320823679da270c2d9ad0cad"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b06c5d4e8701ac2ba99a2ef835e4e1b187d41095a9c619c5b185c9068ed2a49"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82764c0bd697159fe9947ad59b6db6d7329e88505c8f98990eb07e84cc0a5d81"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b1a195efd347ede8bcf723e932300292eb13a9d2a3c1f84eb8f37cbbc905b7f"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7efb12e5071ad8d5b547487bdad489fbd4a5a35a0fc36a1941517a6ad7f23e0"}, + {file = "pydantic_core-2.23.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5dd0ec5f514ed40e49bf961d49cf1bc2c72e9b50f29a163b2cc9030c6742aa73"}, + {file = "pydantic_core-2.23.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:820f6ee5c06bc868335e3b6e42d7ef41f50dfb3ea32fbd523ab679d10d8741c0"}, + {file = "pydantic_core-2.23.2-cp312-none-win32.whl", hash = "sha256:3713dc093d5048bfaedbba7a8dbc53e74c44a140d45ede020dc347dda18daf3f"}, + {file = "pydantic_core-2.23.2-cp312-none-win_amd64.whl", hash = "sha256:e1895e949f8849bc2757c0dbac28422a04be031204df46a56ab34bcf98507342"}, + {file = "pydantic_core-2.23.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:da43cbe593e3c87d07108d0ebd73771dc414488f1f91ed2e204b0370b94b37ac"}, + {file = "pydantic_core-2.23.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:64d094ea1aa97c6ded4748d40886076a931a8bf6f61b6e43e4a1041769c39dd2"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:084414ffe9a85a52940b49631321d636dadf3576c30259607b75516d131fecd0"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:043ef8469f72609c4c3a5e06a07a1f713d53df4d53112c6d49207c0bd3c3bd9b"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3649bd3ae6a8ebea7dc381afb7f3c6db237fc7cebd05c8ac36ca8a4187b03b30"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6db09153d8438425e98cdc9a289c5fade04a5d2128faff8f227c459da21b9703"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5668b3173bb0b2e65020b60d83f5910a7224027232c9f5dc05a71a1deac9f960"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1c7b81beaf7c7ebde978377dc53679c6cba0e946426fc7ade54251dfe24a7604"}, + {file = "pydantic_core-2.23.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:ae579143826c6f05a361d9546446c432a165ecf1c0b720bbfd81152645cb897d"}, + {file = "pydantic_core-2.23.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:19f1352fe4b248cae22a89268720fc74e83f008057a652894f08fa931e77dced"}, + {file = "pydantic_core-2.23.2-cp313-none-win32.whl", hash = "sha256:e1a79ad49f346aa1a2921f31e8dbbab4d64484823e813a002679eaa46cba39e1"}, + {file = "pydantic_core-2.23.2-cp313-none-win_amd64.whl", hash = "sha256:582871902e1902b3c8e9b2c347f32a792a07094110c1bca6c2ea89b90150caac"}, + {file = "pydantic_core-2.23.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:743e5811b0c377eb830150d675b0847a74a44d4ad5ab8845923d5b3a756d8100"}, + {file = "pydantic_core-2.23.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6650a7bbe17a2717167e3e23c186849bae5cef35d38949549f1c116031b2b3aa"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56e6a12ec8d7679f41b3750ffa426d22b44ef97be226a9bab00a03365f217b2b"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:810ca06cca91de9107718dc83d9ac4d2e86efd6c02cba49a190abcaf33fb0472"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:785e7f517ebb9890813d31cb5d328fa5eda825bb205065cde760b3150e4de1f7"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ef71ec876fcc4d3bbf2ae81961959e8d62f8d74a83d116668409c224012e3af"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d50ac34835c6a4a0d456b5db559b82047403c4317b3bc73b3455fefdbdc54b0a"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16b25a4a120a2bb7dab51b81e3d9f3cde4f9a4456566c403ed29ac81bf49744f"}, + {file = "pydantic_core-2.23.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:41ae8537ad371ec018e3c5da0eb3f3e40ee1011eb9be1da7f965357c4623c501"}, + {file = "pydantic_core-2.23.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07049ec9306ec64e955b2e7c40c8d77dd78ea89adb97a2013d0b6e055c5ee4c5"}, + {file = "pydantic_core-2.23.2-cp38-none-win32.whl", hash = "sha256:086c5db95157dc84c63ff9d96ebb8856f47ce113c86b61065a066f8efbe80acf"}, + {file = "pydantic_core-2.23.2-cp38-none-win_amd64.whl", hash = "sha256:67b6655311b00581914aba481729971b88bb8bc7996206590700a3ac85e457b8"}, + {file = "pydantic_core-2.23.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:358331e21a897151e54d58e08d0219acf98ebb14c567267a87e971f3d2a3be59"}, + {file = "pydantic_core-2.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c4d9f15ffe68bcd3898b0ad7233af01b15c57d91cd1667f8d868e0eacbfe3f87"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0123655fedacf035ab10c23450163c2f65a4174f2bb034b188240a6cf06bb123"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e6e3ccebdbd6e53474b0bb7ab8b88e83c0cfe91484b25e058e581348ee5a01a5"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc535cb898ef88333cf317777ecdfe0faac1c2a3187ef7eb061b6f7ecf7e6bae"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aab9e522efff3993a9e98ab14263d4e20211e62da088298089a03056980a3e69"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05b366fb8fe3d8683b11ac35fa08947d7b92be78ec64e3277d03bd7f9b7cda79"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7568f682c06f10f30ef643a1e8eec4afeecdafde5c4af1b574c6df079e96f96c"}, + {file = "pydantic_core-2.23.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:cdd02a08205dc90238669f082747612cb3c82bd2c717adc60f9b9ecadb540f80"}, + {file = "pydantic_core-2.23.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a2ab4f410f4b886de53b6bddf5dd6f337915a29dd9f22f20f3099659536b2f6"}, + {file = "pydantic_core-2.23.2-cp39-none-win32.whl", hash = "sha256:0448b81c3dfcde439551bb04a9f41d7627f676b12701865c8a2574bcea034437"}, + {file = "pydantic_core-2.23.2-cp39-none-win_amd64.whl", hash = "sha256:4cebb9794f67266d65e7e4cbe5dcf063e29fc7b81c79dc9475bd476d9534150e"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e758d271ed0286d146cf7c04c539a5169a888dd0b57026be621547e756af55bc"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f477d26183e94eaafc60b983ab25af2a809a1b48ce4debb57b343f671b7a90b6"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da3131ef2b940b99106f29dfbc30d9505643f766704e14c5d5e504e6a480c35e"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:329a721253c7e4cbd7aad4a377745fbcc0607f9d72a3cc2102dd40519be75ed2"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7706e15cdbf42f8fab1e6425247dfa98f4a6f8c63746c995d6a2017f78e619ae"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e64ffaf8f6e17ca15eb48344d86a7a741454526f3a3fa56bc493ad9d7ec63936"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dd59638025160056687d598b054b64a79183f8065eae0d3f5ca523cde9943940"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:12625e69b1199e94b0ae1c9a95d000484ce9f0182f9965a26572f054b1537e44"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5d813fd871b3d5c3005157622ee102e8908ad6011ec915a18bd8fde673c4360e"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1eb37f7d6a8001c0f86dc8ff2ee8d08291a536d76e49e78cda8587bb54d8b329"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ce7eaf9a98680b4312b7cebcdd9352531c43db00fca586115845df388f3c465"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f087879f1ffde024dd2788a30d55acd67959dcf6c431e9d3682d1c491a0eb474"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ce883906810b4c3bd90e0ada1f9e808d9ecf1c5f0b60c6b8831d6100bcc7dd6"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a8031074a397a5925d06b590121f8339d34a5a74cfe6970f8a1124eb8b83f4ac"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:23af245b8f2f4ee9e2c99cb3f93d0e22fb5c16df3f2f643f5a8da5caff12a653"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c57e493a0faea1e4c38f860d6862ba6832723396c884fbf938ff5e9b224200e2"}, + {file = "pydantic_core-2.23.2.tar.gz", hash = "sha256:95d6bf449a1ac81de562d65d180af5d8c19672793c81877a2eda8fde5d08f2fd"}, ] [package.dependencies] @@ -2629,6 +2643,17 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + [[package]] name = "unidecode" version = "1.3.8" From f30c7ad33702dba714835e1f2ee645f33767e1a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 04:57:38 +0000 Subject: [PATCH 055/254] Bump pytest-env from 1.1.3 to 1.1.4 Bumps [pytest-env](https://github.com/pytest-dev/pytest-env) from 1.1.3 to 1.1.4. - [Release notes](https://github.com/pytest-dev/pytest-env/releases) - [Commits](https://github.com/pytest-dev/pytest-env/compare/1.1.3...1.1.4) --- updated-dependencies: - dependency-name: pytest-env dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 705791091..6b044f23e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1867,13 +1867,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "8.3.1" +version = "8.3.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.1-py3-none-any.whl", hash = "sha256:e9600ccf4f563976e2c99fa02c7624ab938296551f280835ee6516df8bc4ae8c"}, - {file = "pytest-8.3.1.tar.gz", hash = "sha256:7e8e5c5abd6e93cb1cc151f23e57adc31fcf8cfd2a3ff2da63e23f732de35db6"}, + {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, + {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, ] [package.dependencies] @@ -1925,21 +1925,21 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] [[package]] name = "pytest-env" -version = "1.1.3" +version = "1.1.4" description = "pytest plugin that allows you to add environment variables." optional = false python-versions = ">=3.8" files = [ - {file = "pytest_env-1.1.3-py3-none-any.whl", hash = "sha256:aada77e6d09fcfb04540a6e462c58533c37df35fa853da78707b17ec04d17dfc"}, - {file = "pytest_env-1.1.3.tar.gz", hash = "sha256:fcd7dc23bb71efd3d35632bde1bbe5ee8c8dc4489d6617fb010674880d96216b"}, + {file = "pytest_env-1.1.4-py3-none-any.whl", hash = "sha256:a4212056d4d440febef311a98fdca56c31256d58fb453d103cba4e8a532b721d"}, + {file = "pytest_env-1.1.4.tar.gz", hash = "sha256:86653658da8f11c6844975db955746c458a9c09f1e64957603161e2ff93f5133"}, ] [package.dependencies] -pytest = ">=7.4.3" +pytest = ">=8.3.2" tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} [package.extras] -test = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "pytest-mock (>=3.12)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "pytest-mock (>=3.14)"] [[package]] name = "pytest-httpserver" From fa67915b2bb5d373630e8e9a37d70672edc51c8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 04:57:57 +0000 Subject: [PATCH 056/254] Bump cloudcheck from 5.0.1.537 to 5.0.1.547 Bumps [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck) from 5.0.1.537 to 5.0.1.547. - [Commits](https://github.com/blacklanternsecurity/cloudcheck/commits) --- updated-dependencies: - dependency-name: cloudcheck dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 705791091..ef5de7d27 100644 --- a/poetry.lock +++ b/poetry.lock @@ -387,13 +387,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cloudcheck" -version = "5.0.1.537" +version = "5.0.1.547" description = "Check whether an IP address belongs to a cloud provider" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "cloudcheck-5.0.1.537-py3-none-any.whl", hash = "sha256:ef5c3326bdacfd7048ecc4b1849e728ca705163e7e4713ba4546493af491024a"}, - {file = "cloudcheck-5.0.1.537.tar.gz", hash = "sha256:8884dd9c3de0f634a191559255f3370222bfde5929b10d356415b58291f5d639"}, + {file = "cloudcheck-5.0.1.547-py3-none-any.whl", hash = "sha256:0c183039d3c35611cab1fac5bb6a451a5a520735deb004f3550e2711a14a50c4"}, + {file = "cloudcheck-5.0.1.547.tar.gz", hash = "sha256:43740b552f0201df1bad1acf3a83fb0c566b695b5c156b9e815c9c83be9ae071"}, ] [package.dependencies] From c4bee308494ddf302c2f7a53de6038eb45d26237 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 04:58:19 +0000 Subject: [PATCH 057/254] Bump mkdocstrings from 0.26.0 to 0.26.1 Bumps [mkdocstrings](https://github.com/mkdocstrings/mkdocstrings) from 0.26.0 to 0.26.1. - [Release notes](https://github.com/mkdocstrings/mkdocstrings/releases) - [Changelog](https://github.com/mkdocstrings/mkdocstrings/blob/main/CHANGELOG.md) - [Commits](https://github.com/mkdocstrings/mkdocstrings/compare/0.26.0...0.26.1) --- updated-dependencies: - dependency-name: mkdocstrings dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 705791091..5c0438b4a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1280,13 +1280,13 @@ files = [ [[package]] name = "mkdocstrings" -version = "0.26.0" +version = "0.26.1" description = "Automatic documentation from sources, for MkDocs." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocstrings-0.26.0-py3-none-any.whl", hash = "sha256:1aa227fe94f88e80737d37514523aacd473fc4b50a7f6852ce41447ab23f2654"}, - {file = "mkdocstrings-0.26.0.tar.gz", hash = "sha256:ff9d0de28c8fa877ed9b29a42fe407cfe6736d70a1c48177aa84fcc3dc8518cd"}, + {file = "mkdocstrings-0.26.1-py3-none-any.whl", hash = "sha256:29738bfb72b4608e8e55cc50fb8a54f325dc7ebd2014e4e3881a49892d5983cf"}, + {file = "mkdocstrings-0.26.1.tar.gz", hash = "sha256:bb8b8854d6713d5348ad05b069a09f3b79edbc6a0f33a34c6821141adb03fe33"}, ] [package.dependencies] From 560dc19ee196b777bf0ed1a5fd2be2fde871f875 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 04:58:38 +0000 Subject: [PATCH 058/254] Bump mkdocstrings-python from 1.10.9 to 1.11.1 Bumps [mkdocstrings-python](https://github.com/mkdocstrings/python) from 1.10.9 to 1.11.1. - [Release notes](https://github.com/mkdocstrings/python/releases) - [Changelog](https://github.com/mkdocstrings/python/blob/main/CHANGELOG.md) - [Commits](https://github.com/mkdocstrings/python/compare/1.10.9...1.11.1) --- updated-dependencies: - dependency-name: mkdocstrings-python dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 705791091..bc864d669 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1308,19 +1308,19 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] [[package]] name = "mkdocstrings-python" -version = "1.10.9" +version = "1.11.1" description = "A Python handler for mkdocstrings." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocstrings_python-1.10.9-py3-none-any.whl", hash = "sha256:cbe98710a6757dfd4dff79bf36cb9731908fb4c69dd2736b15270ae7a488243d"}, - {file = "mkdocstrings_python-1.10.9.tar.gz", hash = "sha256:f344aaa47e727d8a2dc911e063025e58e2b7fb31a41110ccc3902aa6be7ca196"}, + {file = "mkdocstrings_python-1.11.1-py3-none-any.whl", hash = "sha256:a21a1c05acef129a618517bb5aae3e33114f569b11588b1e7af3e9d4061a71af"}, + {file = "mkdocstrings_python-1.11.1.tar.gz", hash = "sha256:8824b115c5359304ab0b5378a91f6202324a849e1da907a3485b59208b797322"}, ] [package.dependencies] griffe = ">=0.49" -mkdocs-autorefs = ">=1.0" -mkdocstrings = ">=0.25" +mkdocs-autorefs = ">=1.2" +mkdocstrings = ">=0.26" [[package]] name = "mmh3" From fc7b0d20d6917a6993f2dfc12aac17a1c2776381 Mon Sep 17 00:00:00 2001 From: Dominic Whewell <122788350+domwhewell-sage@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:18:13 +0100 Subject: [PATCH 059/254] Move adaptive card from self to in format message --- bbot/modules/output/teams.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index 98ab3e432..52b17ae1a 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -15,26 +15,10 @@ class Teams(WebhookOutputModule): "min_severity": "Only allow VULNERABILITY events of this severity or higher", } _module_threads = 5 - adaptive_card = { - "type": "message", - "attachments": [ - { - "contentType": "application/vnd.microsoft.card.adaptive", - "contentUrl": None, - "content": { - "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", - "type": "AdaptiveCard", - "version": "1.2", - "msteams": {"width": "full"}, - "body": [], - }, - } - ], - } async def handle_event(self, event): while 1: - data = self.format_message(self.adaptive_card.copy(), event) + data = self.format_message(event) response = await self.helpers.request( url=self.webhook_url, @@ -89,7 +73,23 @@ def get_severity_color(self, event): color = "Good" return color - def format_message(self, adaptive_card, event): + def format_message(self, event): + adaptive_card = { + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "contentUrl": None, + "content": { + "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.2", + "msteams": {"width": "full"}, + "body": [], + }, + } + ], + } heading = {"type": "TextBlock", "text": f"{event.type}", "wrap": True, "size": "Large", "style": "heading"} body = adaptive_card["attachments"][0]["content"]["body"] body.append(heading) From c7831b2404fa62af150ec7ee3c527fc6cd47c496 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 9 Sep 2024 09:28:30 -0400 Subject: [PATCH 060/254] switch parent_chain to use uuids, add parent_uuid attribute to events --- bbot/core/event/base.py | 21 ++++++++++++++++--- bbot/core/helpers/regexes.py | 2 ++ bbot/scanner/scanner.py | 18 ++++++++++------- bbot/test/test_step_1/test_events.py | 30 +++++++++++++++++++++++----- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index b058bc3aa..4ee9a1285 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -59,10 +59,10 @@ class BaseEvent: Attributes: type (str): Specifies the type of the event, e.g., `IP_ADDRESS`, `DNS_NAME`. - id (str): An identifier for the event (event type + sha1 hash of data). + id (str): An identifier for the event (event type + sha1 hash of data). NOT universally unique. uuid (UUID): A universally unique identifier for the event. data (str or dict): The main data for the event, e.g., a URL or IP address. - data_graph (str): Representation of `self.data` for Neo4j graph nodes. + data_graph (str): Representation of `self.data` for graph nodes (e.g. Neo4j). data_human (str): Representation of `self.data` for human output. data_id (str): Representation of `self.data` used to calculate the event's ID (and ultimately its hash, which is used for deduplication) data_json (str): Representation of `self.data` to be used in JSON serialization. @@ -77,6 +77,7 @@ class BaseEvent: resolved_hosts (list of str): List of hosts to which the event data resolves, applicable for URLs and DNS names. parent (BaseEvent): The parent event that led to the discovery of this event. parent_id (str): The `id` attribute of the parent event. + parent_uuid (str): The `uuid` attribute of the parent event. tags (set of str): Descriptive tags for the event, e.g., `mx-record`, `in-scope`. module (BaseModule): The module that discovered the event. module_sequence (str): The sequence of modules that participated in the discovery. @@ -168,6 +169,7 @@ def __init__( self._parent = None self._priority = None self._parent_id = None + self._parent_uuid = None self._host_original = None self._scope_distance = None self._module_priority = None @@ -395,7 +397,7 @@ def parent_chain(self): parent_chain = [] if self.parent is not None and self.parent is not self: parent_chain = self.parent.parent_chain - return parent_chain + [self.id] + return parent_chain + [str(self.uuid)] @property def words(self): @@ -569,6 +571,13 @@ def parent_id(self): return parent_id return self._parent_id + @property + def parent_uuid(self): + parent_uuid = getattr(self.get_parent(), "uuid", None) + if parent_uuid is not None: + return parent_uuid + return self._parent_uuid + @property def validators(self): """ @@ -772,6 +781,9 @@ def json(self, mode="json", siem_friendly=False): parent_id = self.parent_id if parent_id: j["parent"] = parent_id + parent_uuid = self.parent_uuid + if parent_uuid: + j["parent_uuid"] = parent_uuid # tags if self.tags: j.update({"tags": list(self.tags)}) @@ -1706,6 +1718,9 @@ def event_from_json(j, siem_friendly=False): parent_id = j.get("parent", None) if parent_id is not None: event._parent_id = parent_id + parent_uuid = j.get("parent_uuid", None) + if parent_uuid is not None: + event._parent_uuid = parent_uuid return event except KeyError as e: raise ValidationError(f"Event missing required field: {e}") diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index 0c01ff022..ed3452c6e 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -152,3 +152,5 @@ # for use in recursive_decode() encoded_regex = re.compile(r"%[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\[ntrbv]") backslash_regex = re.compile(r"(?P\\+)(?P[ntrvb])") + +uuid_regex = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 111b6bf80..fc62fb4f1 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -115,6 +115,8 @@ def __init__( dispatcher (Dispatcher, optional): Dispatcher object to use. Defaults to new Dispatcher. **kwargs (list[str], optional): Additional keyword arguments (passed through to `Preset`). """ + self._root_event = None + if scan_id is not None: self.id = str(id) else: @@ -945,13 +947,15 @@ def root_event(self): } ``` """ - root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True) - root_event._id = self.id - root_event.scope_distance = 0 - root_event.parent = root_event - root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") - root_event.discovery_context = f"Scan {self.name} started at {root_event.timestamp}" - return root_event + if self._root_event is None: + root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True) + root_event._id = self.id + root_event.scope_distance = 0 + root_event.parent = root_event + root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") + root_event.discovery_context = f"Scan {self.name} started at {root_event.timestamp}" + self._root_event = root_event + return self._root_event @property def dns_strings(self): diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index eacc55c00..d9cfc2e98 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -4,6 +4,7 @@ from ..bbot_fixtures import * from bbot.scanner import Scanner +from bbot.core.helpers.regexes import uuid_regex @pytest.mark.asyncio @@ -415,30 +416,47 @@ async def test_events(events, helpers): # test event uuid import uuid - event1 = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context") + parent_event1 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context") + parent_event2 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context") + + event1 = scan.make_event("evilcorp.com:80", parent=parent_event1, context="test context") assert hasattr(event1, "uuid") assert isinstance(event1.uuid, uuid.UUID) - event2 = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context") + event2 = scan.make_event("evilcorp.com:80", parent=parent_event2, context="test context") assert hasattr(event2, "uuid") assert isinstance(event2.uuid, uuid.UUID) # ids should match because the event type + data is the same assert event1.id == event2.id # but uuids should be unique! assert event1.uuid != event2.uuid + # parent ids should match + assert event1.parent_id == event2.parent_id == parent_event1.id == parent_event2.id + # uuids should not + assert event1.parent_uuid == parent_event1.uuid + assert event2.parent_uuid == parent_event2.uuid + assert event1.parent_uuid != event2.parent_uuid # test event serialization from bbot.core.event import event_from_json db_event = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context") + assert db_event.parent == scan.root_event + assert db_event.parent is scan.root_event db_event._resolved_hosts = {"127.0.0.1"} db_event.scope_distance = 1 assert db_event.discovery_context == "test context" assert db_event.discovery_path == ["test context"] - assert db_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"] + assert len(db_event.parent_chain) == 1 + assert all([uuid_regex.match(u) for u in db_event.parent_chain]) + assert db_event.parent_chain[0] == str(db_event.uuid) + assert db_event.parent.uuid == scan.root_event.uuid + assert db_event.parent_uuid == scan.root_event.uuid timestamp = db_event.timestamp.isoformat() json_event = db_event.json() assert isinstance(json_event["uuid"], str) assert json_event["uuid"] == str(db_event.uuid) + print(f"{json_event} / {db_event.uuid} / {db_event.parent_uuid} / {scan.root_event.uuid}") + assert json_event["parent_uuid"] == str(scan.root_event.uuid) assert json_event["scope_distance"] == 1 assert json_event["data"] == "evilcorp.com:80" assert json_event["type"] == "OPEN_TCP_PORT" @@ -446,10 +464,12 @@ async def test_events(events, helpers): assert json_event["timestamp"] == timestamp assert json_event["discovery_context"] == "test context" assert json_event["discovery_path"] == ["test context"] - assert json_event["parent_chain"] == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"] + assert json_event["parent_chain"] == db_event.parent_chain + assert json_event["parent_chain"][0] == str(db_event.uuid) reconstituted_event = event_from_json(json_event) assert isinstance(reconstituted_event.uuid, uuid.UUID) assert str(reconstituted_event.uuid) == json_event["uuid"] + assert str(reconstituted_event.parent_uuid) == json_event["parent_uuid"] assert reconstituted_event.uuid == db_event.uuid assert reconstituted_event.scope_distance == 1 assert reconstituted_event.timestamp.isoformat() == timestamp @@ -458,7 +478,7 @@ async def test_events(events, helpers): assert reconstituted_event.host == "evilcorp.com" assert reconstituted_event.discovery_context == "test context" assert reconstituted_event.discovery_path == ["test context"] - assert reconstituted_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"] + assert reconstituted_event.parent_chain == db_event.parent_chain assert "127.0.0.1" in reconstituted_event.resolved_hosts hostless_event = scan.make_event("asdf", "ASDF", dummy=True) hostless_event_json = hostless_event.json() From ef55df40126d936be9cd15d94fb2bcc62f56a915 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 9 Sep 2024 10:09:16 -0400 Subject: [PATCH 061/254] update eventjson order --- bbot/core/event/base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 4ee9a1285..61004143f 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -745,12 +745,11 @@ def json(self, mode="json", siem_friendly=False): dict: JSON-serializable dictionary representation of the event object. """ j = dict() - j["uuid"] = str(self.uuid) # type, ID, scope description - for i in ("type", "id", "scope_description"): + for i in ("type", "id", "uuid", "scope_description"): v = getattr(self, i, "") if v: - j.update({i: v}) + j.update({i: str(v)}) # event data data_attr = getattr(self, f"data_{mode}", None) if data_attr is not None: From 8b1c094271e2bb7b9f9c012d91c1cc4c8ad4e366 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 9 Sep 2024 11:52:42 -0400 Subject: [PATCH 062/254] fix json tests --- bbot/core/event/base.py | 2 +- bbot/test/bbot_fixtures.py | 12 ++++++------ bbot/test/test_step_1/test_events.py | 1 + .../module_tests/test_module_json.py | 17 +++++++++++++++-- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 61004143f..d5a9552e2 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1719,7 +1719,7 @@ def event_from_json(j, siem_friendly=False): event._parent_id = parent_id parent_uuid = j.get("parent_uuid", None) if parent_uuid is not None: - event._parent_uuid = parent_uuid + event._parent_uuid = uuid.UUID(parent_uuid) return event except KeyError as e: raise ValidationError(f"Event missing required field: {e}") diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index abad144d1..181b2473f 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -217,9 +217,9 @@ class bbot_events: return bbot_events -@pytest.fixture(scope="session", autouse=True) -def install_all_python_deps(): - deps_pip = set() - for module in DEFAULT_PRESET.module_loader.preloaded().values(): - deps_pip.update(set(module.get("deps", {}).get("pip", []))) - subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) +# @pytest.fixture(scope="session", autouse=True) +# def install_all_python_deps(): +# deps_pip = set() +# for module in DEFAULT_PRESET.module_loader.preloaded().values(): +# deps_pip.update(set(module.get("deps", {}).get("pip", []))) +# subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index d9cfc2e98..cf5aebb42 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -471,6 +471,7 @@ async def test_events(events, helpers): assert str(reconstituted_event.uuid) == json_event["uuid"] assert str(reconstituted_event.parent_uuid) == json_event["parent_uuid"] assert reconstituted_event.uuid == db_event.uuid + assert reconstituted_event.parent_uuid == scan.root_event.uuid assert reconstituted_event.scope_distance == 1 assert reconstituted_event.timestamp.isoformat() == timestamp assert reconstituted_event.data == "evilcorp.com:80" diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 7d9e052e7..ad3417539 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -9,6 +9,9 @@ def check(self, module_test, events): dns_data = "blacklanternsecurity.com" context_data = f"Scan {module_test.scan.name} seeded with DNS_NAME: blacklanternsecurity.com" + scan_event = [e for e in events if e.type == "SCAN"][0] + dns_event = [e for e in events if e.type == "DNS_NAME"][0] + # json events txt_file = module_test.scan.home / "output.json" lines = list(module_test.scan.helpers.read_file(txt_file)) @@ -22,24 +25,34 @@ def check(self, module_test, events): dns_json = dns_json[0] assert scan_json["data"]["name"] == module_test.scan.name assert scan_json["data"]["id"] == module_test.scan.id + assert scan_json["id"] == module_test.scan.id + assert scan_json["uuid"] == str(module_test.scan.root_event.uuid) + assert scan_json["parent_uuid"] == str(module_test.scan.root_event.uuid) assert scan_json["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data + assert dns_json["id"] == str(dns_event.id) + assert dns_json["uuid"] == str(dns_event.uuid) + assert dns_json["parent_uuid"] == str(module_test.scan.root_event.uuid) assert dns_json["discovery_context"] == context_data assert dns_json["discovery_path"] == [context_data] - assert dns_json["parent_chain"] == ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc"] + assert dns_json["parent_chain"] == [dns_json["uuid"]] # event objects reconstructed from json scan_reconstructed = event_from_json(scan_json) dns_reconstructed = event_from_json(dns_json) assert scan_reconstructed.data["name"] == module_test.scan.name assert scan_reconstructed.data["id"] == module_test.scan.id + assert scan_reconstructed.uuid == scan_event.uuid + assert scan_reconstructed.parent_uuid == scan_event.uuid assert scan_reconstructed.data["target"]["seeds"] == ["blacklanternsecurity.com"] assert scan_reconstructed.data["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_reconstructed.data == dns_data + assert dns_reconstructed.uuid == dns_event.uuid + assert dns_reconstructed.parent_uuid == module_test.scan.root_event.uuid assert dns_reconstructed.discovery_context == context_data assert dns_reconstructed.discovery_path == [context_data] - assert dns_reconstructed.parent_chain == ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc"] + assert dns_reconstructed.parent_chain == [dns_json["uuid"]] class TestJSONSIEMFriendly(ModuleTestBase): From ba3fab02ec3fba44f415d31cbe7d41cd01f7ffcd Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 9 Sep 2024 11:54:13 -0400 Subject: [PATCH 063/254] flaked --- bbot/test/bbot_fixtures.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 181b2473f..abad144d1 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -217,9 +217,9 @@ class bbot_events: return bbot_events -# @pytest.fixture(scope="session", autouse=True) -# def install_all_python_deps(): -# deps_pip = set() -# for module in DEFAULT_PRESET.module_loader.preloaded().values(): -# deps_pip.update(set(module.get("deps", {}).get("pip", []))) -# subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) +@pytest.fixture(scope="session", autouse=True) +def install_all_python_deps(): + deps_pip = set() + for module in DEFAULT_PRESET.module_loader.preloaded().values(): + deps_pip.update(set(module.get("deps", {}).get("pip", []))) + subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) From 66b7e94258cb8020ae613ac397819274d62e3b0b Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 10 Sep 2024 16:24:16 -0400 Subject: [PATCH 064/254] add scan finish event --- bbot/scanner/scanner.py | 97 +++++++++++++------ bbot/test/test_step_1/test_dns.py | 16 +-- bbot/test/test_step_1/test_events.py | 2 +- .../test_step_1/test_manager_deduplication.py | 2 +- .../test_manager_scope_accuracy.py | 40 ++++---- bbot/test/test_step_1/test_modules_basic.py | 4 +- bbot/test/test_step_1/test_scope.py | 7 +- bbot/test/test_step_1/test_target.py | 6 +- .../module_tests/test_module_credshed.py | 2 +- .../module_tests/test_module_dehashed.py | 2 +- .../module_tests/test_module_dnsbrute.py | 2 +- .../module_tests/test_module_dnscommonsrv.py | 2 +- .../module_tests/test_module_github_org.py | 8 +- .../test_module_github_workflows.py | 2 +- .../module_tests/test_module_json.py | 20 ++-- .../module_tests/test_module_sslcert.py | 2 +- .../module_tests/test_module_stdout.py | 10 +- .../test_template_subdomain_enum.py | 4 +- 18 files changed, 136 insertions(+), 92 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index fc62fb4f1..f31ae5b0f 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -116,6 +116,13 @@ def __init__( **kwargs (list[str], optional): Additional keyword arguments (passed through to `Preset`). """ self._root_event = None + self.start_time = None + self.end_time = None + self.duration = None + self.duration_human = None + self.duration_seconds = None + + self._success = False if scan_id is not None: self.id = str(id) @@ -306,13 +313,13 @@ async def async_start_without_generator(self): async def async_start(self): """ """ - failed = True - scan_start_time = datetime.now() + self.start_time = datetime.now() + self.root_event.data["started_at"] = self.start_time.isoformat() try: await self._prep() self._start_log_handlers() - self.trace(f'Ran BBOT {__version__} at {scan_start_time}, command: {" ".join(sys.argv)}') + self.trace(f'Ran BBOT {__version__} at {self.start_time}, command: {" ".join(sys.argv)}') self.trace(f"Target: {self.preset.target.json}") self.trace(f"Preset: {self.preset.to_dict(redact_secrets=True)}") @@ -363,16 +370,19 @@ async def async_start(self): if self._finished_init and self.modules_finished: new_activity = await self.finish() if not new_activity: + self._success = True + await self._mark_finished() + yield self.root_event break await asyncio.sleep(0.1) - failed = False + self._success = True except BaseException as e: if self.helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)): self.stop() - failed = False + self._success = True else: try: raise @@ -396,24 +406,46 @@ async def async_start(self): await self._report() await self._cleanup() - log_fn = self.hugesuccess - if self.status == "ABORTING": - self.status = "ABORTED" - log_fn = self.hugewarning - elif failed: - self.status = "FAILED" - log_fn = self.critical - else: - self.status = "FINISHED" - - scan_run_time = datetime.now() - scan_start_time - scan_run_time = self.helpers.human_timedelta(scan_run_time) - log_fn(f"Scan {self.name} completed in {scan_run_time} with status {self.status}") - await self.dispatcher.on_finish(self) self._stop_log_handlers() + async def _mark_finished(self): + log_fn = self.hugesuccess + if self.status == "ABORTING": + status = "ABORTED" + log_fn = self.hugewarning + elif not self._success: + status = "FAILED" + log_fn = self.critical + else: + status = "FINISHED" + + self.end_time = datetime.now() + self.duration = self.end_time - self.start_time + self.duration_seconds = self.duration.total_seconds() + self.duration_human = self.helpers.human_timedelta(self.duration) + + status_message = f"Scan {self.name} completed in {self.duration_human} with status {status}" + + scan_finish_event = self.make_root_event(status_message) + scan_finish_event.data["status"] = status + + # queue final scan event with output modules + output_modules = [m for m in self.modules.values() if m._type == "output" and m.name != "python"] + for m in output_modules: + await m.queue_event(scan_finish_event) + # wait until output modules are flushed + while 1: + modules_finished = all([m.finished for m in output_modules]) + self.verbose(modules_finished) + if modules_finished: + break + await asyncio.sleep(0.05) + + self.status = status + log_fn(status_message) + def _start_modules(self): self.verbose(f"Starting module worker loops") for module in self.modules.values(): @@ -727,8 +759,8 @@ async def finish(self): await module.queue_event(finished_event) self.verbose("Completed finish()") return True - # Return False if no new events were generated since last time self.verbose("Completed final finish()") + # Return False if no new events were generated since last time return False def _drain_queues(self): @@ -948,15 +980,18 @@ def root_event(self): ``` """ if self._root_event is None: - root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True) - root_event._id = self.id - root_event.scope_distance = 0 - root_event.parent = root_event - root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") - root_event.discovery_context = f"Scan {self.name} started at {root_event.timestamp}" - self._root_event = root_event + self._root_event = self.make_root_event(f"Scan {self.name} started at {self.start_time}") + self._root_event.data["status"] = self.status return self._root_event + def make_root_event(self, context): + root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True, context=context) + root_event._id = self.id + root_event.scope_distance = 0 + root_event.parent = root_event + root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") + return root_event + @property def dns_strings(self): """ @@ -1030,6 +1065,14 @@ def json(self): j.update({i: v}) j["target"] = self.preset.target.json j["preset"] = self.preset.to_dict(redact_secrets=True) + if self.start_time is not None: + j["started_at"] = self.start_time.isoformat() + if self.end_time is not None: + j["finished_at"] = self.end_time.isoformat() + if self.duration is not None: + j["duration_seconds"] = self.duration_seconds + if self.duration_human is not None: + j["duration"] = self.duration_human return j def debug(self, *args, trace=False, **kwargs): diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index f4bfb218a..4829125a4 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -263,7 +263,7 @@ def custom_lookup(query, rdtype): await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) events = [e async for e in scan.async_start()] - assert len(events) == 11 + assert len(events) == 12 assert len([e for e in events if e.type == "DNS_NAME"]) == 5 assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ @@ -320,7 +320,7 @@ def custom_lookup(query, rdtype): await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) events = [e async for e in scan.async_start()] - assert len(events) == 11 + assert len(events) == 12 assert len([e for e in events if e.type == "DNS_NAME"]) == 5 assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ @@ -418,7 +418,7 @@ def custom_lookup(query, rdtype): events = [e async for e in scan.async_start()] - assert len(events) == 10 + assert len(events) == 11 assert len([e for e in events if e.type == "DNS_NAME"]) == 5 assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ @@ -546,8 +546,8 @@ def custom_lookup(query, rdtype): ) await scan2.ingress_module.queue_event(other_event, {}) events = [e async for e in scan2.async_start()] - assert len(events) == 3 - assert 1 == len([e for e in events if e.type == "SCAN"]) + assert len(events) == 4 + assert 2 == len([e for e in events if e.type == "SCAN"]) unmodified_wildcard_events = [ e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io" ] @@ -592,8 +592,8 @@ def custom_lookup(query, rdtype): ) await scan2.ingress_module.queue_event(other_event, {}) events = [e async for e in scan2.async_start()] - assert len(events) == 3 - assert 1 == len([e for e in events if e.type == "SCAN"]) + assert len(events) == 4 + assert 2 == len([e for e in events if e.type == "SCAN"]) unmodified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and "_wildcard" not in e.data] assert len(unmodified_wildcard_events) == 2 assert 1 == len( @@ -729,7 +729,7 @@ async def test_dns_graph_structure(bbot_scanner): } ) events = [e async for e in scan.async_start()] - assert len(events) == 5 + assert len(events) == 6 non_scan_events = [e for e in events if e.type != "SCAN"] assert sorted([e.type for e in non_scan_events]) == ["DNS_NAME", "DNS_NAME", "DNS_NAME", "URL_UNVERIFIED"] events_by_data = {e.data: e for e in non_scan_events} diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index cf5aebb42..f7ab9db2b 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -656,7 +656,7 @@ async def handle_event(self, event): ) events = [e async for e in scan.async_start()] - assert len(events) == 6 + assert len(events) == 7 assert 1 == len( [ diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index 06fc026f7..65fbaeb17 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -91,7 +91,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - assert len(events) == 21 + assert len(events) == 22 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"]) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index ef254f38c..f860f1746 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -138,7 +138,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - assert len(events) == 2 + assert len(events) == 3 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) @@ -153,7 +153,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 0 == len([e for e in _all_events if e.type == "DNS_NAME" and e.data == "www.test.notreal"]) assert 0 == len([e for e in _all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) - assert len(graph_output_events) == 2 + assert len(graph_output_events) == 3 assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) @@ -167,7 +167,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - assert len(events) == 3 + assert len(events) == 4 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) @@ -195,7 +195,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 5 + assert len(_graph_output_events) == 6 assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 1]) @@ -211,7 +211,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - assert len(events) == 6 + assert len(events) == 7 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == False and e.scope_distance == 1]) @@ -239,7 +239,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == False and e.scope_distance == 1]) @@ -282,7 +282,7 @@ def custom_setup(scan): _dns_mock=dns_mock_chain, ) - assert len(events) == 4 + assert len(events) == 5 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) @@ -304,7 +304,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e.internal == False and e.scope_distance == 3]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 2]) @@ -326,7 +326,7 @@ def custom_setup(scan): _dns_mock={}, ) - assert len(events) == 6 + assert len(events) == 7 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -366,7 +366,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == True and e.scope_distance == 1]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -393,7 +393,7 @@ def custom_setup(scan): }, ) - assert len(events) == 7 + assert len(events) == 8 # 2024-08-01 # Removed OPEN_TCP_PORT("127.0.0.77:8888") # before, this event was speculated off the URL_UNVERIFIED, and that's what was used by httpx to generate the URL. it was graph-important. @@ -450,7 +450,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 7 + assert len(_graph_output_events) == 8 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -481,7 +481,7 @@ def custom_setup(scan): }, ) - assert len(events) == 7 + assert len(events) == 8 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -541,7 +541,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.99:8888/" and e.internal == True and e.scope_distance == 3]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 7 + assert len(_graph_output_events) == 8 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -574,7 +574,7 @@ def custom_setup(scan): }, ) - assert len(events) == 9 + assert len(events) == 10 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0]) @@ -659,7 +659,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888" and e.internal == True and e.scope_distance == 1]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 9 + assert len(_graph_output_events) == 10 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0]) @@ -695,7 +695,7 @@ def custom_setup(scan): _dns_mock={"www.bbottest.notreal": {"A": ["127.0.1.0"]}, "test.notreal": {"A": ["127.0.0.1"]}}, ) - assert len(events) == 6 + assert len(events) == 7 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) @@ -731,7 +731,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -752,7 +752,7 @@ def custom_setup(scan): _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}}, ) - assert len(events) == 3 + assert len(events) == 4 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) @@ -783,7 +783,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 5 + assert len(_graph_output_events) == 6 assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 10cfa5bb6..9fbaed085 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -380,8 +380,8 @@ async def handle_event(self, event): scan.modules["dummy"] = dummy(scan) events = [e async for e in scan.async_start()] - assert len(events) == 8 - assert 1 == len([e for e in events if e.type == "SCAN"]) + assert len(events) == 9 + assert 2 == len([e for e in events if e.type == "SCAN"]) assert 3 == len([e for e in events if e.type == "DNS_NAME"]) # one from target and one from speculate assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com"]) diff --git a/bbot/test/test_step_1/test_scope.py b/bbot/test/test_step_1/test_scope.py index 7435b82af..ac2d8c042 100644 --- a/bbot/test/test_step_1/test_scope.py +++ b/bbot/test/test_step_1/test_scope.py @@ -12,7 +12,8 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert len(events) == 6 + assert len(events) == 7 + assert 2 == len([e for e in events if e.type == "SCAN"]) assert 1 == len( [ e @@ -62,7 +63,7 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert len(events) == 1 + assert len(events) == 2 assert not any(e.type == "URL" for e in events) assert not any(str(e.host) == "127.0.0.1" for e in events) @@ -72,7 +73,7 @@ class TestScopeWhitelist(TestScopeBlacklist): whitelist = ["255.255.255.255"] def check(self, module_test, events): - assert len(events) == 3 + assert len(events) == 4 assert not any(e.type == "URL" for e in events) assert 1 == len( [ diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index efdf089d3..5b974bd45 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -182,14 +182,14 @@ async def test_target(bbot_scanner): for org_target in ("ORG:evilcorp", "ORG_STUB:evilcorp"): scan = bbot_scanner(org_target) events = [e async for e in scan.async_start()] - assert len(events) == 2 + assert len(events) == 3 assert set([e.type for e in events]) == {"SCAN", "ORG_STUB"} # test username as target for user_target in ("USER:vancerefrigeration", "USERNAME:vancerefrigeration"): scan = bbot_scanner(user_target) events = [e async for e in scan.async_start()] - assert len(events) == 2 + assert len(events) == 3 assert set([e.type for e in events]) == {"SCAN", "USERNAME"} # verify hash values @@ -225,7 +225,7 @@ async def test_target(bbot_scanner): ) events = [e async for e in scan.async_start()] scan_events = [e for e in events if e.type == "SCAN"] - assert len(scan_events) == 1 + assert len(scan_events) == 2 target_dict = scan_events[0].data["target"] assert target_dict["strict_scope"] == False assert target_dict["hash"] == b"\x0b\x908\xe3\xef\n=\x13d\xdf\x00;\xack\x0c\xbc\xd2\xcc'\xba".hex() diff --git a/bbot/test/test_step_2/module_tests/test_module_credshed.py b/bbot/test/test_step_2/module_tests/test_module_credshed.py index 4b9566077..44b9133c9 100644 --- a/bbot/test/test_step_2/module_tests/test_module_credshed.py +++ b/bbot/test/test_step_2/module_tests/test_module_credshed.py @@ -69,7 +69,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 10 + assert len(events) == 11 assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "bob@blacklanternsecurity.com"]) assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "judy@blacklanternsecurity.com"]) assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "tim@blacklanternsecurity.com"]) diff --git a/bbot/test/test_step_2/module_tests/test_module_dehashed.py b/bbot/test/test_step_2/module_tests/test_module_dehashed.py index e8c90273f..0ac91c3b8 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dehashed.py +++ b/bbot/test/test_step_2/module_tests/test_module_dehashed.py @@ -56,7 +56,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 11 + assert len(events) == 12 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"]) assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity"]) assert 1 == len( diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py index 34702b2af..12427b050 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py @@ -92,7 +92,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs): assert reason == f'"ptr1234.blacklanternsecurity.com" looks like an autogenerated PTR' def check(self, module_test, events): - assert len(events) == 3 + assert len(events) == 4 assert 1 == len( [e for e in events if e.data == "asdf.blacklanternsecurity.com" and str(e.module) == "dnsbrute"] ) diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py index 947574dfd..6c5023db1 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py @@ -56,7 +56,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs): ) def check(self, module_test, events): - assert len(events) == 19 + assert len(events) == 20 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"]) assert 1 == len( [ diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py index a4313d182..039c6125b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_github_org.py +++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py @@ -284,7 +284,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 6 + assert len(events) == 7 assert 1 == len( [ e @@ -335,7 +335,7 @@ class TestGithub_Org_No_Members(TestGithub_Org): config_overrides = {"modules": {"github_org": {"include_members": False}}} def check(self, module_test, events): - assert len(events) == 5 + assert len(events) == 6 assert 1 == len( [ e @@ -363,7 +363,7 @@ class TestGithub_Org_MemberRepos(TestGithub_Org): config_overrides = {"modules": {"github_org": {"include_member_repos": True}}} def check(self, module_test, events): - assert len(events) == 7 + assert len(events) == 8 assert 1 == len( [ e @@ -381,7 +381,7 @@ class TestGithub_Org_Custom_Target(TestGithub_Org): config_overrides = {"scope": {"report_distance": 10}, "omit_event_types": [], "speculate": True} def check(self, module_test, events): - assert len(events) == 7 + assert len(events) == 8 assert 1 == len( [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 0] ) diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py index f3c4a2cf5..621df5871 100644 --- a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py +++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py @@ -477,7 +477,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 8 + assert len(events) == 9 assert 1 == len( [ e diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index ad3417539..dc5ebd842 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -19,17 +19,17 @@ def check(self, module_test, events): json_events = [json.loads(line) for line in lines] scan_json = [e for e in json_events if e["type"] == "SCAN"] dns_json = [e for e in json_events if e["type"] == "DNS_NAME"] - assert len(scan_json) == 1 + assert len(scan_json) == 2 assert len(dns_json) == 1 - scan_json = scan_json[0] dns_json = dns_json[0] - assert scan_json["data"]["name"] == module_test.scan.name - assert scan_json["data"]["id"] == module_test.scan.id - assert scan_json["id"] == module_test.scan.id - assert scan_json["uuid"] == str(module_test.scan.root_event.uuid) - assert scan_json["parent_uuid"] == str(module_test.scan.root_event.uuid) - assert scan_json["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] - assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] + for scan in scan_json: + assert scan["data"]["name"] == module_test.scan.name + assert scan["data"]["id"] == module_test.scan.id + assert scan["id"] == module_test.scan.id + assert scan["uuid"] == str(module_test.scan.root_event.uuid) + assert scan["parent_uuid"] == str(module_test.scan.root_event.uuid) + assert scan["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] + assert scan["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["id"] == str(dns_event.id) assert dns_json["uuid"] == str(dns_event.uuid) @@ -39,7 +39,7 @@ def check(self, module_test, events): assert dns_json["parent_chain"] == [dns_json["uuid"]] # event objects reconstructed from json - scan_reconstructed = event_from_json(scan_json) + scan_reconstructed = event_from_json(scan_json[0]) dns_reconstructed = event_from_json(dns_json) assert scan_reconstructed.data["name"] == module_test.scan.name assert scan_reconstructed.data["id"] == module_test.scan.id diff --git a/bbot/test/test_step_2/module_tests/test_module_sslcert.py b/bbot/test/test_step_2/module_tests/test_module_sslcert.py index f8c4948fd..a81482ff5 100644 --- a/bbot/test/test_step_2/module_tests/test_module_sslcert.py +++ b/bbot/test/test_step_2/module_tests/test_module_sslcert.py @@ -6,7 +6,7 @@ class TestSSLCert(ModuleTestBase): config_overrides = {"scope": {"report_distance": 1}} def check(self, module_test, events): - assert len(events) == 6 + assert len(events) == 7 assert 1 == len( [ e diff --git a/bbot/test/test_step_2/module_tests/test_module_stdout.py b/bbot/test/test_step_2/module_tests/test_module_stdout.py index 2c9eaf9bd..7abfc1b79 100644 --- a/bbot/test/test_step_2/module_tests/test_module_stdout.py +++ b/bbot/test/test_step_2/module_tests/test_module_stdout.py @@ -17,7 +17,7 @@ class TestStdoutEventTypes(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() - assert len(out.splitlines()) == 1 + assert len(out.splitlines()) == 2 assert out.startswith("[DNS_NAME] \tblacklanternsecurity.com\tTARGET") @@ -41,7 +41,7 @@ class TestStdoutJSON(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 2 + assert len(lines) == 3 for i, line in enumerate(lines): event = json.loads(line) if i == 0: @@ -56,7 +56,7 @@ class TestStdoutJSONFields(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 2 + assert len(lines) == 3 for line in lines: event = json.loads(line) assert set(event) == {"data", "module_sequence"} @@ -79,7 +79,7 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 3 + assert len(lines) == 4 assert out.count("[IP_ADDRESS] \t127.0.0.2") == 2 @@ -97,5 +97,5 @@ class TestStdoutNoDupes(TestStdoutDupes): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 2 + assert len(lines) == 3 assert out.count("[IP_ADDRESS] \t127.0.0.2") == 1 diff --git a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py index c0bcb25a5..bfa186707 100644 --- a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py +++ b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py @@ -120,7 +120,7 @@ async def mock_query(query): def check(self, module_test, events): assert self.queries == ["walmart.cn"] - assert len(events) == 6 + assert len(events) == 7 assert 2 == len( [ e @@ -185,7 +185,7 @@ def custom_lookup(query, rdtype): def check(self, module_test, events): # no subdomain enum should happen on this domain! assert self.queries == [] - assert len(events) == 6 + assert len(events) == 7 assert 2 == len( [e for e in events if e.type == "IP_ADDRESS" and str(e.module) == "A" and e.scope_distance == 1] ) From 4576f96a45bdb4c4bbaa53115732a07c64418df2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 10 Sep 2024 16:27:06 -0400 Subject: [PATCH 065/254] changing baddns submodule selection --- bbot/modules/baddns.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 7425a985e..45ed328f7 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -15,24 +15,19 @@ class baddns(BaseModule): "created_date": "2024-01-18", "author": "@liquidsec", } - options = {"custom_nameservers": [], "only_high_confidence": False, "enable_references": False} + options = {"custom_nameservers": [], "only_high_confidence": False, "enabled_submodules": []} options_desc = { "custom_nameservers": "Force BadDNS to use a list of custom nameservers", "only_high_confidence": "Do not emit low-confidence or generic detections", - "enable_references": "Enable the references module (off by default)", + "enabled_submodules": "A list of submodules to enable. Empty list enables CNAME Only", } module_threads = 8 deps_pip = ["baddns~=1.1.815"] def select_modules(self): - - module_list = ["CNAME", "NS", "MX", "TXT"] - if self.config.get("enable_references", False): - module_list.append("references") - selected_modules = [] for m in get_all_modules(): - if m.name in module_list: + if m.name in self.enabled_modules: selected_modules.append(m) return selected_modules @@ -43,6 +38,18 @@ async def setup(self): self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers) self.only_high_confidence = self.config.get("only_high_confidence", False) self.signatures = load_signatures() + + self.enabled_modules = self.config.get("enabled_submodules", []) + if self.enabled_modules == []: + self.enabled_modules = ["CNAME"] + all_submodules_list = [m.name for m in get_all_modules()] + for m in self.enabled_modules: + if m not in all_submodules_list: + self.hugewarning( + f"Selected BadDNS submodule [{m}] does not exist. Available submodules: [{','.join(all_submodules_list)}]" + ) + return False + self.critical(f"Enabled BadDNS Submodules: [{','.join(self.enabled_modules)}]") return True async def handle_event(self, event): From 17548ea4ff442d8223fb0b46ac181d745d141fff Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 10 Sep 2024 16:31:14 -0400 Subject: [PATCH 066/254] adding preset, more submodule tweaks --- bbot/modules/baddns.py | 18 +++++++++--------- bbot/presets/baddns-thorough.yml | 11 +++++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 bbot/presets/baddns-thorough.yml diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 45ed328f7..ca235f40f 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -25,11 +25,11 @@ class baddns(BaseModule): deps_pip = ["baddns~=1.1.815"] def select_modules(self): - selected_modules = [] + selected_submodules = [] for m in get_all_modules(): - if m.name in self.enabled_modules: - selected_modules.append(m) - return selected_modules + if m.name in self.enabled_submodules: + selected_submodules.append(m) + return selected_submodules async def setup(self): self.preset.core.logger.include_logger(logging.getLogger("baddns")) @@ -39,17 +39,17 @@ async def setup(self): self.only_high_confidence = self.config.get("only_high_confidence", False) self.signatures = load_signatures() - self.enabled_modules = self.config.get("enabled_submodules", []) - if self.enabled_modules == []: - self.enabled_modules = ["CNAME"] + self.enabled_submodules = self.config.get("enabled_submodules", []) + if self.enabled_submodules == []: + self.enabled_submodules = ["CNAME"] all_submodules_list = [m.name for m in get_all_modules()] - for m in self.enabled_modules: + for m in self.enabled_submodules: if m not in all_submodules_list: self.hugewarning( f"Selected BadDNS submodule [{m}] does not exist. Available submodules: [{','.join(all_submodules_list)}]" ) return False - self.critical(f"Enabled BadDNS Submodules: [{','.join(self.enabled_modules)}]") + self.critical(f"Enabled BadDNS Submodules: [{','.join(self.enabled_submodules)}]") return True async def handle_event(self, event): diff --git a/bbot/presets/baddns-thorough.yml b/bbot/presets/baddns-thorough.yml new file mode 100644 index 000000000..dcd98f90f --- /dev/null +++ b/bbot/presets/baddns-thorough.yml @@ -0,0 +1,11 @@ +description: Comprehensive scan for all IIS/.NET specific modules and module settings + + +modules: + - baddns + - baddns_zone + +config: + modules: + baddns: + enabled_submodules: [NSEC,CNAME,references,zonetransfer,MX,NS,TXT] From 45ae30e9a4dd0d79a876dbfefe8ce39c7dc73d7e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 10 Sep 2024 16:54:14 -0400 Subject: [PATCH 067/254] changing default submodules --- bbot/modules/baddns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index ca235f40f..3090a0571 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -19,7 +19,7 @@ class baddns(BaseModule): options_desc = { "custom_nameservers": "Force BadDNS to use a list of custom nameservers", "only_high_confidence": "Do not emit low-confidence or generic detections", - "enabled_submodules": "A list of submodules to enable. Empty list enables CNAME Only", + "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", } module_threads = 8 deps_pip = ["baddns~=1.1.815"] @@ -41,7 +41,7 @@ async def setup(self): self.enabled_submodules = self.config.get("enabled_submodules", []) if self.enabled_submodules == []: - self.enabled_submodules = ["CNAME"] + self.enabled_submodules = ["CNAME","MX","TXT"] all_submodules_list = [m.name for m in get_all_modules()] for m in self.enabled_submodules: if m not in all_submodules_list: From 7c635fec90f6ea2832fc3d2188e5a4506ddb22bf Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 10 Sep 2024 16:57:03 -0400 Subject: [PATCH 068/254] names --- bbot/core/helpers/names_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 27c983514..baf9126d1 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -67,6 +67,7 @@ "effeminate", "elden", "eldritch", + "effervescent", "embarrassed", "encrypted", "enigmatic", @@ -438,6 +439,7 @@ "gregory", "gus", "hagrid", + "hank", "hannah", "harold", "harry", From 630f0fee8f4a91004171a6da8310f2ccc0e3ef9a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 10 Sep 2024 16:59:39 -0400 Subject: [PATCH 069/254] black --- bbot/modules/baddns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 3090a0571..ef77f11e7 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -41,7 +41,7 @@ async def setup(self): self.enabled_submodules = self.config.get("enabled_submodules", []) if self.enabled_submodules == []: - self.enabled_submodules = ["CNAME","MX","TXT"] + self.enabled_submodules = ["CNAME", "MX", "TXT"] all_submodules_list = [m.name for m in get_all_modules()] for m in self.enabled_submodules: if m not in all_submodules_list: From 535de478f4d831794b08399b0bb43c3f6529880a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 10 Sep 2024 17:40:39 -0400 Subject: [PATCH 070/254] fixing name --- bbot/core/helpers/names_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index baf9126d1..95130de90 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -65,9 +65,9 @@ "dramatic", "drunk", "effeminate", + "effervescent", "elden", "eldritch", - "effervescent", "embarrassed", "encrypted", "enigmatic", From 071cd07796c4a0311f933263a5ded2ffcf635da8 Mon Sep 17 00:00:00 2001 From: Paul Mueller Date: Tue, 10 Sep 2024 18:51:46 -0400 Subject: [PATCH 071/254] Update baddns-thorough.yml --- bbot/presets/baddns-thorough.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/presets/baddns-thorough.yml b/bbot/presets/baddns-thorough.yml index dcd98f90f..92c05a99a 100644 --- a/bbot/presets/baddns-thorough.yml +++ b/bbot/presets/baddns-thorough.yml @@ -1,4 +1,4 @@ -description: Comprehensive scan for all IIS/.NET specific modules and module settings +description: Run all baddns and baddns_zone submodules. modules: From b9b57d0ab779efa695d1579fc802cbcd06051d80 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 11 Sep 2024 21:37:33 -0400 Subject: [PATCH 072/254] fix neo4j warning --- bbot/modules/output/neo4j.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index bb7c9e5c4..f9219ab36 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -1,10 +1,15 @@ import json +import logging from contextlib import suppress from neo4j import AsyncGraphDatabase from bbot.modules.output.base import BaseOutputModule +# silence annoying neo4j logger +logging.getLogger("neo4j").setLevel(logging.CRITICAL) + + class neo4j(BaseOutputModule): """ # start Neo4j in the background with docker @@ -110,7 +115,7 @@ async def merge_events(self, events, event_type, id_only=False): cypher = f"""UNWIND $events AS event MERGE (_:{event_type} {{ id: event.id }}) - SET _ += event + SET _ += properties(event) RETURN event.data as event_data, event.id as event_id, elementId(_) as neo4j_id""" neo4j_ids = {} # insert events From fe13282980b903dceabd6685846a392622473428 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 12 Sep 2024 13:29:58 -0400 Subject: [PATCH 073/254] adding baddns_direct initial --- bbot/modules/baddns_direct.py | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 bbot/modules/baddns_direct.py diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py new file mode 100644 index 000000000..50e3d22dc --- /dev/null +++ b/bbot/modules/baddns_direct.py @@ -0,0 +1,76 @@ +from baddns.base import get_all_modules +from baddns.lib.loader import load_signatures +from urllib.parse import urlparse +from .base import BaseModule + +import asyncio +import logging + +class baddns_direct(BaseModule): + watched_events = ["STORAGE_BUCKET"] + produced_events = ["FINDING", "VULNERABILITY"] + flags = ["active", "safe", "subdomain-enum", "baddns", "cloud-enum"] + meta = { + "description": "Check for unusual subdomain / service takeover edge cases that require direct detection", + "created_date": "2024-01-29", + "author": "@liquidsec", + } + options = {"custom_nameservers": []} + options_desc = { + "custom_nameservers": "Force BadDNS to use a list of custom nameservers", + } + module_threads = 8 + deps_pip = ["baddns~=1.1.815"] + + scope_distance_modifier = 1 + + async def setup(self): + self.preset.core.logger.include_logger(logging.getLogger("baddns")) + self.custom_nameservers = self.config.get("custom_nameservers", []) or None + if self.custom_nameservers: + self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers) + self.only_high_confidence = self.config.get("only_high_confidence", False) + self.signatures = load_signatures() + return True + + def select_modules(self): + selected_modules = [] + for m in get_all_modules(): + if m.name in ["CNAME"]: + selected_modules.append(m) + return selected_modules + + + async def handle_event(self, event): + + self.critical("HANDLE EVENT") + parsed_url = urlparse(event.data["url"]) + domain = parsed_url.netloc + + self.critical(domain) + + + + CNAME_direct_module = self.select_modules()[0] + kwargs = { + "http_client_class": self.scan.helpers.web.AsyncClient, + "dns_client": self.scan.helpers.dns.resolver, + "custom_nameservers": self.custom_nameservers, + "signatures": self.signatures, + } + + CNAME_direct_instance = CNAME_direct_module(domain, **kwargs) + await CNAME_direct_instance.dispatch() + print(CNAME_direct_instance) + results = CNAME_direct_instance.analyze() + self.hugewarning(results) + if results and len(results) > 0: + for r in results: + r_dict = r.to_dict() + self.critical(r_dict) + + async def filter_event(self, event): + if event.type == "STORAGE_BUCKET" and str(event.module).startswith("bucket_"): + self.critical("KILLED BUCKET") + return False + return True From 03d4e54b22d4652bb234d678b3606750f08f4418 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 12 Sep 2024 14:00:37 -0400 Subject: [PATCH 074/254] fixing excavate hostname rule bug, tests, happy little accidents --- bbot/modules/internal/excavate.py | 13 +++++-------- bbot/scanner/scanner.py | 1 + bbot/test/test_step_1/test_scan.py | 3 +++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index edb2a766d..2db7a67cd 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -805,9 +805,9 @@ async def setup(self): if Path(self.custom_yara_rules).is_file(): with open(self.custom_yara_rules) as f: rules_content = f.read() - self.debug(f"Successfully loaded secrets file [{self.custom_yara_rules}]") + self.debug(f"Successfully loaded custom yara rules file [{self.custom_yara_rules}]") else: - self.debug(f"Custom secrets is NOT a file. Will attempt to treat it as rule content") + self.debug(f"Custom yara rules file is NOT a file. Will attempt to treat it as rule content") rules_content = self.custom_yara_rules self.debug(f"Final combined yara rule contents: {rules_content}") @@ -816,13 +816,11 @@ async def setup(self): try: yara.compile(source=rule_content) except yara.SyntaxError as e: - self.hugewarning(f"Custom Yara rule failed to compile: {e}") - return False + return False, f"Custom Yara rule failed to compile: {e}" rule_match = await self.helpers.re.search(self.yara_rule_name_regex, rule_content) if not rule_match: - self.hugewarning(f"Custom Yara formatted incorrectly: could not find rule name") - return False + return False, f"Custom Yara formatted incorrectly: could not find rule name" rule_name = rule_match.groups(1)[0] c = CustomExtractor(self) @@ -838,9 +836,8 @@ async def setup(self): try: self.yara_rules = yara.compile(source=yara_rules_combined) except yara.SyntaxError as e: - self.hugewarning(f"Yara Rules failed to compile with error: [{e}]") self.debug(yara_rules_combined) - return False + return False, f"Yara Rules failed to compile with error: [{e}]" # pre-load valid URL schemes valid_schemes_filename = self.helpers.wordlist_dir / "valid_url_schemes.txt" diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index fc62fb4f1..be25768f1 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -971,6 +971,7 @@ def dns_strings(self): dns_strings = [] for t in dns_targets: if not any(x in dns_targets_set for x in self.helpers.domain_parents(t, include_self=True)): + dns_targets_set.add(t) dns_strings.append(t) self._dns_strings = dns_strings return self._dns_strings diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index d907025c0..bbff8a60b 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -78,6 +78,9 @@ async def test_scan( for scan in (scan0, scan1, scan2, scan4, scan5): await scan._cleanup() + scan6 = bbot_scanner("a.foobar.io", "b.foobar.io", "c.foobar.io", "foobar.io") + assert len(scan6.dns_strings) == 1 + @pytest.mark.asyncio async def test_url_extension_handling(bbot_scanner): From d6d03920e457b419a76cda66f8245c133e0a4c5b Mon Sep 17 00:00:00 2001 From: GitHub Date: Fri, 13 Sep 2024 00:23:43 +0000 Subject: [PATCH 075/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 04fe5c3fc..320fad3fe 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -14,7 +14,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.81.10", + "version": "3.82.1", "config": "", "only_verified": True, "concurrency": 8, From ebcfc403509cced0734e5ff74b299fc15407d2e6 Mon Sep 17 00:00:00 2001 From: GitHub Date: Sat, 14 Sep 2024 00:23:27 +0000 Subject: [PATCH 076/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 320fad3fe..9ae66ab37 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -14,7 +14,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.82.1", + "version": "3.82.2", "config": "", "only_verified": True, "concurrency": 8, From 2eb52071879d15ac3dc41d09b20457d1cdcb21ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 04:31:32 +0000 Subject: [PATCH 077/254] Bump poetry-dynamic-versioning from 1.2.0 to 1.4.1 Bumps [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning) from 1.2.0 to 1.4.1. - [Release notes](https://github.com/mtkennerly/poetry-dynamic-versioning/releases) - [Changelog](https://github.com/mtkennerly/poetry-dynamic-versioning/blob/master/CHANGELOG.md) - [Commits](https://github.com/mtkennerly/poetry-dynamic-versioning/compare/v1.2.0...v1.4.1) --- updated-dependencies: - dependency-name: poetry-dynamic-versioning dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 12 ++++++------ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8b23b7c50..f7c51fca3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1544,17 +1544,17 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "poetry-dynamic-versioning" -version = "1.2.0" +version = "1.4.1" description = "Plugin for Poetry to enable dynamic versioning based on VCS tags" optional = false -python-versions = ">=3.7,<4.0" +python-versions = "<4.0,>=3.7" files = [ - {file = "poetry_dynamic_versioning-1.2.0-py3-none-any.whl", hash = "sha256:8dbef9728d866eb3d1a4edbb29c7ac8abdf96e3ca659473e950e2c016f3785ec"}, - {file = "poetry_dynamic_versioning-1.2.0.tar.gz", hash = "sha256:1a7bbdba2530499e73dfc6ac0af19de29020ab4aaa3e507573877114e6b71ed6"}, + {file = "poetry_dynamic_versioning-1.4.1-py3-none-any.whl", hash = "sha256:44866ccbf869849d32baed4fc5fadf97f786180d8efa1719c88bf17a471bd663"}, + {file = "poetry_dynamic_versioning-1.4.1.tar.gz", hash = "sha256:21584d21ca405aa7d83d23d38372e3c11da664a8742995bdd517577e8676d0e1"}, ] [package.dependencies] -dunamai = ">=1.18.0,<2.0.0" +dunamai = ">=1.21.0,<2.0.0" jinja2 = ">=2.11.1,<4" tomlkit = ">=0.4" @@ -3000,4 +3000,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "c4d16176c75de710966e2d5dadfa8f0c181d90f9ad3ca4fd250702dc23909b98" +content-hash = "37d8f059d576c870383faa9e0cfc432e7d147213c6632a083487d8e910894ec9" diff --git a/pyproject.toml b/pyproject.toml index 4006d67ab..743b1ec7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ httpx = "^0.27.0" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" -poetry-dynamic-versioning = ">=0.21.4,<1.3.0" +poetry-dynamic-versioning = ">=0.21.4,<1.5.0" urllib3 = "^2.0.2" werkzeug = ">=2.3.4,<4.0.0" pytest-env = ">=0.8.2,<1.2.0" From dde291e61cb0fe1f1583443a65a425cce62423a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 04:31:50 +0000 Subject: [PATCH 078/254] Bump pytest from 8.3.2 to 8.3.3 Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.3.2 to 8.3.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.2...8.3.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8b23b7c50..7026caeb1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1881,13 +1881,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "8.3.2" +version = "8.3.3" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, - {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, ] [package.dependencies] From 0138dd6455146007244feb8318f00aca8e4aa098 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 04:32:09 +0000 Subject: [PATCH 079/254] Bump regex from 2024.7.24 to 2024.9.11 Bumps [regex](https://github.com/mrabarnett/mrab-regex) from 2024.7.24 to 2024.9.11. - [Changelog](https://github.com/mrabarnett/mrab-regex/blob/hg/changelog.txt) - [Commits](https://github.com/mrabarnett/mrab-regex/compare/2024.7.24...2024.9.11) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 175 ++++++++++++++++++++++++++++------------------------ 1 file changed, 95 insertions(+), 80 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8b23b7c50..32d1b697f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2258,90 +2258,105 @@ files = [ [[package]] name = "regex" -version = "2024.7.24" +version = "2024.9.11" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.8" files = [ - {file = "regex-2024.7.24-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b0d3f567fafa0633aee87f08b9276c7062da9616931382993c03808bb68ce"}, - {file = "regex-2024.7.24-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3426de3b91d1bc73249042742f45c2148803c111d1175b283270177fdf669024"}, - {file = "regex-2024.7.24-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f273674b445bcb6e4409bf8d1be67bc4b58e8b46fd0d560055d515b8830063cd"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23acc72f0f4e1a9e6e9843d6328177ae3074b4182167e34119ec7233dfeccf53"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65fd3d2e228cae024c411c5ccdffae4c315271eee4a8b839291f84f796b34eca"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c414cbda77dbf13c3bc88b073a1a9f375c7b0cb5e115e15d4b73ec3a2fbc6f59"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf7a89eef64b5455835f5ed30254ec19bf41f7541cd94f266ab7cbd463f00c41"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19c65b00d42804e3fbea9708f0937d157e53429a39b7c61253ff15670ff62cb5"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7a5486ca56c8869070a966321d5ab416ff0f83f30e0e2da1ab48815c8d165d46"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6f51f9556785e5a203713f5efd9c085b4a45aecd2a42573e2b5041881b588d1f"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a4997716674d36a82eab3e86f8fa77080a5d8d96a389a61ea1d0e3a94a582cf7"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c0abb5e4e8ce71a61d9446040c1e86d4e6d23f9097275c5bd49ed978755ff0fe"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:18300a1d78cf1290fa583cd8b7cde26ecb73e9f5916690cf9d42de569c89b1ce"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:416c0e4f56308f34cdb18c3f59849479dde5b19febdcd6e6fa4d04b6c31c9faa"}, - {file = "regex-2024.7.24-cp310-cp310-win32.whl", hash = "sha256:fb168b5924bef397b5ba13aabd8cf5df7d3d93f10218d7b925e360d436863f66"}, - {file = "regex-2024.7.24-cp310-cp310-win_amd64.whl", hash = "sha256:6b9fc7e9cc983e75e2518496ba1afc524227c163e43d706688a6bb9eca41617e"}, - {file = "regex-2024.7.24-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:382281306e3adaaa7b8b9ebbb3ffb43358a7bbf585fa93821300a418bb975281"}, - {file = "regex-2024.7.24-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4fdd1384619f406ad9037fe6b6eaa3de2749e2e12084abc80169e8e075377d3b"}, - {file = "regex-2024.7.24-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3d974d24edb231446f708c455fd08f94c41c1ff4f04bcf06e5f36df5ef50b95a"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a2ec4419a3fe6cf8a4795752596dfe0adb4aea40d3683a132bae9c30b81e8d73"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb563dd3aea54c797adf513eeec819c4213d7dbfc311874eb4fd28d10f2ff0f2"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:45104baae8b9f67569f0f1dca5e1f1ed77a54ae1cd8b0b07aba89272710db61e"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:994448ee01864501912abf2bad9203bffc34158e80fe8bfb5b031f4f8e16da51"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fac296f99283ac232d8125be932c5cd7644084a30748fda013028c815ba3364"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7e37e809b9303ec3a179085415cb5f418ecf65ec98cdfe34f6a078b46ef823ee"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:01b689e887f612610c869421241e075c02f2e3d1ae93a037cb14f88ab6a8934c"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f6442f0f0ff81775eaa5b05af8a0ffa1dda36e9cf6ec1e0d3d245e8564b684ce"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:871e3ab2838fbcb4e0865a6e01233975df3a15e6fce93b6f99d75cacbd9862d1"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c918b7a1e26b4ab40409820ddccc5d49871a82329640f5005f73572d5eaa9b5e"}, - {file = "regex-2024.7.24-cp311-cp311-win32.whl", hash = "sha256:2dfbb8baf8ba2c2b9aa2807f44ed272f0913eeeba002478c4577b8d29cde215c"}, - {file = "regex-2024.7.24-cp311-cp311-win_amd64.whl", hash = "sha256:538d30cd96ed7d1416d3956f94d54e426a8daf7c14527f6e0d6d425fcb4cca52"}, - {file = "regex-2024.7.24-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:fe4ebef608553aff8deb845c7f4f1d0740ff76fa672c011cc0bacb2a00fbde86"}, - {file = "regex-2024.7.24-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:74007a5b25b7a678459f06559504f1eec2f0f17bca218c9d56f6a0a12bfffdad"}, - {file = "regex-2024.7.24-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7df9ea48641da022c2a3c9c641650cd09f0cd15e8908bf931ad538f5ca7919c9"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a1141a1dcc32904c47f6846b040275c6e5de0bf73f17d7a409035d55b76f289"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80c811cfcb5c331237d9bad3bea2c391114588cf4131707e84d9493064d267f9"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7214477bf9bd195894cf24005b1e7b496f46833337b5dedb7b2a6e33f66d962c"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d55588cba7553f0b6ec33130bc3e114b355570b45785cebdc9daed8c637dd440"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:558a57cfc32adcf19d3f791f62b5ff564922942e389e3cfdb538a23d65a6b610"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a512eed9dfd4117110b1881ba9a59b31433caed0c4101b361f768e7bcbaf93c5"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:86b17ba823ea76256b1885652e3a141a99a5c4422f4a869189db328321b73799"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5eefee9bfe23f6df09ffb6dfb23809f4d74a78acef004aa904dc7c88b9944b05"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:731fcd76bbdbf225e2eb85b7c38da9633ad3073822f5ab32379381e8c3c12e94"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eaef80eac3b4cfbdd6de53c6e108b4c534c21ae055d1dbea2de6b3b8ff3def38"}, - {file = "regex-2024.7.24-cp312-cp312-win32.whl", hash = "sha256:185e029368d6f89f36e526764cf12bf8d6f0e3a2a7737da625a76f594bdfcbfc"}, - {file = "regex-2024.7.24-cp312-cp312-win_amd64.whl", hash = "sha256:2f1baff13cc2521bea83ab2528e7a80cbe0ebb2c6f0bfad15be7da3aed443908"}, - {file = "regex-2024.7.24-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:66b4c0731a5c81921e938dcf1a88e978264e26e6ac4ec96a4d21ae0354581ae0"}, - {file = "regex-2024.7.24-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:88ecc3afd7e776967fa16c80f974cb79399ee8dc6c96423321d6f7d4b881c92b"}, - {file = "regex-2024.7.24-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64bd50cf16bcc54b274e20235bf8edbb64184a30e1e53873ff8d444e7ac656b2"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb462f0e346fcf41a901a126b50f8781e9a474d3927930f3490f38a6e73b6950"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a82465ebbc9b1c5c50738536fdfa7cab639a261a99b469c9d4c7dcbb2b3f1e57"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68a8f8c046c6466ac61a36b65bb2395c74451df2ffb8458492ef49900efed293"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac8e84fff5d27420f3c1e879ce9929108e873667ec87e0c8eeb413a5311adfe"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba2537ef2163db9e6ccdbeb6f6424282ae4dea43177402152c67ef869cf3978b"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:43affe33137fcd679bdae93fb25924979517e011f9dea99163f80b82eadc7e53"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:c9bb87fdf2ab2370f21e4d5636e5317775e5d51ff32ebff2cf389f71b9b13750"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:945352286a541406f99b2655c973852da7911b3f4264e010218bbc1cc73168f2"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:8bc593dcce679206b60a538c302d03c29b18e3d862609317cb560e18b66d10cf"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:3f3b6ca8eae6d6c75a6cff525c8530c60e909a71a15e1b731723233331de4169"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c51edc3541e11fbe83f0c4d9412ef6c79f664a3745fab261457e84465ec9d5a8"}, - {file = "regex-2024.7.24-cp38-cp38-win32.whl", hash = "sha256:d0a07763776188b4db4c9c7fb1b8c494049f84659bb387b71c73bbc07f189e96"}, - {file = "regex-2024.7.24-cp38-cp38-win_amd64.whl", hash = "sha256:8fd5afd101dcf86a270d254364e0e8dddedebe6bd1ab9d5f732f274fa00499a5"}, - {file = "regex-2024.7.24-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0ffe3f9d430cd37d8fa5632ff6fb36d5b24818c5c986893063b4e5bdb84cdf24"}, - {file = "regex-2024.7.24-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:25419b70ba00a16abc90ee5fce061228206173231f004437730b67ac77323f0d"}, - {file = "regex-2024.7.24-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:33e2614a7ce627f0cdf2ad104797d1f68342d967de3695678c0cb84f530709f8"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d33a0021893ede5969876052796165bab6006559ab845fd7b515a30abdd990dc"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04ce29e2c5fedf296b1a1b0acc1724ba93a36fb14031f3abfb7abda2806c1535"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b16582783f44fbca6fcf46f61347340c787d7530d88b4d590a397a47583f31dd"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:836d3cc225b3e8a943d0b02633fb2f28a66e281290302a79df0e1eaa984ff7c1"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:438d9f0f4bc64e8dea78274caa5af971ceff0f8771e1a2333620969936ba10be"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:973335b1624859cb0e52f96062a28aa18f3a5fc77a96e4a3d6d76e29811a0e6e"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c5e69fd3eb0b409432b537fe3c6f44ac089c458ab6b78dcec14478422879ec5f"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:fbf8c2f00904eaf63ff37718eb13acf8e178cb940520e47b2f05027f5bb34ce3"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ae2757ace61bc4061b69af19e4689fa4416e1a04840f33b441034202b5cd02d4"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:44fc61b99035fd9b3b9453f1713234e5a7c92a04f3577252b45feefe1b327759"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:84c312cdf839e8b579f504afcd7b65f35d60b6285d892b19adea16355e8343c9"}, - {file = "regex-2024.7.24-cp39-cp39-win32.whl", hash = "sha256:ca5b2028c2f7af4e13fb9fc29b28d0ce767c38c7facdf64f6c2cd040413055f1"}, - {file = "regex-2024.7.24-cp39-cp39-win_amd64.whl", hash = "sha256:7c479f5ae937ec9985ecaf42e2e10631551d909f203e31308c12d703922742f9"}, - {file = "regex-2024.7.24.tar.gz", hash = "sha256:9cfd009eed1a46b27c14039ad5bbc5e71b6367c5b2e6d5f5da0ea91600817506"}, + {file = "regex-2024.9.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1494fa8725c285a81d01dc8c06b55287a1ee5e0e382d8413adc0a9197aac6408"}, + {file = "regex-2024.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e12c481ad92d129c78f13a2a3662317e46ee7ef96c94fd332e1c29131875b7d"}, + {file = "regex-2024.9.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:16e13a7929791ac1216afde26f712802e3df7bf0360b32e4914dca3ab8baeea5"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46989629904bad940bbec2106528140a218b4a36bb3042d8406980be1941429c"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a906ed5e47a0ce5f04b2c981af1c9acf9e8696066900bf03b9d7879a6f679fc8"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9a091b0550b3b0207784a7d6d0f1a00d1d1c8a11699c1a4d93db3fbefc3ad35"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ddcd9a179c0a6fa8add279a4444015acddcd7f232a49071ae57fa6e278f1f71"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6b41e1adc61fa347662b09398e31ad446afadff932a24807d3ceb955ed865cc8"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ced479f601cd2f8ca1fd7b23925a7e0ad512a56d6e9476f79b8f381d9d37090a"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:635a1d96665f84b292e401c3d62775851aedc31d4f8784117b3c68c4fcd4118d"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c0256beda696edcf7d97ef16b2a33a8e5a875affd6fa6567b54f7c577b30a137"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:3ce4f1185db3fbde8ed8aa223fc9620f276c58de8b0d4f8cc86fd1360829edb6"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:09d77559e80dcc9d24570da3745ab859a9cf91953062e4ab126ba9d5993688ca"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a22ccefd4db3f12b526eccb129390942fe874a3a9fdbdd24cf55773a1faab1a"}, + {file = "regex-2024.9.11-cp310-cp310-win32.whl", hash = "sha256:f745ec09bc1b0bd15cfc73df6fa4f726dcc26bb16c23a03f9e3367d357eeedd0"}, + {file = "regex-2024.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:01c2acb51f8a7d6494c8c5eafe3d8e06d76563d8a8a4643b37e9b2dd8a2ff623"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2cce2449e5927a0bf084d346da6cd5eb016b2beca10d0013ab50e3c226ffc0df"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b37fa423beefa44919e009745ccbf353d8c981516e807995b2bd11c2c77d268"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64ce2799bd75039b480cc0360907c4fb2f50022f030bf9e7a8705b636e408fad"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4cc92bb6db56ab0c1cbd17294e14f5e9224f0cc6521167ef388332604e92679"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d05ac6fa06959c4172eccd99a222e1fbf17b5670c4d596cb1e5cde99600674c4"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040562757795eeea356394a7fb13076ad4f99d3c62ab0f8bdfb21f99a1f85664"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6113c008a7780792efc80f9dfe10ba0cd043cbf8dc9a76ef757850f51b4edc50"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e5fb5f77c8745a60105403a774fe2c1759b71d3e7b4ca237a5e67ad066c7199"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:54d9ff35d4515debf14bc27f1e3b38bfc453eff3220f5bce159642fa762fe5d4"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df5cbb1fbc74a8305b6065d4ade43b993be03dbe0f8b30032cced0d7740994bd"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7fb89ee5d106e4a7a51bce305ac4efb981536301895f7bdcf93ec92ae0d91c7f"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a738b937d512b30bf75995c0159c0ddf9eec0775c9d72ac0202076c72f24aa96"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e28f9faeb14b6f23ac55bfbbfd3643f5c7c18ede093977f1df249f73fd22c7b1"}, + {file = "regex-2024.9.11-cp311-cp311-win32.whl", hash = "sha256:18e707ce6c92d7282dfce370cd205098384b8ee21544e7cb29b8aab955b66fa9"}, + {file = "regex-2024.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:313ea15e5ff2a8cbbad96ccef6be638393041b0a7863183c2d31e0c6116688cf"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b0d0a6c64fcc4ef9c69bd5b3b3626cc3776520a1637d8abaa62b9edc147a58f7"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:49b0e06786ea663f933f3710a51e9385ce0cba0ea56b67107fd841a55d56a231"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5b513b6997a0b2f10e4fd3a1313568e373926e8c252bd76c960f96fd039cd28d"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee439691d8c23e76f9802c42a95cfeebf9d47cf4ffd06f18489122dbb0a7ad64"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8f877c89719d759e52783f7fe6e1c67121076b87b40542966c02de5503ace42"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23b30c62d0f16827f2ae9f2bb87619bc4fba2044911e2e6c2eb1af0161cdb766"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85ab7824093d8f10d44330fe1e6493f756f252d145323dd17ab6b48733ff6c0a"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dee5b4810a89447151999428fe096977346cf2f29f4d5e29609d2e19e0199c9"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98eeee2f2e63edae2181c886d7911ce502e1292794f4c5ee71e60e23e8d26b5d"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:57fdd2e0b2694ce6fc2e5ccf189789c3e2962916fb38779d3e3521ff8fe7a822"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d552c78411f60b1fdaafd117a1fca2f02e562e309223b9d44b7de8be451ec5e0"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a0b2b80321c2ed3fcf0385ec9e51a12253c50f146fddb2abbb10f033fe3d049a"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:18406efb2f5a0e57e3a5881cd9354c1512d3bb4f5c45d96d110a66114d84d23a"}, + {file = "regex-2024.9.11-cp312-cp312-win32.whl", hash = "sha256:e464b467f1588e2c42d26814231edecbcfe77f5ac414d92cbf4e7b55b2c2a776"}, + {file = "regex-2024.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:9e8719792ca63c6b8340380352c24dcb8cd7ec49dae36e963742a275dfae6009"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c157bb447303070f256e084668b702073db99bbb61d44f85d811025fcf38f784"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4db21ece84dfeefc5d8a3863f101995de646c6cb0536952c321a2650aa202c36"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:220e92a30b426daf23bb67a7962900ed4613589bab80382be09b48896d211e92"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1ae19e64c14c7ec1995f40bd932448713d3c73509e82d8cd7744dc00e29e86"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f47cd43a5bfa48f86925fe26fbdd0a488ff15b62468abb5d2a1e092a4fb10e85"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9d4a76b96f398697fe01117093613166e6aa8195d63f1b4ec3f21ab637632963"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ea51dcc0835eea2ea31d66456210a4e01a076d820e9039b04ae8d17ac11dee6"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7aaa315101c6567a9a45d2839322c51c8d6e81f67683d529512f5bcfb99c802"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c57d08ad67aba97af57a7263c2d9006d5c404d721c5f7542f077f109ec2a4a29"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8404bf61298bb6f8224bb9176c1424548ee1181130818fcd2cbffddc768bed8"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dd4490a33eb909ef5078ab20f5f000087afa2a4daa27b4c072ccb3cb3050ad84"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:eee9130eaad130649fd73e5cd92f60e55708952260ede70da64de420cdcad554"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a2644a93da36c784e546de579ec1806bfd2763ef47babc1b03d765fe560c9f8"}, + {file = "regex-2024.9.11-cp313-cp313-win32.whl", hash = "sha256:e997fd30430c57138adc06bba4c7c2968fb13d101e57dd5bb9355bf8ce3fa7e8"}, + {file = "regex-2024.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:042c55879cfeb21a8adacc84ea347721d3d83a159da6acdf1116859e2427c43f"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:35f4a6f96aa6cb3f2f7247027b07b15a374f0d5b912c0001418d1d55024d5cb4"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:55b96e7ce3a69a8449a66984c268062fbaa0d8ae437b285428e12797baefce7e"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb130fccd1a37ed894824b8c046321540263013da72745d755f2d35114b81a60"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:323c1f04be6b2968944d730e5c2091c8c89767903ecaa135203eec4565ed2b2b"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be1c8ed48c4c4065ecb19d882a0ce1afe0745dfad8ce48c49586b90a55f02366"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5b029322e6e7b94fff16cd120ab35a253236a5f99a79fb04fda7ae71ca20ae8"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6fff13ef6b5f29221d6904aa816c34701462956aa72a77f1f151a8ec4f56aeb"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:587d4af3979376652010e400accc30404e6c16b7df574048ab1f581af82065e4"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:079400a8269544b955ffa9e31f186f01d96829110a3bf79dc338e9910f794fca"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f9268774428ec173654985ce55fc6caf4c6d11ade0f6f914d48ef4719eb05ebb"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:23f9985c8784e544d53fc2930fc1ac1a7319f5d5332d228437acc9f418f2f168"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ae2941333154baff9838e88aa71c1d84f4438189ecc6021a12c7573728b5838e"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:e93f1c331ca8e86fe877a48ad64e77882c0c4da0097f2212873a69bbfea95d0c"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:846bc79ee753acf93aef4184c040d709940c9d001029ceb7b7a52747b80ed2dd"}, + {file = "regex-2024.9.11-cp38-cp38-win32.whl", hash = "sha256:c94bb0a9f1db10a1d16c00880bdebd5f9faf267273b8f5bd1878126e0fbde771"}, + {file = "regex-2024.9.11-cp38-cp38-win_amd64.whl", hash = "sha256:2b08fce89fbd45664d3df6ad93e554b6c16933ffa9d55cb7e01182baaf971508"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:07f45f287469039ffc2c53caf6803cd506eb5f5f637f1d4acb37a738f71dd066"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4838e24ee015101d9f901988001038f7f0d90dc0c3b115541a1365fb439add62"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6edd623bae6a737f10ce853ea076f56f507fd7726bee96a41ee3d68d347e4d16"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c69ada171c2d0e97a4b5aa78fbb835e0ffbb6b13fc5da968c09811346564f0d3"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02087ea0a03b4af1ed6ebab2c54d7118127fee8d71b26398e8e4b05b78963199"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:69dee6a020693d12a3cf892aba4808fe168d2a4cef368eb9bf74f5398bfd4ee8"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:297f54910247508e6e5cae669f2bc308985c60540a4edd1c77203ef19bfa63ca"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecea58b43a67b1b79805f1a0255730edaf5191ecef84dbc4cc85eb30bc8b63b9"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eab4bb380f15e189d1313195b062a6aa908f5bd687a0ceccd47c8211e9cf0d4a"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0cbff728659ce4bbf4c30b2a1be040faafaa9eca6ecde40aaff86f7889f4ab39"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:54c4a097b8bc5bb0dfc83ae498061d53ad7b5762e00f4adaa23bee22b012e6ba"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:73d6d2f64f4d894c96626a75578b0bf7d9e56dcda8c3d037a2118fdfe9b1c664"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:e53b5fbab5d675aec9f0c501274c467c0f9a5d23696cfc94247e1fb56501ed89"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ffbcf9221e04502fc35e54d1ce9567541979c3fdfb93d2c554f0ca583a19b35"}, + {file = "regex-2024.9.11-cp39-cp39-win32.whl", hash = "sha256:e4c22e1ac1f1ec1e09f72e6c44d8f2244173db7eb9629cc3a346a8d7ccc31142"}, + {file = "regex-2024.9.11-cp39-cp39-win_amd64.whl", hash = "sha256:faa3c142464efec496967359ca99696c896c591c56c53506bac1ad465f66e919"}, + {file = "regex-2024.9.11.tar.gz", hash = "sha256:6c188c307e8433bcb63dc1915022deb553b4203a70722fc542c363bf120a01fd"}, ] [[package]] From 0338e7ff94154d5db8556007ab6f9ab523a527c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 04:32:57 +0000 Subject: [PATCH 080/254] Bump idna from 3.8 to 3.10 Bumps [idna](https://github.com/kjd/idna) from 3.8 to 3.10. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v3.8...v3.10) --- updated-dependencies: - dependency-name: idna dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8b23b7c50..4462a15fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -758,15 +758,18 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "3.8" +version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" files = [ - {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, - {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + [[package]] name = "importlib-metadata" version = "6.2.1" From 3f5cf302e897ef33b9481031381e957315cdce63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 04:33:17 +0000 Subject: [PATCH 081/254] Bump pydantic from 2.9.0 to 2.9.1 Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.9.0 to 2.9.1. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.9.0...v2.9.1) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 203 +++++++++++++++++++++++++--------------------------- 1 file changed, 96 insertions(+), 107 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8b23b7c50..1b1606b24 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1683,123 +1683,123 @@ files = [ [[package]] name = "pydantic" -version = "2.9.0" +version = "2.9.1" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.9.0-py3-none-any.whl", hash = "sha256:f66a7073abd93214a20c5f7b32d56843137a7a2e70d02111f3be287035c45370"}, - {file = "pydantic-2.9.0.tar.gz", hash = "sha256:c7a8a9fdf7d100afa49647eae340e2d23efa382466a8d177efcd1381e9be5598"}, + {file = "pydantic-2.9.1-py3-none-any.whl", hash = "sha256:7aff4db5fdf3cf573d4b3c30926a510a10e19a0774d38fc4967f78beb6deb612"}, + {file = "pydantic-2.9.1.tar.gz", hash = "sha256:1363c7d975c7036df0db2b4a61f2e062fbc0aa5ab5f2772e0ffc7191a4f4bce2"}, ] [package.dependencies] -annotated-types = ">=0.4.0" -pydantic-core = "2.23.2" +annotated-types = ">=0.6.0" +pydantic-core = "2.23.3" typing-extensions = [ {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, {version = ">=4.6.1", markers = "python_version < \"3.13\""}, ] -tzdata = {version = "*", markers = "python_version >= \"3.9\""} [package.extras] email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.23.2" +version = "2.23.3" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.23.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7d0324a35ab436c9d768753cbc3c47a865a2cbc0757066cb864747baa61f6ece"}, - {file = "pydantic_core-2.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:276ae78153a94b664e700ac362587c73b84399bd1145e135287513442e7dfbc7"}, - {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:964c7aa318da542cdcc60d4a648377ffe1a2ef0eb1e996026c7f74507b720a78"}, - {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1cf842265a3a820ebc6388b963ead065f5ce8f2068ac4e1c713ef77a67b71f7c"}, - {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae90b9e50fe1bd115b24785e962b51130340408156d34d67b5f8f3fa6540938e"}, - {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ae65fdfb8a841556b52935dfd4c3f79132dc5253b12c0061b96415208f4d622"}, - {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c8aa40f6ca803f95b1c1c5aeaee6237b9e879e4dfb46ad713229a63651a95fb"}, - {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c53100c8ee5a1e102766abde2158077d8c374bee0639201f11d3032e3555dfbc"}, - {file = "pydantic_core-2.23.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6b9dd6aa03c812017411734e496c44fef29b43dba1e3dd1fa7361bbacfc1354"}, - {file = "pydantic_core-2.23.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b18cf68255a476b927910c6873d9ed00da692bb293c5b10b282bd48a0afe3ae2"}, - {file = "pydantic_core-2.23.2-cp310-none-win32.whl", hash = "sha256:e460475719721d59cd54a350c1f71c797c763212c836bf48585478c5514d2854"}, - {file = "pydantic_core-2.23.2-cp310-none-win_amd64.whl", hash = "sha256:5f3cf3721eaf8741cffaf092487f1ca80831202ce91672776b02b875580e174a"}, - {file = "pydantic_core-2.23.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7ce8e26b86a91e305858e018afc7a6e932f17428b1eaa60154bd1f7ee888b5f8"}, - {file = "pydantic_core-2.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7e9b24cca4037a561422bf5dc52b38d390fb61f7bfff64053ce1b72f6938e6b2"}, - {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753294d42fb072aa1775bfe1a2ba1012427376718fa4c72de52005a3d2a22178"}, - {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:257d6a410a0d8aeb50b4283dea39bb79b14303e0fab0f2b9d617701331ed1515"}, - {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c8319e0bd6a7b45ad76166cc3d5d6a36c97d0c82a196f478c3ee5346566eebfd"}, - {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a05c0240f6c711eb381ac392de987ee974fa9336071fb697768dfdb151345ce"}, - {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d5b0ff3218858859910295df6953d7bafac3a48d5cd18f4e3ed9999efd2245f"}, - {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:96ef39add33ff58cd4c112cbac076726b96b98bb8f1e7f7595288dcfb2f10b57"}, - {file = "pydantic_core-2.23.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0102e49ac7d2df3379ef8d658d3bc59d3d769b0bdb17da189b75efa861fc07b4"}, - {file = "pydantic_core-2.23.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a6612c2a844043e4d10a8324c54cdff0042c558eef30bd705770793d70b224aa"}, - {file = "pydantic_core-2.23.2-cp311-none-win32.whl", hash = "sha256:caffda619099cfd4f63d48462f6aadbecee3ad9603b4b88b60cb821c1b258576"}, - {file = "pydantic_core-2.23.2-cp311-none-win_amd64.whl", hash = "sha256:6f80fba4af0cb1d2344869d56430e304a51396b70d46b91a55ed4959993c0589"}, - {file = "pydantic_core-2.23.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:4c83c64d05ffbbe12d4e8498ab72bdb05bcc1026340a4a597dc647a13c1605ec"}, - {file = "pydantic_core-2.23.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6294907eaaccf71c076abdd1c7954e272efa39bb043161b4b8aa1cd76a16ce43"}, - {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a801c5e1e13272e0909c520708122496647d1279d252c9e6e07dac216accc41"}, - {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc0c316fba3ce72ac3ab7902a888b9dc4979162d320823679da270c2d9ad0cad"}, - {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b06c5d4e8701ac2ba99a2ef835e4e1b187d41095a9c619c5b185c9068ed2a49"}, - {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82764c0bd697159fe9947ad59b6db6d7329e88505c8f98990eb07e84cc0a5d81"}, - {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b1a195efd347ede8bcf723e932300292eb13a9d2a3c1f84eb8f37cbbc905b7f"}, - {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7efb12e5071ad8d5b547487bdad489fbd4a5a35a0fc36a1941517a6ad7f23e0"}, - {file = "pydantic_core-2.23.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5dd0ec5f514ed40e49bf961d49cf1bc2c72e9b50f29a163b2cc9030c6742aa73"}, - {file = "pydantic_core-2.23.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:820f6ee5c06bc868335e3b6e42d7ef41f50dfb3ea32fbd523ab679d10d8741c0"}, - {file = "pydantic_core-2.23.2-cp312-none-win32.whl", hash = "sha256:3713dc093d5048bfaedbba7a8dbc53e74c44a140d45ede020dc347dda18daf3f"}, - {file = "pydantic_core-2.23.2-cp312-none-win_amd64.whl", hash = "sha256:e1895e949f8849bc2757c0dbac28422a04be031204df46a56ab34bcf98507342"}, - {file = "pydantic_core-2.23.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:da43cbe593e3c87d07108d0ebd73771dc414488f1f91ed2e204b0370b94b37ac"}, - {file = "pydantic_core-2.23.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:64d094ea1aa97c6ded4748d40886076a931a8bf6f61b6e43e4a1041769c39dd2"}, - {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:084414ffe9a85a52940b49631321d636dadf3576c30259607b75516d131fecd0"}, - {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:043ef8469f72609c4c3a5e06a07a1f713d53df4d53112c6d49207c0bd3c3bd9b"}, - {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3649bd3ae6a8ebea7dc381afb7f3c6db237fc7cebd05c8ac36ca8a4187b03b30"}, - {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6db09153d8438425e98cdc9a289c5fade04a5d2128faff8f227c459da21b9703"}, - {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5668b3173bb0b2e65020b60d83f5910a7224027232c9f5dc05a71a1deac9f960"}, - {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1c7b81beaf7c7ebde978377dc53679c6cba0e946426fc7ade54251dfe24a7604"}, - {file = "pydantic_core-2.23.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:ae579143826c6f05a361d9546446c432a165ecf1c0b720bbfd81152645cb897d"}, - {file = "pydantic_core-2.23.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:19f1352fe4b248cae22a89268720fc74e83f008057a652894f08fa931e77dced"}, - {file = "pydantic_core-2.23.2-cp313-none-win32.whl", hash = "sha256:e1a79ad49f346aa1a2921f31e8dbbab4d64484823e813a002679eaa46cba39e1"}, - {file = "pydantic_core-2.23.2-cp313-none-win_amd64.whl", hash = "sha256:582871902e1902b3c8e9b2c347f32a792a07094110c1bca6c2ea89b90150caac"}, - {file = "pydantic_core-2.23.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:743e5811b0c377eb830150d675b0847a74a44d4ad5ab8845923d5b3a756d8100"}, - {file = "pydantic_core-2.23.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6650a7bbe17a2717167e3e23c186849bae5cef35d38949549f1c116031b2b3aa"}, - {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56e6a12ec8d7679f41b3750ffa426d22b44ef97be226a9bab00a03365f217b2b"}, - {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:810ca06cca91de9107718dc83d9ac4d2e86efd6c02cba49a190abcaf33fb0472"}, - {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:785e7f517ebb9890813d31cb5d328fa5eda825bb205065cde760b3150e4de1f7"}, - {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ef71ec876fcc4d3bbf2ae81961959e8d62f8d74a83d116668409c224012e3af"}, - {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d50ac34835c6a4a0d456b5db559b82047403c4317b3bc73b3455fefdbdc54b0a"}, - {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16b25a4a120a2bb7dab51b81e3d9f3cde4f9a4456566c403ed29ac81bf49744f"}, - {file = "pydantic_core-2.23.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:41ae8537ad371ec018e3c5da0eb3f3e40ee1011eb9be1da7f965357c4623c501"}, - {file = "pydantic_core-2.23.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07049ec9306ec64e955b2e7c40c8d77dd78ea89adb97a2013d0b6e055c5ee4c5"}, - {file = "pydantic_core-2.23.2-cp38-none-win32.whl", hash = "sha256:086c5db95157dc84c63ff9d96ebb8856f47ce113c86b61065a066f8efbe80acf"}, - {file = "pydantic_core-2.23.2-cp38-none-win_amd64.whl", hash = "sha256:67b6655311b00581914aba481729971b88bb8bc7996206590700a3ac85e457b8"}, - {file = "pydantic_core-2.23.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:358331e21a897151e54d58e08d0219acf98ebb14c567267a87e971f3d2a3be59"}, - {file = "pydantic_core-2.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c4d9f15ffe68bcd3898b0ad7233af01b15c57d91cd1667f8d868e0eacbfe3f87"}, - {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0123655fedacf035ab10c23450163c2f65a4174f2bb034b188240a6cf06bb123"}, - {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e6e3ccebdbd6e53474b0bb7ab8b88e83c0cfe91484b25e058e581348ee5a01a5"}, - {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc535cb898ef88333cf317777ecdfe0faac1c2a3187ef7eb061b6f7ecf7e6bae"}, - {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aab9e522efff3993a9e98ab14263d4e20211e62da088298089a03056980a3e69"}, - {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05b366fb8fe3d8683b11ac35fa08947d7b92be78ec64e3277d03bd7f9b7cda79"}, - {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7568f682c06f10f30ef643a1e8eec4afeecdafde5c4af1b574c6df079e96f96c"}, - {file = "pydantic_core-2.23.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:cdd02a08205dc90238669f082747612cb3c82bd2c717adc60f9b9ecadb540f80"}, - {file = "pydantic_core-2.23.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a2ab4f410f4b886de53b6bddf5dd6f337915a29dd9f22f20f3099659536b2f6"}, - {file = "pydantic_core-2.23.2-cp39-none-win32.whl", hash = "sha256:0448b81c3dfcde439551bb04a9f41d7627f676b12701865c8a2574bcea034437"}, - {file = "pydantic_core-2.23.2-cp39-none-win_amd64.whl", hash = "sha256:4cebb9794f67266d65e7e4cbe5dcf063e29fc7b81c79dc9475bd476d9534150e"}, - {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e758d271ed0286d146cf7c04c539a5169a888dd0b57026be621547e756af55bc"}, - {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f477d26183e94eaafc60b983ab25af2a809a1b48ce4debb57b343f671b7a90b6"}, - {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da3131ef2b940b99106f29dfbc30d9505643f766704e14c5d5e504e6a480c35e"}, - {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:329a721253c7e4cbd7aad4a377745fbcc0607f9d72a3cc2102dd40519be75ed2"}, - {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7706e15cdbf42f8fab1e6425247dfa98f4a6f8c63746c995d6a2017f78e619ae"}, - {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e64ffaf8f6e17ca15eb48344d86a7a741454526f3a3fa56bc493ad9d7ec63936"}, - {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dd59638025160056687d598b054b64a79183f8065eae0d3f5ca523cde9943940"}, - {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:12625e69b1199e94b0ae1c9a95d000484ce9f0182f9965a26572f054b1537e44"}, - {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5d813fd871b3d5c3005157622ee102e8908ad6011ec915a18bd8fde673c4360e"}, - {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1eb37f7d6a8001c0f86dc8ff2ee8d08291a536d76e49e78cda8587bb54d8b329"}, - {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ce7eaf9a98680b4312b7cebcdd9352531c43db00fca586115845df388f3c465"}, - {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f087879f1ffde024dd2788a30d55acd67959dcf6c431e9d3682d1c491a0eb474"}, - {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ce883906810b4c3bd90e0ada1f9e808d9ecf1c5f0b60c6b8831d6100bcc7dd6"}, - {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a8031074a397a5925d06b590121f8339d34a5a74cfe6970f8a1124eb8b83f4ac"}, - {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:23af245b8f2f4ee9e2c99cb3f93d0e22fb5c16df3f2f643f5a8da5caff12a653"}, - {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c57e493a0faea1e4c38f860d6862ba6832723396c884fbf938ff5e9b224200e2"}, - {file = "pydantic_core-2.23.2.tar.gz", hash = "sha256:95d6bf449a1ac81de562d65d180af5d8c19672793c81877a2eda8fde5d08f2fd"}, + {file = "pydantic_core-2.23.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7f10a5d1b9281392f1bf507d16ac720e78285dfd635b05737c3911637601bae6"}, + {file = "pydantic_core-2.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c09a7885dd33ee8c65266e5aa7fb7e2f23d49d8043f089989726391dd7350c5"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6470b5a1ec4d1c2e9afe928c6cb37eb33381cab99292a708b8cb9aa89e62429b"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9172d2088e27d9a185ea0a6c8cebe227a9139fd90295221d7d495944d2367700"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86fc6c762ca7ac8fbbdff80d61b2c59fb6b7d144aa46e2d54d9e1b7b0e780e01"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0cb80fd5c2df4898693aa841425ea1727b1b6d2167448253077d2a49003e0ed"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03667cec5daf43ac4995cefa8aaf58f99de036204a37b889c24a80927b629cec"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:047531242f8e9c2db733599f1c612925de095e93c9cc0e599e96cf536aaf56ba"}, + {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5499798317fff7f25dbef9347f4451b91ac2a4330c6669821c8202fd354c7bee"}, + {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bbb5e45eab7624440516ee3722a3044b83fff4c0372efe183fd6ba678ff681fe"}, + {file = "pydantic_core-2.23.3-cp310-none-win32.whl", hash = "sha256:8b5b3ed73abb147704a6e9f556d8c5cb078f8c095be4588e669d315e0d11893b"}, + {file = "pydantic_core-2.23.3-cp310-none-win_amd64.whl", hash = "sha256:2b603cde285322758a0279995b5796d64b63060bfbe214b50a3ca23b5cee3e83"}, + {file = "pydantic_core-2.23.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c889fd87e1f1bbeb877c2ee56b63bb297de4636661cc9bbfcf4b34e5e925bc27"}, + {file = "pydantic_core-2.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea85bda3189fb27503af4c45273735bcde3dd31c1ab17d11f37b04877859ef45"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7f7f72f721223f33d3dc98a791666ebc6a91fa023ce63733709f4894a7dc611"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b2b55b0448e9da68f56b696f313949cda1039e8ec7b5d294285335b53104b61"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c24574c7e92e2c56379706b9a3f07c1e0c7f2f87a41b6ee86653100c4ce343e5"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2b05e6ccbee333a8f4b8f4d7c244fdb7a979e90977ad9c51ea31261e2085ce0"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c409ce1c219c091e47cb03feb3c4ed8c2b8e004efc940da0166aaee8f9d6c8"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d965e8b325f443ed3196db890d85dfebbb09f7384486a77461347f4adb1fa7f8"}, + {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f56af3a420fb1ffaf43ece3ea09c2d27c444e7c40dcb7c6e7cf57aae764f2b48"}, + {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5b01a078dd4f9a52494370af21aa52964e0a96d4862ac64ff7cea06e0f12d2c5"}, + {file = "pydantic_core-2.23.3-cp311-none-win32.whl", hash = "sha256:560e32f0df04ac69b3dd818f71339983f6d1f70eb99d4d1f8e9705fb6c34a5c1"}, + {file = "pydantic_core-2.23.3-cp311-none-win_amd64.whl", hash = "sha256:c744fa100fdea0d000d8bcddee95213d2de2e95b9c12be083370b2072333a0fa"}, + {file = "pydantic_core-2.23.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e0ec50663feedf64d21bad0809f5857bac1ce91deded203efc4a84b31b2e4305"}, + {file = "pydantic_core-2.23.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db6e6afcb95edbe6b357786684b71008499836e91f2a4a1e55b840955b341dbb"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98ccd69edcf49f0875d86942f4418a4e83eb3047f20eb897bffa62a5d419c8fa"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a678c1ac5c5ec5685af0133262103defb427114e62eafeda12f1357a12140162"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01491d8b4d8db9f3391d93b0df60701e644ff0894352947f31fff3e52bd5c801"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fcf31facf2796a2d3b7fe338fe8640aa0166e4e55b4cb108dbfd1058049bf4cb"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7200fd561fb3be06827340da066df4311d0b6b8eb0c2116a110be5245dceb326"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc1636770a809dee2bd44dd74b89cc80eb41172bcad8af75dd0bc182c2666d4c"}, + {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:67a5def279309f2e23014b608c4150b0c2d323bd7bccd27ff07b001c12c2415c"}, + {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:748bdf985014c6dd3e1e4cc3db90f1c3ecc7246ff5a3cd4ddab20c768b2f1dab"}, + {file = "pydantic_core-2.23.3-cp312-none-win32.whl", hash = "sha256:255ec6dcb899c115f1e2a64bc9ebc24cc0e3ab097775755244f77360d1f3c06c"}, + {file = "pydantic_core-2.23.3-cp312-none-win_amd64.whl", hash = "sha256:40b8441be16c1e940abebed83cd006ddb9e3737a279e339dbd6d31578b802f7b"}, + {file = "pydantic_core-2.23.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6daaf5b1ba1369a22c8b050b643250e3e5efc6a78366d323294aee54953a4d5f"}, + {file = "pydantic_core-2.23.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d015e63b985a78a3d4ccffd3bdf22b7c20b3bbd4b8227809b3e8e75bc37f9cb2"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3fc572d9b5b5cfe13f8e8a6e26271d5d13f80173724b738557a8c7f3a8a3791"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f6bd91345b5163ee7448bee201ed7dd601ca24f43f439109b0212e296eb5b423"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc379c73fd66606628b866f661e8785088afe2adaba78e6bbe80796baf708a63"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbdce4b47592f9e296e19ac31667daed8753c8367ebb34b9a9bd89dacaa299c9"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc3cf31edf405a161a0adad83246568647c54404739b614b1ff43dad2b02e6d5"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e22b477bf90db71c156f89a55bfe4d25177b81fce4aa09294d9e805eec13855"}, + {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:0a0137ddf462575d9bce863c4c95bac3493ba8e22f8c28ca94634b4a1d3e2bb4"}, + {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:203171e48946c3164fe7691fc349c79241ff8f28306abd4cad5f4f75ed80bc8d"}, + {file = "pydantic_core-2.23.3-cp313-none-win32.whl", hash = "sha256:76bdab0de4acb3f119c2a4bff740e0c7dc2e6de7692774620f7452ce11ca76c8"}, + {file = "pydantic_core-2.23.3-cp313-none-win_amd64.whl", hash = "sha256:37ba321ac2a46100c578a92e9a6aa33afe9ec99ffa084424291d84e456f490c1"}, + {file = "pydantic_core-2.23.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d063c6b9fed7d992bcbebfc9133f4c24b7a7f215d6b102f3e082b1117cddb72c"}, + {file = "pydantic_core-2.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6cb968da9a0746a0cf521b2b5ef25fc5a0bee9b9a1a8214e0a1cfaea5be7e8a4"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbefe079a520c5984e30e1f1f29325054b59534729c25b874a16a5048028d16"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbaaf2ef20d282659093913da9d402108203f7cb5955020bd8d1ae5a2325d1c4"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb539d7e5dc4aac345846f290cf504d2fd3c1be26ac4e8b5e4c2b688069ff4cf"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e6f33503c5495059148cc486867e1d24ca35df5fc064686e631e314d959ad5b"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04b07490bc2f6f2717b10c3969e1b830f5720b632f8ae2f3b8b1542394c47a8e"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03795b9e8a5d7fda05f3873efc3f59105e2dcff14231680296b87b80bb327295"}, + {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c483dab0f14b8d3f0df0c6c18d70b21b086f74c87ab03c59250dbf6d3c89baba"}, + {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8b2682038e255e94baf2c473dca914a7460069171ff5cdd4080be18ab8a7fd6e"}, + {file = "pydantic_core-2.23.3-cp38-none-win32.whl", hash = "sha256:f4a57db8966b3a1d1a350012839c6a0099f0898c56512dfade8a1fe5fb278710"}, + {file = "pydantic_core-2.23.3-cp38-none-win_amd64.whl", hash = "sha256:13dd45ba2561603681a2676ca56006d6dee94493f03d5cadc055d2055615c3ea"}, + {file = "pydantic_core-2.23.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82da2f4703894134a9f000e24965df73cc103e31e8c31906cc1ee89fde72cbd8"}, + {file = "pydantic_core-2.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dd9be0a42de08f4b58a3cc73a123f124f65c24698b95a54c1543065baca8cf0e"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89b731f25c80830c76fdb13705c68fef6a2b6dc494402987c7ea9584fe189f5d"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6de1ec30c4bb94f3a69c9f5f2182baeda5b809f806676675e9ef6b8dc936f28"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb68b41c3fa64587412b104294b9cbb027509dc2f6958446c502638d481525ef"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c3980f2843de5184656aab58698011b42763ccba11c4a8c35936c8dd6c7068c"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94f85614f2cba13f62c3c6481716e4adeae48e1eaa7e8bac379b9d177d93947a"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:510b7fb0a86dc8f10a8bb43bd2f97beb63cffad1203071dc434dac26453955cd"}, + {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1eba2f7ce3e30ee2170410e2171867ea73dbd692433b81a93758ab2de6c64835"}, + {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b259fd8409ab84b4041b7b3f24dcc41e4696f180b775961ca8142b5b21d0e70"}, + {file = "pydantic_core-2.23.3-cp39-none-win32.whl", hash = "sha256:40d9bd259538dba2f40963286009bf7caf18b5112b19d2b55b09c14dde6db6a7"}, + {file = "pydantic_core-2.23.3-cp39-none-win_amd64.whl", hash = "sha256:5a8cd3074a98ee70173a8633ad3c10e00dcb991ecec57263aacb4095c5efb958"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f399e8657c67313476a121a6944311fab377085ca7f490648c9af97fc732732d"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:6b5547d098c76e1694ba85f05b595720d7c60d342f24d5aad32c3049131fa5c4"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dda0290a6f608504882d9f7650975b4651ff91c85673341789a476b1159f211"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65b6e5da855e9c55a0c67f4db8a492bf13d8d3316a59999cfbaf98cc6e401961"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09e926397f392059ce0afdcac920df29d9c833256354d0c55f1584b0b70cf07e"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:87cfa0ed6b8c5bd6ae8b66de941cece179281239d482f363814d2b986b79cedc"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e61328920154b6a44d98cabcb709f10e8b74276bc709c9a513a8c37a18786cc4"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce3317d155628301d649fe5e16a99528d5680af4ec7aa70b90b8dacd2d725c9b"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e89513f014c6be0d17b00a9a7c81b1c426f4eb9224b15433f3d98c1a071f8433"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4f62c1c953d7ee375df5eb2e44ad50ce2f5aff931723b398b8bc6f0ac159791a"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2718443bc671c7ac331de4eef9b673063b10af32a0bb385019ad61dcf2cc8f6c"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d90e08b2727c5d01af1b5ef4121d2f0c99fbee692c762f4d9d0409c9da6541"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2b676583fc459c64146debea14ba3af54e540b61762dfc0613dc4e98c3f66eeb"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:50e4661f3337977740fdbfbae084ae5693e505ca2b3130a6d4eb0f2281dc43b8"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:68f4cf373f0de6abfe599a38307f4417c1c867ca381c03df27c873a9069cda25"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:59d52cf01854cb26c46958552a21acb10dd78a52aa34c86f284e66b209db8cab"}, + {file = "pydantic_core-2.23.3.tar.gz", hash = "sha256:3cb0f65d8b4121c1b015c60104a685feb929a29d7cf204387c7f2688c7974690"}, ] [package.dependencies] @@ -2643,17 +2643,6 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] -[[package]] -name = "tzdata" -version = "2024.1" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -files = [ - {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, - {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, -] - [[package]] name = "unidecode" version = "1.3.8" From f0ba024a8f9ec114d9045450bff3dfd76f0cce5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:07:41 +0000 Subject: [PATCH 082/254] Bump urllib3 from 2.2.1 to 2.2.3 Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.2.1 to 2.2.3. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.2.1...2.2.3) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2f19fa538..696492ec1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2674,13 +2674,13 @@ files = [ [[package]] name = "urllib3" -version = "2.2.1" +version = "2.2.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, - {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, ] [package.extras] From b77a02203852014cbb61f6f446014b229075aa5d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 16 Sep 2024 13:11:31 -0400 Subject: [PATCH 083/254] baddns direct initial storage bucket / cloudflare --- bbot/modules/baddns.py | 2 +- bbot/modules/baddns_direct.py | 62 +++++++++++++++++++++-------------- bbot/modules/baddns_zone.py | 2 +- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index ef77f11e7..d35a16bf8 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -22,7 +22,7 @@ class baddns(BaseModule): "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", } module_threads = 8 - deps_pip = ["baddns~=1.1.815"] + deps_pip = ["baddns~=1.1.839"] def select_modules(self): selected_submodules = [] diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 50e3d22dc..36e4641f0 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -6,8 +6,9 @@ import asyncio import logging + class baddns_direct(BaseModule): - watched_events = ["STORAGE_BUCKET"] + watched_events = ["URL", "STORAGE_BUCKET"] produced_events = ["FINDING", "VULNERABILITY"] flags = ["active", "safe", "subdomain-enum", "baddns", "cloud-enum"] meta = { @@ -40,37 +41,50 @@ def select_modules(self): selected_modules.append(m) return selected_modules - async def handle_event(self, event): - - self.critical("HANDLE EVENT") - parsed_url = urlparse(event.data["url"]) - domain = parsed_url.netloc - - self.critical(domain) - - - - CNAME_direct_module = self.select_modules()[0] + self.critical(event.type) + CNAME_direct_module = self.select_modules()[0] kwargs = { "http_client_class": self.scan.helpers.web.AsyncClient, "dns_client": self.scan.helpers.dns.resolver, "custom_nameservers": self.custom_nameservers, "signatures": self.signatures, + "direct_mode": True, } - CNAME_direct_instance = CNAME_direct_module(domain, **kwargs) - await CNAME_direct_instance.dispatch() - print(CNAME_direct_instance) - results = CNAME_direct_instance.analyze() - self.hugewarning(results) - if results and len(results) > 0: - for r in results: - r_dict = r.to_dict() - self.critical(r_dict) + CNAME_direct_instance = CNAME_direct_module(event.host, **kwargs) + if await CNAME_direct_instance.dispatch(): + results = CNAME_direct_instance.analyze() + if results and len(results) > 0: + for r in results: + r_dict = r.to_dict() + + data = { + "description": f"Possible [{r_dict['signature']}] via direct BadDNS analysis. Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]", + "host": str(event.host), + } + + await self.emit_event( + data, + "FINDING", + event, + tags=[f"baddns-{CNAME_direct_module.name.lower()}"], + context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}', + ) async def filter_event(self, event): - if event.type == "STORAGE_BUCKET" and str(event.module).startswith("bucket_"): - self.critical("KILLED BUCKET") - return False + if event.type == "STORAGE_BUCKET": + if str(event.module).startswith("bucket_"): + return False + if event.type == "URL": + if event.scope_distance > 0: + self.critical( + f"Rejecting {event.host} due to not being in scope (scope distance: {str(event.scope_distance)})" + ) + return False + if "cdn-cloudflare" not in event.tags: + self.critical(f"Rejecting {event.host} due to not being behind CloudFlare") + return False + if "status-200" in event.tags or "status-301" in event.tags: + self.critical(f"Rejecting {event.host} due to lack of non-standard status code") return True diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index a356f61b3..d242fdaab 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -17,7 +17,7 @@ class baddns_zone(baddns_module): "only_high_confidence": "Do not emit low-confidence or generic detections", } module_threads = 8 - deps_pip = ["baddns~=1.1.815"] + deps_pip = ["baddns~=1.1.839"] def select_modules(self): selected_modules = [] From c82585a5442828d20a0b548f7d54d42a696d0c17 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 16 Sep 2024 13:30:35 -0400 Subject: [PATCH 084/254] adding direct to preset --- bbot/presets/baddns-thorough.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/presets/baddns-thorough.yml b/bbot/presets/baddns-thorough.yml index 92c05a99a..08c335a69 100644 --- a/bbot/presets/baddns-thorough.yml +++ b/bbot/presets/baddns-thorough.yml @@ -4,6 +4,7 @@ description: Run all baddns and baddns_zone submodules. modules: - baddns - baddns_zone + - baddns_direct config: modules: From f6c19a0bf40973dadffa599fff337ceafb93675d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 16 Sep 2024 13:33:11 -0400 Subject: [PATCH 085/254] flake8 --- bbot/modules/baddns_direct.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 36e4641f0..33c0b879e 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -1,9 +1,7 @@ from baddns.base import get_all_modules from baddns.lib.loader import load_signatures -from urllib.parse import urlparse from .base import BaseModule -import asyncio import logging From e725cec43e13c732ff3ccec9ed26c1ba7fc6edbd Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 16 Sep 2024 13:37:12 -0400 Subject: [PATCH 086/254] missing return --- bbot/modules/baddns_direct.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 33c0b879e..475f6b780 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -85,4 +85,5 @@ async def filter_event(self, event): return False if "status-200" in event.tags or "status-301" in event.tags: self.critical(f"Rejecting {event.host} due to lack of non-standard status code") + return False return True From 3a8636c6e955a1553d4cc8f0097d69112c6de300 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 17 Sep 2024 14:53:47 -0400 Subject: [PATCH 087/254] fix output_dir preset bug --- bbot/core/engine.py | 2 +- bbot/modules/output/neo4j.py | 2 +- bbot/scanner/preset/args.py | 25 +++++++++------- bbot/test/test_step_1/test_cli.py | 41 +++++++++++++++++++++++++++ bbot/test/test_step_1/test_presets.py | 17 +++++++++++ 5 files changed, 74 insertions(+), 13 deletions(-) diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 9d42c9719..5f12bfb84 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -666,7 +666,7 @@ async def cancel_task(self, client_id): for task in [parent_task] + list(child_tasks): await self._cancel_task(task) - async def _cancel_task(self, task): + async def _await_cancelled_task(self, task): try: await asyncio.wait_for(task, timeout=10) except (TimeoutError, asyncio.exceptions.TimeoutError): diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index f9219ab36..b859af516 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -53,7 +53,7 @@ async def setup(self): ), ) self.session = self.driver.session() - await self.handle_event(self.scan.root_event) + await self.session.run("Match () Return 1 Limit 1") except Exception as e: return False, f"Error setting up Neo4j: {e}" return True diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 9886cae4a..986fd909f 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -95,14 +95,6 @@ def preset_from_args(self): name="args_preset", ) - # then we set verbosity levels (so if the user enables -d they can see debug output) - if self.parsed.silent: - args_preset.silent = True - if self.parsed.verbose: - args_preset.verbose = True - if self.parsed.debug: - args_preset.debug = True - # then we load requested preset # this is important so we can load custom module directories, pull in custom flags, module config options, etc. for preset_arg in self.parsed.preset: @@ -113,6 +105,14 @@ def preset_from_args(self): except Exception as e: raise BBOTArgumentError(f'Error parsing preset "{preset_arg}": {e}') + # then we set verbosity levels (so if the user enables -d they can see debug output) + if self.parsed.silent: + args_preset.silent = True + if self.parsed.verbose: + args_preset.verbose = True + if self.parsed.debug: + args_preset.debug = True + # modules + flags args_preset.exclude_modules.update(set(self.parsed.exclude_modules)) args_preset.exclude_flags.update(set(self.parsed.exclude_flags)) @@ -142,9 +142,12 @@ def preset_from_args(self): args_preset.core.custom_config["deps_behavior"] = "ignore_failed" # other scan options - args_preset.scan_name = self.parsed.name - args_preset.output_dir = self.parsed.output_dir - args_preset.force_start = self.parsed.force + if self.parsed.name is not None: + args_preset.scan_name = self.parsed.name + if self.parsed.output_dir is not None: + args_preset.output_dir = self.parsed.output_dir + if self.parsed.force: + args_preset.force_start = self.parsed.force if self.parsed.custom_headers: args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}}) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index be861b8a2..af87f8eb1 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -661,3 +661,44 @@ def test_cli_presets(monkeypatch, capsys, caplog): preset1_file.unlink() preset2_file.unlink() + + # test output dir preset + output_dir_preset_file = preset_dir / "output_dir_preset.yml" + scan_name = "cli_output_dir_test" + output_dir = bbot_test_dir / "cli_output_dir_preset" + scan_dir = output_dir / scan_name + output_file = scan_dir / "output.txt" + + with open(output_dir_preset_file, "w") as f: + f.write( + f""" +output_dir: {output_dir} +scan_name: {scan_name} + """ + ) + + assert not output_dir.exists() + assert not scan_dir.exists() + assert not output_file.exists() + + monkeypatch.setattr("sys.argv", ["bbot", "-p", str(output_dir_preset_file.resolve()), "--current-preset"]) + cli.main() + captured = capsys.readouterr() + stdout_preset = yaml.safe_load(captured.out) + assert stdout_preset["output_dir"] == str(output_dir) + assert stdout_preset["scan_name"] == scan_name + + shutil.rmtree(output_dir, ignore_errors=True) + shutil.rmtree(scan_dir, ignore_errors=True) + shutil.rmtree(output_file, ignore_errors=True) + + assert not output_dir.exists() + assert not scan_dir.exists() + assert not output_file.exists() + + monkeypatch.setattr("sys.argv", ["bbot", "-p", str(output_dir_preset_file.resolve())]) + cli.main() + captured = capsys.readouterr() + assert output_dir.is_dir() + assert scan_dir.is_dir() + assert output_file.is_file() diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 4488318b6..ccf8d8c14 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -902,3 +902,20 @@ def get_module_flags(p): assert any("active" in flags for module, flags in module_flags) assert any("safe" in flags for module, flags in module_flags) assert any("aggressive" in flags for module, flags in module_flags) + + +@pytest.mark.asyncio +async def test_preset_output_dir(): + output_dir = bbot_test_dir / "preset_output_dir" + preset = Preset.from_yaml_string( + f""" +output_dir: {output_dir} +scan_name: bbot_test +""" + ) + scan = Scanner(preset=preset) + await scan.async_start_without_generator() + scan_dir = output_dir / "bbot_test" + assert scan_dir.is_dir() + output_file = scan_dir / "output.txt" + assert output_file.is_file() From 3f6051aa14282c604f102b778788132b1f3a88c0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 17 Sep 2024 15:26:15 -0400 Subject: [PATCH 088/254] await canceled tasks --- bbot/core/engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 5f12bfb84..26288ab8d 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -643,6 +643,7 @@ async def finished_tasks(self, tasks, timeout=None): self.log.warning(f"{self.name}: Timeout after {timeout:,} seconds in finished_tasks({tasks})") for task in tasks: task.cancel() + self._await_cancelled_task(task) else: if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)): self.log.error(f"{self.name}: Unhandled exception in finished_tasks({tasks}): {e}") @@ -664,7 +665,7 @@ async def cancel_task(self, client_id): child_task.cancel() for task in [parent_task] + list(child_tasks): - await self._cancel_task(task) + await self._await_cancelled_task(task) async def _await_cancelled_task(self, task): try: @@ -683,4 +684,4 @@ async def cancel_all_tasks(self): await self.cancel_task(client_id) for client_id, tasks in self.child_tasks.items(): for task in tasks: - await self._cancel_task(task) + await self._await_cancelled_task(task) From 4e92e2968f7342557f82931a066509d6cc7e36a9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 17 Sep 2024 16:06:58 -0400 Subject: [PATCH 089/254] fix tests --- bbot/test/test_step_1/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index af87f8eb1..37e2fb25c 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -663,7 +663,7 @@ def test_cli_presets(monkeypatch, capsys, caplog): preset2_file.unlink() # test output dir preset - output_dir_preset_file = preset_dir / "output_dir_preset.yml" + output_dir_preset_file = bbot_test_dir / "output_dir_preset.yml" scan_name = "cli_output_dir_test" output_dir = bbot_test_dir / "cli_output_dir_preset" scan_dir = output_dir / scan_name From b1bed05ac1a6e6da4f3afa5e6cb9804cd0c3457c Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Wed, 18 Sep 2024 18:54:03 +0100 Subject: [PATCH 090/254] Add a new module to download postman workspaces --- bbot/modules/postman.py | 8 - bbot/modules/postman_download.py | 184 +++++++++++ bbot/modules/templates/postman.py | 9 + .../test_module_postman_download.py | 285 ++++++++++++++++++ 4 files changed, 478 insertions(+), 8 deletions(-) create mode 100644 bbot/modules/postman_download.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_postman_download.py diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index 982c9ff96..82a3c8b28 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -11,14 +11,6 @@ class postman(postman): "author": "@domwhewell-sage", } - headers = { - "Content-Type": "application/json", - "X-App-Version": "10.18.8-230926-0808", - "X-Entity-Team-Id": "0", - "Origin": "https://www.postman.com", - "Referer": "https://www.postman.com/search?q=&scope=public&type=all", - } - reject_wildcards = False async def handle_event(self, event): diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py new file mode 100644 index 000000000..807f2e9e9 --- /dev/null +++ b/bbot/modules/postman_download.py @@ -0,0 +1,184 @@ +import zipfile +import json +from pathlib import Path +from bbot.modules.templates.postman import postman + + +class postman_download(postman): + watched_events = ["CODE_REPOSITORY"] + produced_events = ["FILESYSTEM"] + flags = ["passive", "subdomain-enum", "safe", "code-enum"] + meta = { + "description": "Download workspaces, collections, requests from Postman", + "created_date": "2024-09-07", + "author": "@domwhewell-sage", + } + options = {"output_folder": "", "api_key": ""} + options_desc = {"output_folder": "Folder to download postman workspaces to"} + scope_distance_modifier = 2 + + async def setup(self): + self.api_key = self.config.get("api_key", "") + self.authorization_headers = {"X-Api-Key": self.api_key} + + output_folder = self.config.get("output_folder") + if output_folder: + self.output_dir = Path(output_folder) / "postman_workspaces" + else: + self.output_dir = self.scan.home / "postman_workspaces" + self.helpers.mkdir(self.output_dir) + return await self.require_api_key() + + async def ping(self): + url = f"{self.api_url}/me" + response = await self.helpers.request(url, headers=self.authorization_headers) + assert getattr(response, "status_code", 0) == 200, response.text + + async def filter_event(self, event): + if event.type == "CODE_REPOSITORY": + if "postman" not in event.tags: + return False, "event is not a postman workspace" + return True + + async def handle_event(self, event): + repo_url = event.data.get("url") + workspace_id = await self.get_workspace_id(repo_url) + if workspace_id: + self.verbose(f"Found workspace ID {workspace_id} for {repo_url}") + workspace_path = await self.download_workspace(workspace_id) + if workspace_path: + self.verbose(f"Downloaded workspace from {repo_url} to {workspace_path}") + codebase_event = self.make_event( + {"path": str(workspace_path)}, "FILESYSTEM", tags=["postman", "workspace"], parent=event + ) + await self.emit_event( + codebase_event, + context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}", + ) + + async def get_workspace_id(self, repo_url): + workspace_id = "" + profile = repo_url.split("/")[-2] + name = repo_url.split("/")[-1] + url = f"{self.base_url}/ws/proxy" + json = { + "service": "workspaces", + "method": "GET", + "path": f"/workspaces?handle={profile}&slug={name}", + } + r = await self.helpers.request(url, method="POST", json=json, headers=self.headers) + if r is None: + return workspace_id + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return workspace_id + data = json.get("data", []) + if len(data) == 1: + workspace_id = data[0]["id"] + return workspace_id + + async def download_workspace(self, id): + zip_path = None + workspace = await self.get_workspace(id) + if workspace: + # Create a folder for the workspace + name = workspace["name"] + folder = self.output_dir / name + self.helpers.mkdir(folder) + zip_path = folder / f"{id}.zip" + + # Main Workspace + self.add_json_to_zip(zip_path, workspace, f"{name}.postman_workspace.json") + + # Workspace global variables + self.verbose(f"Downloading globals for workspace {name}") + globals = await self.get_globals(id) + globals_id = globals["id"] + self.add_json_to_zip(zip_path, globals, f"{globals_id}.postman_environment.json") + + # Workspace Environments + workspace_environments = workspace.get("environments", []) + if workspace_environments: + self.verbose(f"Downloading environments for workspace {name}") + for _ in workspace_environments: + environment_id = _["uid"] + environment = await self.get_environment(environment_id) + self.add_json_to_zip(zip_path, environment, f"{environment_id}.postman_environment.json") + + # Workspace Collections + workspace_collections = workspace.get("collections", []) + if workspace_collections: + self.verbose(f"Downloading collections for workspace {name}") + for _ in workspace_collections: + collection_id = _["uid"] + collection = await self.get_collection(collection_id) + self.add_json_to_zip(zip_path, collection, f"{collection_id}.postman_collection.json") + return zip_path + + async def get_workspace(self, workspace_id): + workspace = {} + workspace_url = f"{self.api_url}/workspaces/{workspace_id}" + r = await self.helpers.request(workspace_url, headers=self.authorization_headers) + if r is None: + return workspace + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return workspace + workspace = json.get("workspace", {}) + return workspace + + async def get_globals(self, workspace_id): + globals = {} + globals_url = f"{self.base_url}/workspace/{workspace_id}/globals" + r = await self.helpers.request(globals_url, headers=self.headers) + if r is None: + return globals + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return globals + globals = json.get("data", {}) + return globals + + async def get_environment(self, environment_id): + environment = {} + environment_url = f"{self.api_url}/environments/{environment_id}" + r = await self.helpers.request(environment_url, headers=self.authorization_headers) + if r is None: + return environment + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return environment + environment = json.get("environment", {}) + return environment + + async def get_collection(self, collection_id): + collection = {} + collection_url = f"{self.api_url}/collections/{collection_id}" + r = await self.helpers.request(collection_url, headers=self.authorization_headers) + if r is None: + return collection + status_code = getattr(r, "status_code", 0) + try: + json = r.json() + except Exception as e: + self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}") + return collection + collection = json.get("collection", {}) + return collection + + def add_json_to_zip(self, zip_path, data, filename): + with zipfile.ZipFile(zip_path, "a") as zipf: + json_content = json.dumps(data, indent=4) + zipf.writestr(filename, json_content) diff --git a/bbot/modules/templates/postman.py b/bbot/modules/templates/postman.py index ec2f987b0..38cc3d04b 100644 --- a/bbot/modules/templates/postman.py +++ b/bbot/modules/templates/postman.py @@ -8,4 +8,13 @@ class postman(BaseModule): """ base_url = "https://www.postman.com/_api" + api_url = "https://api.getpostman.com" html_url = "https://www.postman.com" + + headers = { + "Content-Type": "application/json", + "X-App-Version": "10.18.8-230926-0808", + "X-Entity-Team-Id": "0", + "Origin": "https://www.postman.com", + "Referer": "https://www.postman.com/search?q=&scope=public&type=all", + } diff --git a/bbot/test/test_step_2/module_tests/test_module_postman_download.py b/bbot/test/test_step_2/module_tests/test_module_postman_download.py new file mode 100644 index 000000000..83b33f9c5 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_postman_download.py @@ -0,0 +1,285 @@ +from .base import ModuleTestBase + + +class TestPostman_Download(ModuleTestBase): + config_overrides = {"modules": {"postman_download": {"api_key": "asdf"}}} + modules_overrides = ["postman", "postman_download", "speculate"] + + async def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/me", + json={ + "user": { + "id": 000000, + "username": "test_key", + "email": "blacklanternsecurity@test.com", + "fullName": "Test Key", + "avatar": "", + "isPublic": True, + "teamId": 0, + "teamDomain": "", + "roles": ["user"], + }, + "operations": [ + {"name": "api_object_usage", "limit": 3, "usage": 0, "overage": 0}, + {"name": "collection_run_limit", "limit": 25, "usage": 0, "overage": 0}, + {"name": "file_storage_limit", "limit": 20, "usage": 0, "overage": 0}, + {"name": "flow_count", "limit": 5, "usage": 0, "overage": 0}, + {"name": "flow_requests", "limit": 5000, "usage": 0, "overage": 0}, + {"name": "performance_test_limit", "limit": 25, "usage": 0, "overage": 0}, + {"name": "postbot_calls", "limit": 50, "usage": 0, "overage": 0}, + {"name": "reusable_packages", "limit": 3, "usage": 0, "overage": 0}, + {"name": "test_data_retrieval", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "test_data_storage", "limit": 10, "usage": 0, "overage": 0}, + {"name": "mock_usage", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "monitor_request_runs", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "api_usage", "limit": 1000, "usage": 0, "overage": 0}, + ], + }, + ) + + async def setup_after_prep(self, module_test): + await module_test.mock_dns( + {"blacklanternsecurity.com": {"A": ["127.0.0.99"]}, "github.com": {"A": ["127.0.0.99"]}} + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 100, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}', + json={ + "data": [ + { + "score": 611.41156, + "normalizedScore": 23, + "document": { + "watcherCount": 6, + "apiCount": 0, + "forkCount": 0, + "isblacklisted": "false", + "createdAt": "2021-06-15T14:03:51", + "publishertype": "team", + "publisherHandle": "blacklanternsecurity", + "id": "11498add-357d-4bc5-a008-0a2d44fb8829", + "slug": "bbot-public", + "updatedAt": "2024-07-30T11:00:35", + "entityType": "workspace", + "visibilityStatus": "public", + "forkcount": "0", + "tags": [], + "createdat": "2021-06-15T14:03:51", + "forkLabel": "", + "publisherName": "blacklanternsecurity", + "name": "BlackLanternSecurity BBOT [Public]", + "dependencyCount": 7, + "collectionCount": 6, + "warehouse__updated_at": "2024-07-30 11:00:00", + "privateNetworkFolders": [], + "isPublisherVerified": False, + "publisherType": "team", + "curatedInList": [], + "creatorId": "6900157", + "description": "", + "forklabel": "", + "publisherId": "299401", + "publisherLogo": "", + "popularity": 5, + "isPublic": True, + "categories": [], + "universaltags": "", + "views": 5788, + "summary": "BLS public workspaces.", + "memberCount": 2, + "isBlacklisted": False, + "publisherid": "299401", + "isPrivateNetworkEntity": False, + "isDomainNonTrivial": True, + "privateNetworkMeta": "", + "updatedat": "2021-10-20T16:19:29", + "documentType": "workspace", + }, + "highlight": {"summary": "BLS BBOT api test."}, + } + ], + "meta": { + "queryText": "blacklanternsecurity", + "total": { + "collection": 0, + "request": 0, + "workspace": 1, + "api": 0, + "team": 0, + "user": 0, + "flow": 0, + "apiDefinition": 0, + "privateNetworkFolder": 0, + }, + "state": "AQ4", + "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None}, + "featureFlags": { + "enabledPublicResultCuration": True, + "boostByPopularity": True, + "reRankPostNormalization": True, + "enableUrlBarHostNameSearch": True, + }, + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}', + json={ + "meta": {"model": "workspace", "action": "find", "nextCursor": ""}, + "data": [ + { + "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "name": "BlackLanternSecurity BBOT [Public]", + "description": None, + "summary": "BLS public workspaces.", + "createdBy": "299401", + "updatedBy": "299401", + "team": None, + "createdAt": "2021-10-20T16:19:29", + "updatedAt": "2021-10-20T16:19:29", + "visibilityStatus": "public", + "profileInfo": { + "slug": "bbot-public", + "profileType": "team", + "profileId": "000000", + "publicHandle": "https://www.postman.com/blacklanternsecurity", + "publicImageURL": "", + "publicName": "BlackLanternSecurity", + "isVerified": False, + }, + } + ], + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + json={ + "workspace": { + "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "name": "BlackLanternSecurity BBOT [Public]", + "type": "personal", + "description": None, + "visibility": "public", + "createdBy": "00000000", + "updatedBy": "00000000", + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-17T08:57:16.000Z", + "collections": [ + { + "id": "2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + "name": "BBOT Public", + "uid": "10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + }, + ], + "environments": [ + { + "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", + "name": "BBOT Test", + "uid": "10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + } + ], + "apis": [], + } + }, + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals", + json={ + "model_id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "meta": {"model": "globals", "action": "find"}, + "data": { + "workspace": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "lastUpdatedBy": "00000000", + "lastRevision": 1637239113000, + "id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "values": [ + { + "key": "endpoint_url", + "value": "https://api.blacklanternsecurity.com/", + "enabled": True, + }, + ], + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-18T12:38:33.000Z", + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + json={ + "environment": { + "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", + "name": "BBOT Test", + "owner": "00000000", + "createdAt": "2021-11-17T06:29:54.000Z", + "updatedAt": "2021-11-23T07:06:53.000Z", + "values": [ + { + "key": "temp_session_endpoint", + "value": "https://api.blacklanternsecurity.com/", + "enabled": True, + }, + ], + "isPublic": True, + } + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/collections/10197090-62b91565-d2e2-4bcd-8248-4dba2e3452f0", + json={ + "collection": { + "info": { + "_postman_id": "62b91565-d2e2-4bcd-8248-4dba2e3452f0", + "name": "BBOT Public", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "updatedAt": "2021-11-17T07:13:16.000Z", + "createdAt": "2021-11-17T07:13:15.000Z", + "lastUpdatedBy": "00000000", + "uid": "00000000-62b91565-d2e2-4bcd-8248-4dba2e3452f0", + }, + "item": [ + { + "name": "Generate API Session", + "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + "protocolProfileBehavior": {"disableBodyPruning": True}, + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": '{"username": "test", "password": "Test"}', + }, + "url": {"raw": "{{endpoint_url}}", "host": ["{{endpoint_url}}"]}, + "description": "", + }, + "response": [], + "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + }, + ], + } + }, + ) + + def check(self, module_test, events): + assert 1 == len( + [ + e + for e in events + if e.type == "CODE_REPOSITORY" + and "postman" in e.tags + and e.data["url"] == "https://www.postman.com/blacklanternsecurity/bbot-public" + and e.scope_distance == 1 + ] + ), "Failed to find blacklanternsecurity postman workspace" + assert 1 == len( + [ + e + for e in events + if e.type == "FILESYSTEM" + and "postman_workspaces/BlackLanternSecurity BBOT [Public]" in e.data["path"] + and "postman" in e.tags + and e.scope_distance == 1 + ] + ), "Failed to find blacklanternsecurity postman workspace" From e19416d97556a5f3ee5a83979b2773b7bcc2ca07 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Wed, 18 Sep 2024 20:21:17 +0100 Subject: [PATCH 091/254] Add a description for the api_key --- bbot/modules/postman_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 807f2e9e9..be8b93aff 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -14,7 +14,7 @@ class postman_download(postman): "author": "@domwhewell-sage", } options = {"output_folder": "", "api_key": ""} - options_desc = {"output_folder": "Folder to download postman workspaces to"} + options_desc = {"output_folder": "Folder to download postman workspaces to", "api_key": "Postman API Key"} scope_distance_modifier = 2 async def setup(self): From c8b15ffc730349f5178a2dcd05a8cf59aa9134dc Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 19 Sep 2024 16:51:59 -0400 Subject: [PATCH 092/254] add description --- bbot/test/test_step_1/test_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 37e2fb25c..e203a440f 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -672,6 +672,7 @@ def test_cli_presets(monkeypatch, capsys, caplog): with open(output_dir_preset_file, "w") as f: f.write( f""" +description: test preset output_dir: {output_dir} scan_name: {scan_name} """ From 447356d9a0cbfda0432b1e37a47d381b07dc7f16 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 19 Sep 2024 20:01:07 -0400 Subject: [PATCH 093/254] fix tests --- bbot/test/test_step_1/test_cli.py | 6 +++++- bbot/test/test_step_1/test_presets.py | 2 ++ docs/dev/module_howto.md | 20 +++++++++++++++----- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index e203a440f..bf7476d49 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -672,7 +672,6 @@ def test_cli_presets(monkeypatch, capsys, caplog): with open(output_dir_preset_file, "w") as f: f.write( f""" -description: test preset output_dir: {output_dir} scan_name: {scan_name} """ @@ -703,3 +702,8 @@ def test_cli_presets(monkeypatch, capsys, caplog): assert output_dir.is_dir() assert scan_dir.is_dir() assert output_file.is_file() + + shutil.rmtree(output_dir, ignore_errors=True) + shutil.rmtree(scan_dir, ignore_errors=True) + shutil.rmtree(output_file, ignore_errors=True) + output_dir_preset_file.unlink() diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index ccf8d8c14..799907474 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -919,3 +919,5 @@ async def test_preset_output_dir(): assert scan_dir.is_dir() output_file = scan_dir / "output.txt" assert output_file.is_file() + + shutil.rmtree(output_dir, ignore_errors=True) diff --git a/docs/dev/module_howto.md b/docs/dev/module_howto.md index ff37fd98c..0096c8d57 100644 --- a/docs/dev/module_howto.md +++ b/docs/dev/module_howto.md @@ -4,7 +4,7 @@ Here we'll go over a basic example of writing a custom BBOT module. ## Create the python file -1. Create a new `.py` file in `bbot/modules` +1. Create a new `.py` file in `bbot/modules` (or in a [custom module directory](#custom-module-directory)) 1. At the top of the file, import `BaseModule` 1. Declare a class that inherits from `BaseModule` - the class must have the same name as your file (case-insensitive) @@ -55,9 +55,9 @@ After saving the module, you can run it with `-m`: bbot -t evilcorp.com -m whois ``` -### Debugging Your Module - BBOT's Colorful Log Functions +### Debugging Your Module -You probably noticed the use of `self.hugesuccess()`. This function is part of BBOT's builtin logging capabilty, and it prints whatever you give it in bright green. These colorful log functions can be useful for debugging. +BBOT has a variety of colorful logging functions like `self.hugesuccess()` that can be useful for debugging. **BBOT log levels**: @@ -68,8 +68,8 @@ You probably noticed the use of `self.hugesuccess()`. This function is part of B - `error`: red - `warning`: orange - `info`: blue -- `verbose`: grey (must use `-v` to see) -- `debug`: grey (must use `-d` to see) +- `verbose`: grey (must enable `-v` to see) +- `debug`: grey (must enable `-d` to see) For details on how tests are written, see [Unit Tests](./tests.md). @@ -191,3 +191,13 @@ class MyModule(BaseModule): }, ] ``` + +## Load Modules from Custom Locations + +If you have a custom module and you want to use it with BBOT, you can add its parent folder to `module_paths`. This saves you from having to copy it into the BBOT install location. To add a custom module directory, add it to `module_paths` in your preset: + +```yaml title="my_preset.yml" +# load BBOT modules from these additional paths +module_paths: + - /home/user/my_modules +``` From 85509bd1db0f314573e781b93c2dc3c4d43c1592 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 19 Sep 2024 21:26:23 -0400 Subject: [PATCH 094/254] description --- bbot/scanner/scanner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index be25768f1..3acc42224 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -153,6 +153,10 @@ def __init__( scan_name = str(preset.scan_name) self.name = scan_name + # make sure the preset has a description + if not self.preset.description: + self.preset.description = self.name + # scan output dir if preset.output_dir is not None: self.home = Path(preset.output_dir).resolve() / self.name From 46df97a532aa93882da911ee49cac7981e240753 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 19 Sep 2024 21:50:51 -0400 Subject: [PATCH 095/254] more test things --- bbot/scanner/preset/preset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 99ad86db5..65f65070e 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -731,6 +731,9 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False) """ preset_dict = {} + if self.description: + preset_dict["description"] = self.description + # config if full_config: config = self.core.config From a32e216925a224873069e804a8afe316e94d387b Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 20 Sep 2024 10:17:31 -0400 Subject: [PATCH 096/254] misc bugfixes --- .github/workflows/tests.yml | 2 ++ bbot/core/modules.py | 20 +++++++++--- bbot/defaults.yml | 2 +- bbot/scanner/preset/path.py | 7 +++-- bbot/scanner/preset/preset.py | 3 ++ bbot/test/test_step_1/test_presets.py | 40 +++++++++++++++++++++++- bbot/test/test_step_1/test_python_api.py | 5 +-- docs/dev/module_howto.md | 4 +-- docs/scanning/configuration.md | 2 +- docs/scanning/presets.md | 10 ++++++ 10 files changed, 80 insertions(+), 15 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5bbd3bf4e..c0ed632ac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,6 +88,8 @@ jobs: if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') steps: - uses: actions/checkout@v3 + with: + token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} - uses: actions/setup-python@v4 with: python-version: "3.x" diff --git a/bbot/core/modules.py b/bbot/core/modules.py index dd1e43698..e64064bbd 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -408,6 +408,7 @@ def preload_module(self, module_file): task["become"] = False preloaded_data = { + "path": str(module_file.resolve()), "watched_events": sorted(watched_events), "produced_events": sorted(produced_events), "flags": sorted(flags), @@ -467,14 +468,23 @@ def load_module(self, module_name): >>> isinstance(module, object) True """ - namespace = self._preloaded[module_name]["namespace"] - import_path = f"{namespace}.{module_name}" - module_variables = importlib.import_module(import_path, "bbot") + preloaded = self._preloaded[module_name] + namespace = preloaded["namespace"] + try: + module_path = preloaded["path"] + except KeyError: + module_path = preloaded["cache_key"][0] + full_namespace = f"{namespace}.{module_name}" + + spec = importlib.util.spec_from_file_location(full_namespace, module_path) + module = importlib.util.module_from_spec(spec) + # sys.modules[module_name] = module + spec.loader.exec_module(module) # for every top-level variable in the .py file - for variable in module_variables.__dict__.keys(): + for variable in module.__dict__.keys(): # get its value - value = getattr(module_variables, variable) + value = getattr(module, variable) with suppress(AttributeError): # if it has watched_events and produced_events if all( diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 18591bfda..ca215a1ed 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -113,7 +113,7 @@ deps: ### ADVANCED OPTIONS ### # Load BBOT modules from these custom paths -module_paths: [] +module_dirs: [] # Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc. speculate: True diff --git a/bbot/scanner/preset/path.py b/bbot/scanner/preset/path.py index ee5235fbf..730b16e63 100644 --- a/bbot/scanner/preset/path.py +++ b/bbot/scanner/preset/path.py @@ -37,9 +37,10 @@ def find(self, filename): for path in paths_to_search: for candidate in file_candidates: for file in path.rglob(candidate): - log.verbose(f'Found preset matching "{filename}" at {file}') - self.add_path(file.parent) - return file.resolve() + if file.is_file(): + log.verbose(f'Found preset matching "{filename}" at {file}') + self.add_path(file.parent) + return file.resolve() raise ValidationError( f'Could not find preset at "{filename}" - file does not exist. Use -lp to list available presets' ) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 65f65070e..74d4ab48a 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -355,6 +355,9 @@ def merge(self, other): self._blacklist.update(other._blacklist) self.strict_scope = self.strict_scope or other.strict_scope + # module dirs + self.module_dirs = self.module_dirs.union(other.module_dirs) + # log verbosity if other.silent: self.silent = other.silent diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 799907474..26c5505b1 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -532,7 +532,8 @@ def test_preset_module_resolution(clean_default_config): assert str(error.value) == 'Unable to add scan module "sslcert" because the module has been excluded' -def test_preset_module_loader(): +@pytest.mark.asyncio +async def test_preset_module_loader(): custom_module_dir = bbot_test_dir / "custom_module_dir" custom_module_dir_2 = custom_module_dir / "asdf" custom_output_module_dir = custom_module_dir / "output" @@ -640,6 +641,43 @@ class TestModule4(BaseModule): # reset module_loader preset2.module_loader.__init__() + # custom module dir via preset + custom_module_dir_3 = bbot_test_dir / "custom_module_dir_3" + custom_module_dir_3.mkdir(exist_ok=True, parents=True) + custom_module_5 = custom_module_dir_3 / "testmodule5.py" + with open(custom_module_5, "w") as f: + f.write( + """ +from bbot.modules.base import BaseModule + +class TestModule5(BaseModule): + watched_events = ["TECHNOLOGY"] + produced_events = ["FINDING"] +""" + ) + + preset = Preset.from_yaml_string( + f""" +modules: + - testmodule5 +""" + ) + # should fail + with pytest.raises(ValidationError): + scan = Scanner(preset=preset) + + preset = Preset.from_yaml_string( + f""" +module_dirs: + - {custom_module_dir_3} +modules: + - testmodule5 +""" + ) + scan = Scanner(preset=preset) + await scan._prep() + assert "testmodule5" in scan.modules + def test_preset_include(): diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 1c2b0bb51..60ab89286 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -122,7 +122,8 @@ def test_python_api_validation(): assert str(error.value) == 'Preset must be of type Preset, not "str"' # include nonexistent preset with pytest.raises(ValidationError) as error: - Preset(include=["asdf"]) + Preset(include=["nonexistent"]) assert ( - str(error.value) == 'Could not find preset at "asdf" - file does not exist. Use -lp to list available presets' + str(error.value) + == 'Could not find preset at "nonexistent" - file does not exist. Use -lp to list available presets' ) diff --git a/docs/dev/module_howto.md b/docs/dev/module_howto.md index 0096c8d57..de5bdd90b 100644 --- a/docs/dev/module_howto.md +++ b/docs/dev/module_howto.md @@ -194,10 +194,10 @@ class MyModule(BaseModule): ## Load Modules from Custom Locations -If you have a custom module and you want to use it with BBOT, you can add its parent folder to `module_paths`. This saves you from having to copy it into the BBOT install location. To add a custom module directory, add it to `module_paths` in your preset: +If you have a custom module and you want to use it with BBOT, you can add its parent folder to `module_dirs`. This saves you from having to copy it into the BBOT install location. To add a custom module directory, add it to `module_dirs` in your preset: ```yaml title="my_preset.yml" # load BBOT modules from these additional paths -module_paths: +module_dirs: - /home/user/my_modules ``` diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 0d786cfff..5e90ab117 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -164,7 +164,7 @@ deps: ### ADVANCED OPTIONS ### # Load BBOT modules from these custom paths -module_paths: [] +module_dirs: [] # Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc. speculate: True diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md index f53129161..d0c16c4f1 100644 --- a/docs/scanning/presets.md +++ b/docs/scanning/presets.md @@ -121,6 +121,16 @@ bbot -p ./mypreset.yml --current-preset BBOT Presets support advanced features like environment variable substitution and custom conditions. +### Custom Modules + +If you want to use a custom BBOT `.py` module, you can either move it into `bbot/modules` where BBOT is installed, or add its parent folder to `module_dirs` like so: + +```yaml title="custom_modules.yml" +# load extra BBOT modules from this locaation +module_dirs: + - /home/user/custom_modules +``` + ### Environment Variables You can insert environment variables into your preset like this: `${env:}`: From 803c1143bde4d28f743e10b7d903fdbcea6cf66b Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Fri, 20 Sep 2024 18:02:53 +0100 Subject: [PATCH 097/254] Validate the workspace is in-scope first before downloading --- bbot/modules/postman_download.py | 126 ++++++++++++++---- .../test_module_postman_download.py | 2 +- 2 files changed, 104 insertions(+), 24 deletions(-) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index be8b93aff..0d3df214c 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -45,15 +45,25 @@ async def handle_event(self, event): workspace_id = await self.get_workspace_id(repo_url) if workspace_id: self.verbose(f"Found workspace ID {workspace_id} for {repo_url}") - workspace_path = await self.download_workspace(workspace_id) - if workspace_path: - self.verbose(f"Downloaded workspace from {repo_url} to {workspace_path}") - codebase_event = self.make_event( - {"path": str(workspace_path)}, "FILESYSTEM", tags=["postman", "workspace"], parent=event - ) - await self.emit_event( - codebase_event, - context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}", + data = await self.request_workspace(workspace_id) + workspace = data["workspace"] + environments = data["environments"] + collections = data["collections"] + in_scope = self.validate_workspace(workspace, environments, collections) + if in_scope: + workspace_path = self.save_workspace(workspace, environments, collections) + if workspace_path: + self.verbose(f"Downloaded workspace from {repo_url} to {workspace_path}") + codebase_event = self.make_event( + {"path": str(workspace_path)}, "FILESYSTEM", tags=["postman", "workspace"], parent=event + ) + await self.emit_event( + codebase_event, + context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}", + ) + else: + self.warning( + f"Failed to validate {repo_url} is in our scope as it does not contain any in-scope dns_names / emails, skipping download" ) async def get_workspace_id(self, repo_url): @@ -80,24 +90,18 @@ async def get_workspace_id(self, repo_url): workspace_id = data[0]["id"] return workspace_id - async def download_workspace(self, id): - zip_path = None + async def request_workspace(self, id): + data = {"workspace": {}, "environments": [], "collections": []} workspace = await self.get_workspace(id) if workspace: - # Create a folder for the workspace - name = workspace["name"] - folder = self.output_dir / name - self.helpers.mkdir(folder) - zip_path = folder / f"{id}.zip" - # Main Workspace - self.add_json_to_zip(zip_path, workspace, f"{name}.postman_workspace.json") + name = workspace["name"] + data["workspace"] = workspace # Workspace global variables self.verbose(f"Downloading globals for workspace {name}") globals = await self.get_globals(id) - globals_id = globals["id"] - self.add_json_to_zip(zip_path, globals, f"{globals_id}.postman_environment.json") + data["environments"].append(globals) # Workspace Environments workspace_environments = workspace.get("environments", []) @@ -106,7 +110,7 @@ async def download_workspace(self, id): for _ in workspace_environments: environment_id = _["uid"] environment = await self.get_environment(environment_id) - self.add_json_to_zip(zip_path, environment, f"{environment_id}.postman_environment.json") + data["environments"].append(environment) # Workspace Collections workspace_collections = workspace.get("collections", []) @@ -115,8 +119,8 @@ async def download_workspace(self, id): for _ in workspace_collections: collection_id = _["uid"] collection = await self.get_collection(collection_id) - self.add_json_to_zip(zip_path, collection, f"{collection_id}.postman_collection.json") - return zip_path + data["collections"].append(collection) + return data async def get_workspace(self, workspace_id): workspace = {} @@ -178,6 +182,82 @@ async def get_collection(self, collection_id): collection = json.get("collection", {}) return collection + def validate_string(self, v): + if ( + isinstance(v, str) + and ( + self.helpers.is_dns_name(v, include_local=False) or self.helpers.is_url(v) or self.helpers.is_email(v) + ) + and self.scan.in_scope(v) + ): + return True + return False + + def unpack_and_validate(self, data, workspace_name): + for k, v in data.items(): + if isinstance(v, dict): + if self.unpack_and_validate(v, workspace_name): + self.verbose( + f'Found in-scope key "{k}": "{v}" for workspace {workspace_name}, it appears to be in-scope' + ) + return True + elif isinstance(v, list): + for item in v: + if isinstance(item, dict): + if self.unpack_and_validate(item, workspace_name): + self.verbose( + f'Found in-scope key "{k}": "{item}" for workspace {workspace_name}, it appears to be in-scope' + ) + return True + elif self.validate_string(item): + self.verbose( + f'Found in-scope key "{k}": "{item}" for workspace {workspace_name}, it appears to be in-scope' + ) + return True + elif self.validate_string(v): + self.verbose( + f'Found in-scope key "{k}": "{v}" for workspace {workspace_name}, it appears to be in-scope' + ) + return True + return False + + def validate_workspace(self, workspace, environments, collections): + name = workspace.get("name", "") + if self.unpack_and_validate(workspace, name): + return True + for environment in environments: + if self.unpack_and_validate(environment, name): + return True + for collection in collections: + if self.unpack_and_validate(collection, name): + return True + return False + + def save_workspace(self, workspace, environments, collections): + zip_path = None + # Create a folder for the workspace + name = workspace["name"] + id = workspace["id"] + folder = self.output_dir / name + self.helpers.mkdir(folder) + zip_path = folder / f"{id}.zip" + + # Main Workspace + self.add_json_to_zip(zip_path, workspace, f"{name}.postman_workspace.json") + + # Workspace Environments + if environments: + for environment in environments: + environment_id = environment["id"] + self.add_json_to_zip(zip_path, environment, f"{environment_id}.postman_environment.json") + + # Workspace Collections + if collections: + for collection in collections: + collection_name = collection["info"]["name"] + self.add_json_to_zip(zip_path, collection, f"{collection_name}.postman_collection.json") + return zip_path + def add_json_to_zip(self, zip_path, data, filename): with zipfile.ZipFile(zip_path, "a") as zipf: json_content = json.dumps(data, indent=4) diff --git a/bbot/test/test_step_2/module_tests/test_module_postman_download.py b/bbot/test/test_step_2/module_tests/test_module_postman_download.py index 83b33f9c5..8ac0175d3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman_download.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman_download.py @@ -227,7 +227,7 @@ async def setup_after_prep(self, module_test): }, ) module_test.httpx_mock.add_response( - url="https://api.getpostman.com/collections/10197090-62b91565-d2e2-4bcd-8248-4dba2e3452f0", + url="https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f", json={ "collection": { "info": { From 581ceba88ca79629612fbccdc8f8876f8612321c Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 20 Sep 2024 13:05:15 -0400 Subject: [PATCH 098/254] fix import error --- bbot/core/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index e64064bbd..7fd38a33f 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -478,7 +478,7 @@ def load_module(self, module_name): spec = importlib.util.spec_from_file_location(full_namespace, module_path) module = importlib.util.module_from_spec(spec) - # sys.modules[module_name] = module + sys.modules[full_namespace] = module spec.loader.exec_module(module) # for every top-level variable in the .py file From ef50e272b982b7ecdef6a01e30f97e1f98b510bf Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Fri, 20 Sep 2024 18:07:50 +0100 Subject: [PATCH 099/254] Remove if statement as postman download will validate before downloading anyway --- bbot/modules/postman.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index 82a3c8b28..a7374a571 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -43,19 +43,17 @@ async def handle_org_stub(self, event): self.verbose(f"Searching for any postman workspaces, collections, requests for {org_name}") for item in await self.query(org_name): workspace = item["document"] - human_readable_name = workspace["name"] name = workspace["slug"] profile = workspace["publisherHandle"] - if org_name.lower() in human_readable_name.lower(): - self.verbose(f"Got {name}") - workspace_url = f"{self.html_url}/{profile}/{name}" - await self.emit_event( - {"url": workspace_url}, - "CODE_REPOSITORY", - tags="postman", - parent=event, - context=f'{{module}} searched postman.com for "{org_name}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}', - ) + self.verbose(f"Got {name}") + workspace_url = f"{self.html_url}/{profile}/{name}" + await self.emit_event( + {"url": workspace_url}, + "CODE_REPOSITORY", + tags="postman", + parent=event, + context=f'{{module}} searched postman.com for "{org_name}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}', + ) async def query(self, query): data = [] From 4b79a63266b76a17df47c544a19a70e063e1a6eb Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Fri, 20 Sep 2024 18:24:25 +0100 Subject: [PATCH 100/254] Change logging types in docker_pull --- bbot/modules/docker_pull.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py index 0d1f63c29..65594736b 100644 --- a/bbot/modules/docker_pull.py +++ b/bbot/modules/docker_pull.py @@ -86,12 +86,12 @@ async def docker_api_request(self, url: str): service = www_authenticate_headers.split('service="')[1].split('"')[0] scope = www_authenticate_headers.split('scope="')[1].split('"')[0] except (KeyError, IndexError): - self.log.error(f"Could not obtain realm, service or scope from {url}") + self.log.warning(f"Could not obtain realm, service or scope from {url}") break auth_url = f"{realm}?service={service}&scope={scope}" auth_response = await self.helpers.request(auth_url) if not auth_response: - self.log.error(f"Could not obtain token from {auth_url}") + self.log.warning(f"Could not obtain token from {auth_url}") break auth_json = auth_response.json() token = auth_json["token"] @@ -103,6 +103,7 @@ async def get_tags(self, registry, repository): r = await self.docker_api_request(url) if r is None or r.status_code != 200: self.log.warning(f"Could not retrieve all tags for {repository} asuming tag:latest only.") + self.log.debug(f"Response: {r}") return ["latest"] try: tags = r.json().get("tags", ["latest"]) @@ -115,14 +116,15 @@ async def get_tags(self, registry, repository): else: return [tags[-1]] except (KeyError, IndexError): - self.log.error(f"Could not retrieve tags for {repository}.") + self.log.warning(f"Could not retrieve tags for {repository}.") return ["latest"] async def get_manifest(self, registry, repository, tag): url = f"{registry}/v2/{repository}/manifests/{tag}" r = await self.docker_api_request(url) if r is None or r.status_code != 200: - self.log.error(f"Could not retrieve manifest for {repository}:{tag}.") + self.log.warning(f"Could not retrieve manifest for {repository}:{tag}.") + self.log.debug(f"Response: {r}") return {} response_json = r.json() if response_json.get("manifests", []): From 5644bfade1b5e67c0a8711d879d33ed2674b0d2c Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 20 Sep 2024 14:29:39 -0400 Subject: [PATCH 101/254] fix dnsbrute tests --- .../test_step_2/module_tests/test_module_dnsbrute_mutations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py index 0a7627f25..0c9b6baaa 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py @@ -47,7 +47,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs): ) def check(self, module_test, events): - assert len(events) == 9 + assert len(events) == 10 assert 1 == len( [ e From 058a816d6b7b04d743668f5653ddd22ba0fdc419 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 20 Sep 2024 14:56:08 -0400 Subject: [PATCH 102/254] add yara helper for extracting in-scope dns names --- bbot/modules/internal/excavate.py | 10 ++-------- bbot/scanner/scanner.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 2db7a67cd..f09931dc7 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -741,14 +741,8 @@ class HostnameExtractor(ExcavateRule): def __init__(self, excavate): super().__init__(excavate) - regexes_component_list = [] - if excavate.scan.dns_regexes_yara: - for i, r in enumerate(excavate.scan.dns_regexes_yara): - regexes_component_list.append(rf"$dns_name_{i} = /\b{r.pattern}/ nocase") - regexes_component = " ".join(regexes_component_list) - self.yara_rules[f"hostname_extraction"] = ( - f'rule hostname_extraction {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}' - ) + if excavate.scan.dns_yara_rules_uncompiled: + self.yara_rules[f"hostname_extraction"] = excavate.scan.dns_yara_rules_uncompiled async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier in yara_results.keys(): diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index be25768f1..6033bdfd9 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -1019,6 +1019,25 @@ def dns_regexes_yara(self): self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)+") return self._dns_regexes_yara + @property + def dns_yara_rules_uncompiled(self): + if self._dns_yara_rules_uncompiled is None: + regexes_component_list = [] + for i, r in enumerate(self.dns_regexes_yara): + regexes_component_list.append(rf"$dns_name_{i} = /\b{r.pattern}/ nocase") + if regexes_component_list: + regexes_component = " ".join(regexes_component_list) + self._dns_yara_rules_uncompiled = f'rule hostname_extraction {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}' + return self._dns_yara_rules_uncompiled + + @property + def dns_yara_rules(self): + if self._dns_yara_rules is None: + import yara + + self._dns_yara_rules = yara.compile(self.dns_yara_rules_uncompiled) + return self._dns_yara_rules + @property def json(self): """ From 298ff6247f816d086529bc13f0dcf86a932c07d2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 20 Sep 2024 22:17:57 -0400 Subject: [PATCH 103/254] add helper for extracting in-scope hostnames --- bbot/scanner/scanner.py | 29 +++++++++++++++++++++++---- bbot/test/test_step_1/test_regexes.py | 23 +++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 6033bdfd9..73dd5a3d2 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -234,6 +234,8 @@ def __init__( self._dns_strings = None self._dns_regexes = None self._dns_regexes_yara = None + self._dns_yara_rules_uncompiled = None + self._dns_yara_rules = None self.__log_handlers = None self._log_handler_backup = [] @@ -1030,14 +1032,33 @@ def dns_yara_rules_uncompiled(self): self._dns_yara_rules_uncompiled = f'rule hostname_extraction {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}' return self._dns_yara_rules_uncompiled - @property - def dns_yara_rules(self): + async def dns_yara_rules(self): if self._dns_yara_rules is None: - import yara + if self.dns_yara_rules_uncompiled is not None: + import yara - self._dns_yara_rules = yara.compile(self.dns_yara_rules_uncompiled) + self._dns_yara_rules = await self.helpers.run_in_executor( + yara.compile, source=self.dns_yara_rules_uncompiled + ) return self._dns_yara_rules + async def extract_in_scope_hostnames(self, s): + """ + Given a string, uses yara to extract hostnames that match scan targets + + Examples: + >>> self.scan.extract_in_scope_hostnames("http://www.evilcorp.com") + ... {"www.evilcorp.com"} + """ + matches = set() + dns_yara_rules = await self.dns_yara_rules() + if dns_yara_rules is not None: + for match in await self.helpers.run_in_executor(dns_yara_rules.match, data=s): + for string in match.strings: + for instance in string.instances: + matches.add(str(instance)) + return matches + @property def json(self): """ diff --git a/bbot/test/test_step_1/test_regexes.py b/bbot/test/test_step_1/test_regexes.py index de3837458..26afa42f6 100644 --- a/bbot/test/test_step_1/test_regexes.py +++ b/bbot/test/test_step_1/test_regexes.py @@ -372,3 +372,26 @@ async def test_regex_helper(): assert matches.count(s) == 2 await scan._cleanup() + + # test yara hostname extractor helper + scan = Scanner("evilcorp.com", "www.evilcorp.net", "evilcorp.co.uk") + host_blob = """ + https://asdf.evilcorp.com/ + https://asdf.www.evilcorp.net/ + https://asdf.www.evilcorp.co.uk/ + https://asdf.www.evilcorp.com/ + https://asdf.www.evilcorp.com/ + """ + extracted = await scan.extract_in_scope_hostnames(host_blob) + assert extracted == { + "asdf.www.evilcorp.net", + "asdf.evilcorp.com", + "asdf.www.evilcorp.com", + "www.evilcorp.com", + "asdf.www.evilcorp.co.uk", + "www.evilcorp.co.uk", + } + + scan = Scanner() + extracted = await scan.extract_in_scope_hostnames(host_blob) + assert extracted == set() From 126d3173879c7e574754c40b9ac2debb3008cccf Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 20 Sep 2024 22:49:45 -0400 Subject: [PATCH 104/254] work on tests --- .../test_step_2/module_tests/test_module_json.py | 16 ++++++++-------- .../module_tests/test_module_postman.py | 2 +- .../module_tests/test_module_stdout.py | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index dc5ebd842..27ed5a55e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -22,14 +22,14 @@ def check(self, module_test, events): assert len(scan_json) == 2 assert len(dns_json) == 1 dns_json = dns_json[0] - for scan in scan_json: - assert scan["data"]["name"] == module_test.scan.name - assert scan["data"]["id"] == module_test.scan.id - assert scan["id"] == module_test.scan.id - assert scan["uuid"] == str(module_test.scan.root_event.uuid) - assert scan["parent_uuid"] == str(module_test.scan.root_event.uuid) - assert scan["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] - assert scan["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] + scan = scan_json[0] + assert scan["data"]["name"] == module_test.scan.name + assert scan["data"]["id"] == module_test.scan.id + assert scan["id"] == module_test.scan.id + assert scan["uuid"] == str(module_test.scan.root_event.uuid) + assert scan["parent_uuid"] == str(module_test.scan.root_event.uuid) + assert scan["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] + assert scan["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["id"] == str(dns_event.id) assert dns_json["uuid"] == str(dns_event.uuid) diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py index b6f807a42..1066c5d73 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman.py @@ -90,7 +90,7 @@ async def setup_after_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 4 + assert len(events) == 5 assert 1 == len( [ e diff --git a/bbot/test/test_step_2/module_tests/test_module_stdout.py b/bbot/test/test_step_2/module_tests/test_module_stdout.py index 7abfc1b79..f21845ded 100644 --- a/bbot/test/test_step_2/module_tests/test_module_stdout.py +++ b/bbot/test/test_step_2/module_tests/test_module_stdout.py @@ -17,7 +17,7 @@ class TestStdoutEventTypes(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() - assert len(out.splitlines()) == 2 + assert len(out.splitlines()) == 1 assert out.startswith("[DNS_NAME] \tblacklanternsecurity.com\tTARGET") @@ -79,7 +79,7 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 4 + assert len(lines) == 3 assert out.count("[IP_ADDRESS] \t127.0.0.2") == 2 @@ -97,5 +97,5 @@ class TestStdoutNoDupes(TestStdoutDupes): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 3 + assert len(lines) == 2 assert out.count("[IP_ADDRESS] \t127.0.0.2") == 1 From c58cf71afb15046f84f7576600e3af12bff01289 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 21 Sep 2024 10:35:05 -0400 Subject: [PATCH 105/254] more work on tests --- bbot/test/test_step_2/module_tests/test_module_stdout.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_stdout.py b/bbot/test/test_step_2/module_tests/test_module_stdout.py index f21845ded..27d8a3059 100644 --- a/bbot/test/test_step_2/module_tests/test_module_stdout.py +++ b/bbot/test/test_step_2/module_tests/test_module_stdout.py @@ -46,8 +46,10 @@ def check(self, module_test, events): event = json.loads(line) if i == 0: assert event["type"] == "SCAN" - elif i == 2: + elif i == 1: assert event["type"] == "DNS_NAME" and event["data"] == "blacklanternsecurity.com" + if i == 2: + assert event["type"] == "SCAN" class TestStdoutJSONFields(TestStdout): From 8e547905aa7bee39377341260ad46104e669e856 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 21 Sep 2024 16:00:30 -0400 Subject: [PATCH 106/254] better debugging for https://github.com/blacklanternsecurity/bbot/issues/1779 --- bbot/core/helpers/misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 13662792a..0c4e26ca9 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -227,13 +227,13 @@ def split_host_port(d): match = bbot_regexes.extract_open_port_regex.match(netloc) if match is None: - raise ValueError(f'split_port() failed to parse netloc "{netloc}"') + raise ValueError(f'split_port() failed to parse netloc "{netloc}" (original value: {d})') host = match.group(2) if host is None: host = match.group(1) if host is None: - raise ValueError(f'split_port() failed to locate host in netloc "{netloc}"') + raise ValueError(f'split_port() failed to locate host in netloc "{netloc}" (original value: {d})') port = match.group(3) if port is None and scheme is not None: From 0a85f351fe390c679e5a5dd45bded9c5ecf40720 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 21 Sep 2024 16:08:11 -0400 Subject: [PATCH 107/254] bump version --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 743b1ec7f..ce82bbbe6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bbot" -version = "2.0.1" +version = "2.1.0" description = "OSINT automation for hackers." authors = [ "TheTechromancer", @@ -98,7 +98,7 @@ extend-exclude = "(test_step_1/test_manager_*)" [tool.poetry-dynamic-versioning] enable = true metadata = false -format-jinja = 'v2.0.1{% if branch == "dev" %}.{{ distance }}rc{% endif %}' +format-jinja = 'v2.1.0{% if branch == "dev" %}.{{ distance }}rc{% endif %}' [tool.poetry-dynamic-versioning.substitution] files = ["*/__init__.py"] From 83f47dc11ebacb02fdbe33c5f7a3bd515a3aaa7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:54:39 +0000 Subject: [PATCH 108/254] Bump mkdocs-material from 9.5.34 to 9.5.36 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.34 to 9.5.36. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.34...9.5.36) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 696492ec1..827dc7184 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1243,13 +1243,13 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.34" +version = "9.5.36" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.34-py3-none-any.whl", hash = "sha256:54caa8be708de2b75167fd4d3b9f3d949579294f49cb242515d4653dbee9227e"}, - {file = "mkdocs_material-9.5.34.tar.gz", hash = "sha256:1e60ddf716cfb5679dfd65900b8a25d277064ed82d9a53cd5190e3f894df7840"}, + {file = "mkdocs_material-9.5.36-py3-none-any.whl", hash = "sha256:36734c1fd9404bea74236242ba3359b267fc930c7233b9fd086b0898825d0ac9"}, + {file = "mkdocs_material-9.5.36.tar.gz", hash = "sha256:140456f761320f72b399effc073fa3f8aac744c77b0970797c201cae2f6c967f"}, ] [package.dependencies] From 57d0d0d9571a814c8cbc95fe9ff12c20da9515c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:55:00 +0000 Subject: [PATCH 109/254] Bump mmh3 from 4.1.0 to 5.0.1 Bumps [mmh3](https://github.com/hajimes/mmh3) from 4.1.0 to 5.0.1. - [Release notes](https://github.com/hajimes/mmh3/releases) - [Changelog](https://github.com/hajimes/mmh3/blob/master/CHANGELOG.md) - [Commits](https://github.com/hajimes/mmh3/compare/v4.1.0...v5.0.1) --- updated-dependencies: - dependency-name: mmh3 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- poetry.lock | 189 +++++++++++++++++++++++++++---------------------- pyproject.toml | 2 +- 2 files changed, 106 insertions(+), 85 deletions(-) diff --git a/poetry.lock b/poetry.lock index 696492ec1..e178b827f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1327,95 +1327,116 @@ mkdocstrings = ">=0.26" [[package]] name = "mmh3" -version = "4.1.0" +version = "5.0.1" description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be5ac76a8b0cd8095784e51e4c1c9c318c19edcd1709a06eb14979c8d850c31a"}, - {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98a49121afdfab67cd80e912b36404139d7deceb6773a83620137aaa0da5714c"}, - {file = "mmh3-4.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5259ac0535874366e7d1a5423ef746e0d36a9e3c14509ce6511614bdc5a7ef5b"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5950827ca0453a2be357696da509ab39646044e3fa15cad364eb65d78797437"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1dd0f652ae99585b9dd26de458e5f08571522f0402155809fd1dc8852a613a39"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d25548070942fab1e4a6f04d1626d67e66d0b81ed6571ecfca511f3edf07e6"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53db8d9bad3cb66c8f35cbc894f336273f63489ce4ac416634932e3cbe79eb5b"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75da0f615eb55295a437264cc0b736753f830b09d102aa4c2a7d719bc445ec05"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b926b07fd678ea84b3a2afc1fa22ce50aeb627839c44382f3d0291e945621e1a"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c5b053334f9b0af8559d6da9dc72cef0a65b325ebb3e630c680012323c950bb6"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bf33dc43cd6de2cb86e0aa73a1cc6530f557854bbbe5d59f41ef6de2e353d7b"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fa7eacd2b830727ba3dd65a365bed8a5c992ecd0c8348cf39a05cc77d22f4970"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:42dfd6742b9e3eec599f85270617debfa0bbb913c545bb980c8a4fa7b2d047da"}, - {file = "mmh3-4.1.0-cp310-cp310-win32.whl", hash = "sha256:2974ad343f0d39dcc88e93ee6afa96cedc35a9883bc067febd7ff736e207fa47"}, - {file = "mmh3-4.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:74699a8984ded645c1a24d6078351a056f5a5f1fe5838870412a68ac5e28d865"}, - {file = "mmh3-4.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:f0dc874cedc23d46fc488a987faa6ad08ffa79e44fb08e3cd4d4cf2877c00a00"}, - {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3280a463855b0eae64b681cd5b9ddd9464b73f81151e87bb7c91a811d25619e6"}, - {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:97ac57c6c3301769e757d444fa7c973ceb002cb66534b39cbab5e38de61cd896"}, - {file = "mmh3-4.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7b6502cdb4dbd880244818ab363c8770a48cdccecf6d729ade0241b736b5ec0"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52ba2da04671a9621580ddabf72f06f0e72c1c9c3b7b608849b58b11080d8f14"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a5fef4c4ecc782e6e43fbeab09cff1bac82c998a1773d3a5ee6a3605cde343e"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5135358a7e00991f73b88cdc8eda5203bf9de22120d10a834c5761dbeb07dd13"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cff9ae76a54f7c6fe0167c9c4028c12c1f6de52d68a31d11b6790bb2ae685560"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f02576a4d106d7830ca90278868bf0983554dd69183b7bbe09f2fcd51cf54f"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:073d57425a23721730d3ff5485e2da489dd3c90b04e86243dd7211f889898106"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:71e32ddec7f573a1a0feb8d2cf2af474c50ec21e7a8263026e8d3b4b629805db"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7cbb20b29d57e76a58b40fd8b13a9130db495a12d678d651b459bf61c0714cea"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a42ad267e131d7847076bb7e31050f6c4378cd38e8f1bf7a0edd32f30224d5c9"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a013979fc9390abadc445ea2527426a0e7a4495c19b74589204f9b71bcaafeb"}, - {file = "mmh3-4.1.0-cp311-cp311-win32.whl", hash = "sha256:1d3b1cdad7c71b7b88966301789a478af142bddcb3a2bee563f7a7d40519a00f"}, - {file = "mmh3-4.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0dc6dc32eb03727467da8e17deffe004fbb65e8b5ee2b502d36250d7a3f4e2ec"}, - {file = "mmh3-4.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:9ae3a5c1b32dda121c7dc26f9597ef7b01b4c56a98319a7fe86c35b8bc459ae6"}, - {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0033d60c7939168ef65ddc396611077a7268bde024f2c23bdc283a19123f9e9c"}, - {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d6af3e2287644b2b08b5924ed3a88c97b87b44ad08e79ca9f93d3470a54a41c5"}, - {file = "mmh3-4.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d82eb4defa245e02bb0b0dc4f1e7ee284f8d212633389c91f7fba99ba993f0a2"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba245e94b8d54765e14c2d7b6214e832557e7856d5183bc522e17884cab2f45d"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb04e2feeabaad6231e89cd43b3d01a4403579aa792c9ab6fdeef45cc58d4ec0"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e3b1a27def545ce11e36158ba5d5390cdbc300cfe456a942cc89d649cf7e3b2"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce0ab79ff736d7044e5e9b3bfe73958a55f79a4ae672e6213e92492ad5e734d5"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b02268be6e0a8eeb8a924d7db85f28e47344f35c438c1e149878bb1c47b1cd3"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:deb887f5fcdaf57cf646b1e062d56b06ef2f23421c80885fce18b37143cba828"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99dd564e9e2b512eb117bd0cbf0f79a50c45d961c2a02402787d581cec5448d5"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:08373082dfaa38fe97aa78753d1efd21a1969e51079056ff552e687764eafdfe"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:54b9c6a2ea571b714e4fe28d3e4e2db37abfd03c787a58074ea21ee9a8fd1740"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a7b1edf24c69e3513f879722b97ca85e52f9032f24a52284746877f6a7304086"}, - {file = "mmh3-4.1.0-cp312-cp312-win32.whl", hash = "sha256:411da64b951f635e1e2284b71d81a5a83580cea24994b328f8910d40bed67276"}, - {file = "mmh3-4.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bebc3ecb6ba18292e3d40c8712482b4477abd6981c2ebf0e60869bd90f8ac3a9"}, - {file = "mmh3-4.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:168473dd608ade6a8d2ba069600b35199a9af837d96177d3088ca91f2b3798e3"}, - {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:372f4b7e1dcde175507640679a2a8790185bb71f3640fc28a4690f73da986a3b"}, - {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:438584b97f6fe13e944faf590c90fc127682b57ae969f73334040d9fa1c7ffa5"}, - {file = "mmh3-4.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6e27931b232fc676675fac8641c6ec6b596daa64d82170e8597f5a5b8bdcd3b6"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:571a92bad859d7b0330e47cfd1850b76c39b615a8d8e7aa5853c1f971fd0c4b1"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a69d6afe3190fa08f9e3a58e5145549f71f1f3fff27bd0800313426929c7068"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afb127be0be946b7630220908dbea0cee0d9d3c583fa9114a07156f98566dc28"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940d86522f36348ef1a494cbf7248ab3f4a1638b84b59e6c9e90408bd11ad729"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3dcccc4935686619a8e3d1f7b6e97e3bd89a4a796247930ee97d35ea1a39341"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01bb9b90d61854dfc2407c5e5192bfb47222d74f29d140cb2dd2a69f2353f7cc"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bcb1b8b951a2c0b0fb8a5426c62a22557e2ffc52539e0a7cc46eb667b5d606a9"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6477a05d5e5ab3168e82e8b106e316210ac954134f46ec529356607900aea82a"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:da5892287e5bea6977364b15712a2573c16d134bc5fdcdd4cf460006cf849278"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:99180d7fd2327a6fffbaff270f760576839dc6ee66d045fa3a450f3490fda7f5"}, - {file = "mmh3-4.1.0-cp38-cp38-win32.whl", hash = "sha256:9b0d4f3949913a9f9a8fb1bb4cc6ecd52879730aab5ff8c5a3d8f5b593594b73"}, - {file = "mmh3-4.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:598c352da1d945108aee0c3c3cfdd0e9b3edef74108f53b49d481d3990402169"}, - {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:475d6d1445dd080f18f0f766277e1237fa2914e5fe3307a3b2a3044f30892103"}, - {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5ca07c41e6a2880991431ac717c2a049056fff497651a76e26fc22224e8b5732"}, - {file = "mmh3-4.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ebe052fef4bbe30c0548d12ee46d09f1b69035ca5208a7075e55adfe091be44"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaefd42e85afb70f2b855a011f7b4d8a3c7e19c3f2681fa13118e4d8627378c5"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0ae43caae5a47afe1b63a1ae3f0986dde54b5fb2d6c29786adbfb8edc9edfb"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6218666f74c8c013c221e7f5f8a693ac9cf68e5ac9a03f2373b32d77c48904de"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac59294a536ba447b5037f62d8367d7d93b696f80671c2c45645fa9f1109413c"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:086844830fcd1e5c84fec7017ea1ee8491487cfc877847d96f86f68881569d2e"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e42b38fad664f56f77f6fbca22d08450f2464baa68acdbf24841bf900eb98e87"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d08b790a63a9a1cde3b5d7d733ed97d4eb884bfbc92f075a091652d6bfd7709a"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:73ea4cc55e8aea28c86799ecacebca09e5f86500414870a8abaedfcbaf74d288"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f90938ff137130e47bcec8dc1f4ceb02f10178c766e2ef58a9f657ff1f62d124"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:aa1f13e94b8631c8cd53259250556edcf1de71738936b60febba95750d9632bd"}, - {file = "mmh3-4.1.0-cp39-cp39-win32.whl", hash = "sha256:a3b680b471c181490cf82da2142029edb4298e1bdfcb67c76922dedef789868d"}, - {file = "mmh3-4.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:fefef92e9c544a8dbc08f77a8d1b6d48006a750c4375bbcd5ff8199d761e263b"}, - {file = "mmh3-4.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:8e2c1f6a2b41723a4f82bd5a762a777836d29d664fc0095f17910bea0adfd4a6"}, - {file = "mmh3-4.1.0.tar.gz", hash = "sha256:a1cf25348b9acd229dda464a094d6170f47d2850a1fcb762a3b6172d2ce6ca4a"}, + {file = "mmh3-5.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f0a4b4bf05778ed77d820d6e7d0e9bd6beb0c01af10e1ce9233f5d2f814fcafa"}, + {file = "mmh3-5.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac7a391039aeab95810c2d020b69a94eb6b4b37d4e2374831e92db3a0cdf71c6"}, + {file = "mmh3-5.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3a2583b5521ca49756d8d8bceba80627a9cc295f255dcab4e3df7ccc2f09679a"}, + {file = "mmh3-5.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:081a8423fe53c1ac94f87165f3e4c500125d343410c1a0c5f1703e898a3ef038"}, + {file = "mmh3-5.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8b4d72713799755dc8954a7d36d5c20a6c8de7b233c82404d122c7c7c1707cc"}, + {file = "mmh3-5.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:389a6fd51efc76d3182d36ec306448559c1244f11227d2bb771bdd0e6cc91321"}, + {file = "mmh3-5.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39f4128edaa074bff721b1d31a72508cba4d2887ee7867f22082e1fe9d4edea0"}, + {file = "mmh3-5.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d5d23a94d91aabba3386b3769048d5f4210fdfef80393fece2f34ba5a7b466c"}, + {file = "mmh3-5.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:16347d038361f8b8f24fd2b7ef378c9b68ddee9f7706e46269b6e0d322814713"}, + {file = "mmh3-5.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6e299408565af7d61f2d20a5ffdd77cf2ed902460fe4e6726839d59ba4b72316"}, + {file = "mmh3-5.0.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42050af21ddfc5445ee5a66e73a8fc758c71790305e3ee9e4a85a8e69e810f94"}, + {file = "mmh3-5.0.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2ae9b1f5ef27ec54659920f0404b7ceb39966e28867c461bfe83a05e8d18ddb0"}, + {file = "mmh3-5.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:50c2495a02045f3047d71d4ae9cdd7a15efc0bcbb7ff17a18346834a8e2d1d19"}, + {file = "mmh3-5.0.1-cp310-cp310-win32.whl", hash = "sha256:c028fa77cddf351ca13b4a56d43c1775652cde0764cadb39120b68f02a23ecf6"}, + {file = "mmh3-5.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:c5e741e421ec14400c4aae30890515c201f518403bdef29ae1e00d375bb4bbb5"}, + {file = "mmh3-5.0.1-cp310-cp310-win_arm64.whl", hash = "sha256:b17156d56fabc73dbf41bca677ceb6faed435cc8544f6566d72ea77d8a17e9d0"}, + {file = "mmh3-5.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a6d5a9b1b923f1643559ba1fc0bf7a5076c90cbb558878d3bf3641ce458f25d"}, + {file = "mmh3-5.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3349b968be555f7334bbcce839da98f50e1e80b1c615d8e2aa847ea4a964a012"}, + {file = "mmh3-5.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1bd3c94b110e55db02ab9b605029f48a2f7f677c6e58c09d44e42402d438b7e1"}, + {file = "mmh3-5.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d47ba84d48608f79adbb10bb09986b6dc33eeda5c2d1bd75d00820081b73bde9"}, + {file = "mmh3-5.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c0217987a8b8525c8d9170f66d036dec4ab45cfbd53d47e8d76125791ceb155e"}, + {file = "mmh3-5.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2797063a34e78d1b61639a98b0edec1c856fa86ab80c7ec859f1796d10ba429"}, + {file = "mmh3-5.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8bba16340adcbd47853a2fbe5afdb397549e8f2e79324ff1dced69a3f8afe7c3"}, + {file = "mmh3-5.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:282797957c9f60b51b9d768a602c25f579420cc9af46feb77d457a27823d270a"}, + {file = "mmh3-5.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e4fb670c29e63f954f9e7a2cdcd57b36a854c2538f579ef62681ccbaa1de2b69"}, + {file = "mmh3-5.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ee7d85438dc6aff328e19ab052086a3c29e8a9b632998a49e5c4b0034e9e8d6"}, + {file = "mmh3-5.0.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b7fb5db231f3092444bc13901e6a8d299667126b00636ffbad4a7b45e1051e2f"}, + {file = "mmh3-5.0.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c100dd441703da5ec136b1d9003ed4a041d8a1136234c9acd887499796df6ad8"}, + {file = "mmh3-5.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:71f3b765138260fd7a7a2dba0ea5727dabcd18c1f80323c9cfef97a7e86e01d0"}, + {file = "mmh3-5.0.1-cp311-cp311-win32.whl", hash = "sha256:9a76518336247fd17689ce3ae5b16883fd86a490947d46a0193d47fb913e26e3"}, + {file = "mmh3-5.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:336bc4df2e44271f1c302d289cc3d78bd52d3eed8d306c7e4bff8361a12bf148"}, + {file = "mmh3-5.0.1-cp311-cp311-win_arm64.whl", hash = "sha256:af6522722fbbc5999aa66f7244d0986767a46f1fb05accc5200f75b72428a508"}, + {file = "mmh3-5.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f2730bb263ed9c388e8860438b057a53e3cc701134a6ea140f90443c4c11aa40"}, + {file = "mmh3-5.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6246927bc293f6d56724536400b85fb85f5be26101fa77d5f97dd5e2a4c69bf2"}, + {file = "mmh3-5.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fbca322519a6e6e25b6abf43e940e1667cf8ea12510e07fb4919b48a0cd1c411"}, + {file = "mmh3-5.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eae8c19903ed8a1724ad9e67e86f15d198a7a1271a4f9be83d47e38f312ed672"}, + {file = "mmh3-5.0.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a09fd6cc72c07c0c07c3357714234b646d78052487c4a3bd5f7f6e08408cff60"}, + {file = "mmh3-5.0.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ff8551fee7ae3b11c5d986b6347ade0dccaadd4670ffdb2b944dee120ffcc84"}, + {file = "mmh3-5.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e39694c73a5a20c8bf36dfd8676ed351e5234d55751ba4f7562d85449b21ef3f"}, + {file = "mmh3-5.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eba6001989a92f72a89c7cf382fda831678bd780707a66b4f8ca90239fdf2123"}, + {file = "mmh3-5.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0771f90c9911811cc606a5c7b7b58f33501c9ee896ed68a6ac22c7d55878ecc0"}, + {file = "mmh3-5.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:09b31ed0c0c0920363e96641fac4efde65b1ab62b8df86293142f35a254e72b4"}, + {file = "mmh3-5.0.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5cf4a8deda0235312db12075331cb417c4ba163770edfe789bde71d08a24b692"}, + {file = "mmh3-5.0.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:41f7090a95185ef20ac018581a99337f0cbc84a2135171ee3290a9c0d9519585"}, + {file = "mmh3-5.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b97b5b368fb7ff22194ec5854f5b12d8de9ab67a0f304728c7f16e5d12135b76"}, + {file = "mmh3-5.0.1-cp312-cp312-win32.whl", hash = "sha256:842516acf04da546f94fad52db125ee619ccbdcada179da51c326a22c4578cb9"}, + {file = "mmh3-5.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:d963be0dbfd9fca209c17172f6110787ebf78934af25e3694fe2ba40e55c1e2b"}, + {file = "mmh3-5.0.1-cp312-cp312-win_arm64.whl", hash = "sha256:a5da292ceeed8ce8e32b68847261a462d30fd7b478c3f55daae841404f433c15"}, + {file = "mmh3-5.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:673e3f1c8d4231d6fb0271484ee34cb7146a6499fc0df80788adb56fd76842da"}, + {file = "mmh3-5.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f795a306bd16a52ad578b663462cc8e95500b3925d64118ae63453485d67282b"}, + {file = "mmh3-5.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5ed57a5e28e502a1d60436cc25c76c3a5ba57545f250f2969af231dc1221e0a5"}, + {file = "mmh3-5.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:632c28e7612e909dbb6cbe2fe496201ada4695b7715584005689c5dc038e59ad"}, + {file = "mmh3-5.0.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53fd6bd525a5985e391c43384672d9d6b317fcb36726447347c7fc75bfed34ec"}, + {file = "mmh3-5.0.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dceacf6b0b961a0e499836af3aa62d60633265607aef551b2a3e3c48cdaa5edd"}, + {file = "mmh3-5.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f0738d478fdfb5d920f6aff5452c78f2c35b0eff72caa2a97dfe38e82f93da2"}, + {file = "mmh3-5.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e70285e7391ab88b872e5bef632bad16b9d99a6d3ca0590656a4753d55988af"}, + {file = "mmh3-5.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:27e5fc6360aa6b828546a4318da1a7da6bf6e5474ccb053c3a6aa8ef19ff97bd"}, + {file = "mmh3-5.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7989530c3c1e2c17bf5a0ec2bba09fd19819078ba90beedabb1c3885f5040b0d"}, + {file = "mmh3-5.0.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:cdad7bee649950da7ecd3cbbbd12fb81f1161072ecbdb5acfa0018338c5cb9cf"}, + {file = "mmh3-5.0.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:e143b8f184c1bb58cecd85ab4a4fd6dc65a2d71aee74157392c3fddac2a4a331"}, + {file = "mmh3-5.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e5eb12e886f3646dd636f16b76eb23fc0c27e8ff3c1ae73d4391e50ef60b40f6"}, + {file = "mmh3-5.0.1-cp313-cp313-win32.whl", hash = "sha256:16e6dddfa98e1c2d021268e72c78951234186deb4df6630e984ac82df63d0a5d"}, + {file = "mmh3-5.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:d3ffb792d70b8c4a2382af3598dad6ae0c5bd9cee5b7ffcc99aa2f5fd2c1bf70"}, + {file = "mmh3-5.0.1-cp313-cp313-win_arm64.whl", hash = "sha256:122fa9ec148383f9124292962bda745f192b47bfd470b2af5fe7bb3982b17896"}, + {file = "mmh3-5.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b12bad8c75e6ff5d67319794fb6a5e8c713826c818d47f850ad08b4aa06960c6"}, + {file = "mmh3-5.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e5bbb066538c1048d542246fc347bb7994bdda29a3aea61c22f9f8b57111ce69"}, + {file = "mmh3-5.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:eee6134273f64e2a106827cc8fd77e70cc7239a285006fc6ab4977d59b015af2"}, + {file = "mmh3-5.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d04d9aa19d48e4c7bbec9cabc2c4dccc6ff3b2402f856d5bf0de03e10f167b5b"}, + {file = "mmh3-5.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79f37da1eed034d06567a69a7988456345c7f29e49192831c3975b464493b16e"}, + {file = "mmh3-5.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:242f77666743337aa828a2bf2da71b6ba79623ee7f93edb11e009f69237c8561"}, + {file = "mmh3-5.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffd943fff690463945f6441a2465555b3146deaadf6a5e88f2590d14c655d71b"}, + {file = "mmh3-5.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565b15f8d7df43acb791ff5a360795c20bfa68bca8b352509e0fbabd06cc48cd"}, + {file = "mmh3-5.0.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:fc6aafb867c2030df98ac7760ff76b500359252867985f357bd387739f3d5287"}, + {file = "mmh3-5.0.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:32898170644d45aa27c974ab0d067809c066205110f5c6d09f47d9ece6978bfe"}, + {file = "mmh3-5.0.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:42865567838d2193eb64e0ef571f678bf361a254fcdef0c5c8e73243217829bd"}, + {file = "mmh3-5.0.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:5ff5c1f301c4a8b6916498969c0fcc7e3dbc56b4bfce5cfe3fe31f3f4609e5ae"}, + {file = "mmh3-5.0.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:be74c2dda8a6f44a504450aa2c3507f8067a159201586fc01dd41ab80efc350f"}, + {file = "mmh3-5.0.1-cp38-cp38-win32.whl", hash = "sha256:5610a842621ff76c04b20b29cf5f809b131f241a19d4937971ba77dc99a7f330"}, + {file = "mmh3-5.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:de15739ac50776fe8aa1ef13f1be46a6ee1fbd45f6d0651084097eb2be0a5aa4"}, + {file = "mmh3-5.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:48e84cf3cc7e8c41bc07de72299a73b92d9e3cde51d97851420055b1484995f7"}, + {file = "mmh3-5.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6dd9dc28c2d168c49928195c2e29b96f9582a5d07bd690a28aede4cc07b0e696"}, + {file = "mmh3-5.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2771a1c56a3d4bdad990309cff5d0a8051f29c8ec752d001f97d6392194ae880"}, + {file = "mmh3-5.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5ff2a8322ba40951a84411550352fba1073ce1c1d1213bb7530f09aed7f8caf"}, + {file = "mmh3-5.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a16bd3ec90682c9e0a343e6bd4c778c09947c8c5395cdb9e5d9b82b2559efbca"}, + {file = "mmh3-5.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d45733a78d68b5b05ff4a823aea51fa664df1d3bf4929b152ff4fd6dea2dd69b"}, + {file = "mmh3-5.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:904285e83cedebc8873b0838ed54c20f7344120be26e2ca5a907ab007a18a7a0"}, + {file = "mmh3-5.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac4aeb1784e43df728034d0ed72e4b2648db1a69fef48fa58e810e13230ae5ff"}, + {file = "mmh3-5.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:cb3d4f751a0b8b4c8d06ef1c085216c8fddcc8b8c8d72445976b5167a40c6d1e"}, + {file = "mmh3-5.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8021851935600e60c42122ed1176399d7692df338d606195cd599d228a04c1c6"}, + {file = "mmh3-5.0.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6182d5924a5efc451900f864cbb021d7e8ad5d524816ca17304a0f663bc09bb5"}, + {file = "mmh3-5.0.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:5f30b834552a4f79c92e3d266336fb87fd92ce1d36dc6813d3e151035890abbd"}, + {file = "mmh3-5.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cd4383f35e915e06d077df27e04ffd3be7513ec6a9de2d31f430393f67e192a7"}, + {file = "mmh3-5.0.1-cp39-cp39-win32.whl", hash = "sha256:1455fb6b42665a97db8fc66e89a861e52b567bce27ed054c47877183f86ea6e3"}, + {file = "mmh3-5.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:9e26a0f4eb9855a143f5938a53592fa14c2d3b25801c2106886ab6c173982780"}, + {file = "mmh3-5.0.1-cp39-cp39-win_arm64.whl", hash = "sha256:0d0a35a69abdad7549c4030a714bb4ad07902edb3bbe61e1bbc403ded5d678be"}, + {file = "mmh3-5.0.1.tar.gz", hash = "sha256:7dab080061aeb31a6069a181f27c473a1f67933854e36a3464931f2716508896"}, ] [package.extras] -test = ["mypy (>=1.0)", "pytest (>=7.0.0)"] +benchmark = ["pymmh3 (==0.0.5)", "pyperf (==2.7.0)", "xxhash (==3.5.0)"] +docs = ["myst-parser (==4.0.0)", "shibuya (==2024.8.30)", "sphinx (==8.0.2)", "sphinx-copybutton (==0.5.2)"] +lint = ["black (==24.8.0)", "clang-format (==18.1.8)", "isort (==5.13.2)", "pylint (==3.2.7)"] +plot = ["matplotlib (==3.9.2)", "pandas (==2.2.2)"] +test = ["pytest (==8.3.3)", "pytest-sugar (==1.0.0)"] +type = ["mypy (==1.11.2)"] [[package]] name = "mypy-extensions" @@ -3007,4 +3028,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "37d8f059d576c870383faa9e0cfc432e7d147213c6632a083487d8e910894ec9" +content-hash = "fb2f904f634456d6e72915cdf3040249aa2dc498106b1879732e0276df7ea082" diff --git a/pyproject.toml b/pyproject.toml index 743b1ec7f..0f98dea23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ regex = "^2024.4.16" unidecode = "^1.3.8" radixtarget = "^1.0.0.15" cloudcheck = "^5.0.0.350" -mmh3 = "^4.1.0" +mmh3 = ">=4.1,<6.0" setproctitle = "^1.3.3" yara-python = "^4.5.1" pyzmq = "^26.0.3" From d36b2afa90b42b570998def1f071ea9c8d42dcd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:55:39 +0000 Subject: [PATCH 110/254] Bump pytest-env from 1.1.4 to 1.1.5 Bumps [pytest-env](https://github.com/pytest-dev/pytest-env) from 1.1.4 to 1.1.5. - [Release notes](https://github.com/pytest-dev/pytest-env/releases) - [Commits](https://github.com/pytest-dev/pytest-env/compare/1.1.4...1.1.5) --- updated-dependencies: - dependency-name: pytest-env dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 696492ec1..675e29343 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1942,21 +1942,21 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] [[package]] name = "pytest-env" -version = "1.1.4" +version = "1.1.5" description = "pytest plugin that allows you to add environment variables." optional = false python-versions = ">=3.8" files = [ - {file = "pytest_env-1.1.4-py3-none-any.whl", hash = "sha256:a4212056d4d440febef311a98fdca56c31256d58fb453d103cba4e8a532b721d"}, - {file = "pytest_env-1.1.4.tar.gz", hash = "sha256:86653658da8f11c6844975db955746c458a9c09f1e64957603161e2ff93f5133"}, + {file = "pytest_env-1.1.5-py3-none-any.whl", hash = "sha256:ce90cf8772878515c24b31cd97c7fa1f4481cd68d588419fd45f10ecaee6bc30"}, + {file = "pytest_env-1.1.5.tar.gz", hash = "sha256:91209840aa0e43385073ac464a554ad2947cc2fd663a9debf88d03b01e0cc1cf"}, ] [package.dependencies] -pytest = ">=8.3.2" +pytest = ">=8.3.3" tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} [package.extras] -test = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "pytest-mock (>=3.14)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "pytest-mock (>=3.14)"] [[package]] name = "pytest-httpserver" From 86417cd31c3d0edf0f6552bdf443740aa532de59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:56:03 +0000 Subject: [PATCH 111/254] Bump pydantic from 2.9.1 to 2.9.2 Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.9.1 to 2.9.2. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.9.1...v2.9.2) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 188 ++++++++++++++++++++++++++-------------------------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/poetry.lock b/poetry.lock index 696492ec1..6400d7b96 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1686,18 +1686,18 @@ files = [ [[package]] name = "pydantic" -version = "2.9.1" +version = "2.9.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.9.1-py3-none-any.whl", hash = "sha256:7aff4db5fdf3cf573d4b3c30926a510a10e19a0774d38fc4967f78beb6deb612"}, - {file = "pydantic-2.9.1.tar.gz", hash = "sha256:1363c7d975c7036df0db2b4a61f2e062fbc0aa5ab5f2772e0ffc7191a4f4bce2"}, + {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, + {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.23.3" +pydantic-core = "2.23.4" typing-extensions = [ {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, {version = ">=4.6.1", markers = "python_version < \"3.13\""}, @@ -1709,100 +1709,100 @@ timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.23.3" +version = "2.23.4" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.23.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7f10a5d1b9281392f1bf507d16ac720e78285dfd635b05737c3911637601bae6"}, - {file = "pydantic_core-2.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c09a7885dd33ee8c65266e5aa7fb7e2f23d49d8043f089989726391dd7350c5"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6470b5a1ec4d1c2e9afe928c6cb37eb33381cab99292a708b8cb9aa89e62429b"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9172d2088e27d9a185ea0a6c8cebe227a9139fd90295221d7d495944d2367700"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86fc6c762ca7ac8fbbdff80d61b2c59fb6b7d144aa46e2d54d9e1b7b0e780e01"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0cb80fd5c2df4898693aa841425ea1727b1b6d2167448253077d2a49003e0ed"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03667cec5daf43ac4995cefa8aaf58f99de036204a37b889c24a80927b629cec"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:047531242f8e9c2db733599f1c612925de095e93c9cc0e599e96cf536aaf56ba"}, - {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5499798317fff7f25dbef9347f4451b91ac2a4330c6669821c8202fd354c7bee"}, - {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bbb5e45eab7624440516ee3722a3044b83fff4c0372efe183fd6ba678ff681fe"}, - {file = "pydantic_core-2.23.3-cp310-none-win32.whl", hash = "sha256:8b5b3ed73abb147704a6e9f556d8c5cb078f8c095be4588e669d315e0d11893b"}, - {file = "pydantic_core-2.23.3-cp310-none-win_amd64.whl", hash = "sha256:2b603cde285322758a0279995b5796d64b63060bfbe214b50a3ca23b5cee3e83"}, - {file = "pydantic_core-2.23.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c889fd87e1f1bbeb877c2ee56b63bb297de4636661cc9bbfcf4b34e5e925bc27"}, - {file = "pydantic_core-2.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea85bda3189fb27503af4c45273735bcde3dd31c1ab17d11f37b04877859ef45"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7f7f72f721223f33d3dc98a791666ebc6a91fa023ce63733709f4894a7dc611"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b2b55b0448e9da68f56b696f313949cda1039e8ec7b5d294285335b53104b61"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c24574c7e92e2c56379706b9a3f07c1e0c7f2f87a41b6ee86653100c4ce343e5"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2b05e6ccbee333a8f4b8f4d7c244fdb7a979e90977ad9c51ea31261e2085ce0"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c409ce1c219c091e47cb03feb3c4ed8c2b8e004efc940da0166aaee8f9d6c8"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d965e8b325f443ed3196db890d85dfebbb09f7384486a77461347f4adb1fa7f8"}, - {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f56af3a420fb1ffaf43ece3ea09c2d27c444e7c40dcb7c6e7cf57aae764f2b48"}, - {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5b01a078dd4f9a52494370af21aa52964e0a96d4862ac64ff7cea06e0f12d2c5"}, - {file = "pydantic_core-2.23.3-cp311-none-win32.whl", hash = "sha256:560e32f0df04ac69b3dd818f71339983f6d1f70eb99d4d1f8e9705fb6c34a5c1"}, - {file = "pydantic_core-2.23.3-cp311-none-win_amd64.whl", hash = "sha256:c744fa100fdea0d000d8bcddee95213d2de2e95b9c12be083370b2072333a0fa"}, - {file = "pydantic_core-2.23.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e0ec50663feedf64d21bad0809f5857bac1ce91deded203efc4a84b31b2e4305"}, - {file = "pydantic_core-2.23.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db6e6afcb95edbe6b357786684b71008499836e91f2a4a1e55b840955b341dbb"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98ccd69edcf49f0875d86942f4418a4e83eb3047f20eb897bffa62a5d419c8fa"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a678c1ac5c5ec5685af0133262103defb427114e62eafeda12f1357a12140162"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01491d8b4d8db9f3391d93b0df60701e644ff0894352947f31fff3e52bd5c801"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fcf31facf2796a2d3b7fe338fe8640aa0166e4e55b4cb108dbfd1058049bf4cb"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7200fd561fb3be06827340da066df4311d0b6b8eb0c2116a110be5245dceb326"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc1636770a809dee2bd44dd74b89cc80eb41172bcad8af75dd0bc182c2666d4c"}, - {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:67a5def279309f2e23014b608c4150b0c2d323bd7bccd27ff07b001c12c2415c"}, - {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:748bdf985014c6dd3e1e4cc3db90f1c3ecc7246ff5a3cd4ddab20c768b2f1dab"}, - {file = "pydantic_core-2.23.3-cp312-none-win32.whl", hash = "sha256:255ec6dcb899c115f1e2a64bc9ebc24cc0e3ab097775755244f77360d1f3c06c"}, - {file = "pydantic_core-2.23.3-cp312-none-win_amd64.whl", hash = "sha256:40b8441be16c1e940abebed83cd006ddb9e3737a279e339dbd6d31578b802f7b"}, - {file = "pydantic_core-2.23.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6daaf5b1ba1369a22c8b050b643250e3e5efc6a78366d323294aee54953a4d5f"}, - {file = "pydantic_core-2.23.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d015e63b985a78a3d4ccffd3bdf22b7c20b3bbd4b8227809b3e8e75bc37f9cb2"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3fc572d9b5b5cfe13f8e8a6e26271d5d13f80173724b738557a8c7f3a8a3791"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f6bd91345b5163ee7448bee201ed7dd601ca24f43f439109b0212e296eb5b423"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc379c73fd66606628b866f661e8785088afe2adaba78e6bbe80796baf708a63"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbdce4b47592f9e296e19ac31667daed8753c8367ebb34b9a9bd89dacaa299c9"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc3cf31edf405a161a0adad83246568647c54404739b614b1ff43dad2b02e6d5"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e22b477bf90db71c156f89a55bfe4d25177b81fce4aa09294d9e805eec13855"}, - {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:0a0137ddf462575d9bce863c4c95bac3493ba8e22f8c28ca94634b4a1d3e2bb4"}, - {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:203171e48946c3164fe7691fc349c79241ff8f28306abd4cad5f4f75ed80bc8d"}, - {file = "pydantic_core-2.23.3-cp313-none-win32.whl", hash = "sha256:76bdab0de4acb3f119c2a4bff740e0c7dc2e6de7692774620f7452ce11ca76c8"}, - {file = "pydantic_core-2.23.3-cp313-none-win_amd64.whl", hash = "sha256:37ba321ac2a46100c578a92e9a6aa33afe9ec99ffa084424291d84e456f490c1"}, - {file = "pydantic_core-2.23.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d063c6b9fed7d992bcbebfc9133f4c24b7a7f215d6b102f3e082b1117cddb72c"}, - {file = "pydantic_core-2.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6cb968da9a0746a0cf521b2b5ef25fc5a0bee9b9a1a8214e0a1cfaea5be7e8a4"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbefe079a520c5984e30e1f1f29325054b59534729c25b874a16a5048028d16"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbaaf2ef20d282659093913da9d402108203f7cb5955020bd8d1ae5a2325d1c4"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb539d7e5dc4aac345846f290cf504d2fd3c1be26ac4e8b5e4c2b688069ff4cf"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e6f33503c5495059148cc486867e1d24ca35df5fc064686e631e314d959ad5b"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04b07490bc2f6f2717b10c3969e1b830f5720b632f8ae2f3b8b1542394c47a8e"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03795b9e8a5d7fda05f3873efc3f59105e2dcff14231680296b87b80bb327295"}, - {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c483dab0f14b8d3f0df0c6c18d70b21b086f74c87ab03c59250dbf6d3c89baba"}, - {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8b2682038e255e94baf2c473dca914a7460069171ff5cdd4080be18ab8a7fd6e"}, - {file = "pydantic_core-2.23.3-cp38-none-win32.whl", hash = "sha256:f4a57db8966b3a1d1a350012839c6a0099f0898c56512dfade8a1fe5fb278710"}, - {file = "pydantic_core-2.23.3-cp38-none-win_amd64.whl", hash = "sha256:13dd45ba2561603681a2676ca56006d6dee94493f03d5cadc055d2055615c3ea"}, - {file = "pydantic_core-2.23.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82da2f4703894134a9f000e24965df73cc103e31e8c31906cc1ee89fde72cbd8"}, - {file = "pydantic_core-2.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dd9be0a42de08f4b58a3cc73a123f124f65c24698b95a54c1543065baca8cf0e"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89b731f25c80830c76fdb13705c68fef6a2b6dc494402987c7ea9584fe189f5d"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6de1ec30c4bb94f3a69c9f5f2182baeda5b809f806676675e9ef6b8dc936f28"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb68b41c3fa64587412b104294b9cbb027509dc2f6958446c502638d481525ef"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c3980f2843de5184656aab58698011b42763ccba11c4a8c35936c8dd6c7068c"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94f85614f2cba13f62c3c6481716e4adeae48e1eaa7e8bac379b9d177d93947a"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:510b7fb0a86dc8f10a8bb43bd2f97beb63cffad1203071dc434dac26453955cd"}, - {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1eba2f7ce3e30ee2170410e2171867ea73dbd692433b81a93758ab2de6c64835"}, - {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b259fd8409ab84b4041b7b3f24dcc41e4696f180b775961ca8142b5b21d0e70"}, - {file = "pydantic_core-2.23.3-cp39-none-win32.whl", hash = "sha256:40d9bd259538dba2f40963286009bf7caf18b5112b19d2b55b09c14dde6db6a7"}, - {file = "pydantic_core-2.23.3-cp39-none-win_amd64.whl", hash = "sha256:5a8cd3074a98ee70173a8633ad3c10e00dcb991ecec57263aacb4095c5efb958"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f399e8657c67313476a121a6944311fab377085ca7f490648c9af97fc732732d"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:6b5547d098c76e1694ba85f05b595720d7c60d342f24d5aad32c3049131fa5c4"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dda0290a6f608504882d9f7650975b4651ff91c85673341789a476b1159f211"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65b6e5da855e9c55a0c67f4db8a492bf13d8d3316a59999cfbaf98cc6e401961"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09e926397f392059ce0afdcac920df29d9c833256354d0c55f1584b0b70cf07e"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:87cfa0ed6b8c5bd6ae8b66de941cece179281239d482f363814d2b986b79cedc"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e61328920154b6a44d98cabcb709f10e8b74276bc709c9a513a8c37a18786cc4"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce3317d155628301d649fe5e16a99528d5680af4ec7aa70b90b8dacd2d725c9b"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e89513f014c6be0d17b00a9a7c81b1c426f4eb9224b15433f3d98c1a071f8433"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4f62c1c953d7ee375df5eb2e44ad50ce2f5aff931723b398b8bc6f0ac159791a"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2718443bc671c7ac331de4eef9b673063b10af32a0bb385019ad61dcf2cc8f6c"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d90e08b2727c5d01af1b5ef4121d2f0c99fbee692c762f4d9d0409c9da6541"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2b676583fc459c64146debea14ba3af54e540b61762dfc0613dc4e98c3f66eeb"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:50e4661f3337977740fdbfbae084ae5693e505ca2b3130a6d4eb0f2281dc43b8"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:68f4cf373f0de6abfe599a38307f4417c1c867ca381c03df27c873a9069cda25"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:59d52cf01854cb26c46958552a21acb10dd78a52aa34c86f284e66b209db8cab"}, - {file = "pydantic_core-2.23.3.tar.gz", hash = "sha256:3cb0f65d8b4121c1b015c60104a685feb929a29d7cf204387c7f2688c7974690"}, + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, + {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, + {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, + {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, + {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, + {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, + {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, + {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, + {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, + {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, + {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, + {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, + {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, + {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, ] [package.dependencies] From 9c6a59c94209fa2cdecdf12347a738d366e6629e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 05:57:31 +0000 Subject: [PATCH 112/254] Bump cloudcheck from 5.0.1.547 to 5.0.1.571 Bumps [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck) from 5.0.1.547 to 5.0.1.571. - [Commits](https://github.com/blacklanternsecurity/cloudcheck/commits) --- updated-dependencies: - dependency-name: cloudcheck dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index c7f627d32..3015ef10f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -387,13 +387,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cloudcheck" -version = "5.0.1.547" +version = "5.0.1.571" description = "Check whether an IP address belongs to a cloud provider" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "cloudcheck-5.0.1.547-py3-none-any.whl", hash = "sha256:0c183039d3c35611cab1fac5bb6a451a5a520735deb004f3550e2711a14a50c4"}, - {file = "cloudcheck-5.0.1.547.tar.gz", hash = "sha256:43740b552f0201df1bad1acf3a83fb0c566b695b5c156b9e815c9c83be9ae071"}, + {file = "cloudcheck-5.0.1.571-py3-none-any.whl", hash = "sha256:5a3a06f74fa7ab6828d27182f42b199e89966b615618f87c87c05f3f347fa7dd"}, + {file = "cloudcheck-5.0.1.571.tar.gz", hash = "sha256:342f0b84bd7f8dddbfcd08277f8bf23fd0c868232a97bc20be7c7af196330b02"}, ] [package.dependencies] From 4b0be8a4384bad3aeb7bf601c5f796046e4247af Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 23 Sep 2024 12:02:44 -0400 Subject: [PATCH 113/254] removing debugging --- bbot/modules/baddns.py | 12 +++++++----- bbot/modules/baddns_direct.py | 10 ++++++---- bbot/modules/baddns_zone.py | 8 ++------ bbot/presets/baddns-thorough.yml | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index d35a16bf8..d8ce14e46 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -31,6 +31,11 @@ def select_modules(self): selected_submodules.append(m) return selected_submodules + def set_modules(self): + self.enabled_submodules = self.config.get("enabled_submodules", []) + if self.enabled_submodules == []: + self.enabled_submodules = ["CNAME", "MX", "TXT"] + async def setup(self): self.preset.core.logger.include_logger(logging.getLogger("baddns")) self.custom_nameservers = self.config.get("custom_nameservers", []) or None @@ -38,10 +43,7 @@ async def setup(self): self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers) self.only_high_confidence = self.config.get("only_high_confidence", False) self.signatures = load_signatures() - - self.enabled_submodules = self.config.get("enabled_submodules", []) - if self.enabled_submodules == []: - self.enabled_submodules = ["CNAME", "MX", "TXT"] + self.set_modules() all_submodules_list = [m.name for m in get_all_modules()] for m in self.enabled_submodules: if m not in all_submodules_list: @@ -49,7 +51,7 @@ async def setup(self): f"Selected BadDNS submodule [{m}] does not exist. Available submodules: [{','.join(all_submodules_list)}]" ) return False - self.critical(f"Enabled BadDNS Submodules: [{','.join(self.enabled_submodules)}]") + self.debug(f"Enabled BadDNS Submodules: [{','.join(self.enabled_submodules)}]") return True async def handle_event(self, event): diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 475f6b780..e575d2bd4 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -40,7 +40,6 @@ def select_modules(self): return selected_modules async def handle_event(self, event): - self.critical(event.type) CNAME_direct_module = self.select_modules()[0] kwargs = { "http_client_class": self.scan.helpers.web.AsyncClient, @@ -74,16 +73,19 @@ async def filter_event(self, event): if event.type == "STORAGE_BUCKET": if str(event.module).startswith("bucket_"): return False + self.debug(f"Processing STORAGE_BUCKET for {event.host}") if event.type == "URL": if event.scope_distance > 0: - self.critical( + self.debug( f"Rejecting {event.host} due to not being in scope (scope distance: {str(event.scope_distance)})" ) return False if "cdn-cloudflare" not in event.tags: - self.critical(f"Rejecting {event.host} due to not being behind CloudFlare") + self.debug(f"Rejecting {event.host} due to not being behind CloudFlare") return False if "status-200" in event.tags or "status-301" in event.tags: - self.critical(f"Rejecting {event.host} due to lack of non-standard status code") + self.debug(f"Rejecting {event.host} due to lack of non-standard status code") return False + + self.debug(f"Passed all checks and is processing {event.host}") return True diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index d242fdaab..b755e5ee2 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -19,12 +19,8 @@ class baddns_zone(baddns_module): module_threads = 8 deps_pip = ["baddns~=1.1.839"] - def select_modules(self): - selected_modules = [] - for m in get_all_modules(): - if m.name in ["NSEC", "zonetransfer"]: - selected_modules.append(m) - return selected_modules + def set_modules(self): + self.enabled_submodules = ["NSEC", "zonetransfer"] # minimize nsec records feeding back into themselves async def filter_event(self, event): diff --git a/bbot/presets/baddns-thorough.yml b/bbot/presets/baddns-thorough.yml index 08c335a69..e2299b841 100644 --- a/bbot/presets/baddns-thorough.yml +++ b/bbot/presets/baddns-thorough.yml @@ -9,4 +9,4 @@ modules: config: modules: baddns: - enabled_submodules: [NSEC,CNAME,references,zonetransfer,MX,NS,TXT] + enabled_submodules: [CNAME,references,MX,NS,TXT] From d74e3df6ae63ec137e7e839afcec1eea80087756 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 23 Sep 2024 12:03:51 -0400 Subject: [PATCH 114/254] flake --- bbot/modules/baddns_zone.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index b755e5ee2..16eb7c1ae 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -1,4 +1,3 @@ -from baddns.base import get_all_modules from .baddns import baddns as baddns_module From d5f98c92ff8a8efb35d30eeb0aaae939638bb469 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 23 Sep 2024 18:20:44 -0400 Subject: [PATCH 115/254] adding baddns direct initial test --- .../module_tests/test_module_baddns_direct.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 bbot/test/test_step_2/module_tests/test_module_baddns_direct.py diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py new file mode 100644 index 000000000..4dd1cd646 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py @@ -0,0 +1,59 @@ +from .base import ModuleTestBase +from bbot.modules.base import BaseModule + +class BaseTestBaddns(ModuleTestBase): + modules_overrides = ["baddns_direct"] + targets = ["bad.dns"] + config_overrides = {"dns": {"minimal": False}, "cloudcheck": True} + + +class TestBaddns_direct_cloudflare(BaseTestBaddns): + targets = ["bad.dns:8888"] + modules_overrides = ["baddns_direct"] + + async def dispatchWHOIS(self): + return None + + class DummyModule(BaseModule): + watched_events = ["DNS_NAME"] + _name = "dummy_module" + events_seen = [] + + async def handle_event(self, event): + if event.data == "bad.dns": + await self.helpers.sleep(0.5) + self.events_seen.append(event.data) + url = "http://bad.dns:8888/" + url_event = self.scan.make_event(url, "URL", parent=self.scan.root_event, tags=["cdn-cloudflare", "in-scope", "status-401"]) + if url_event is not None: + await self.emit_event(url_event) + + async def setup_after_prep(self, module_test): + from baddns.base import BadDNS_base + from baddns.lib.whoismanager import WhoisManager + + def set_target(self, target): + return "127.0.0.1:8888" + + self.module_test = module_test + + self.dummy_module = self.DummyModule(module_test.scan) + module_test.scan.modules["dummy_module"] = self.dummy_module + + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "The specified bucket does not exist", "status": 401} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + await module_test.mock_dns( + {"bad.dns": {"A": ["127.0.0.1"]}} + ) + + module_test.monkeypatch.setattr(BadDNS_base, "set_target", set_target) + module_test.monkeypatch.setattr(WhoisManager, "dispatchWHOIS", self.dispatchWHOIS) + + def check(self, module_test, events): + assert any( + [e.type == "FINDING" and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" in e.data["description"] for e in events] + ), "Failed to emit FINDING" + assert any(["baddns-cname" in e.tags for e in events]), "Failed to add baddns tag" + From 399e36b2e196b75d5b9a2770120421cb1e09adc8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 23 Sep 2024 20:03:20 -0400 Subject: [PATCH 116/254] black --- .../module_tests/test_module_baddns_direct.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py index 4dd1cd646..596d3c89e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py +++ b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py @@ -1,6 +1,7 @@ from .base import ModuleTestBase from bbot.modules.base import BaseModule + class BaseTestBaddns(ModuleTestBase): modules_overrides = ["baddns_direct"] targets = ["bad.dns"] @@ -24,9 +25,11 @@ async def handle_event(self, event): await self.helpers.sleep(0.5) self.events_seen.append(event.data) url = "http://bad.dns:8888/" - url_event = self.scan.make_event(url, "URL", parent=self.scan.root_event, tags=["cdn-cloudflare", "in-scope", "status-401"]) + url_event = self.scan.make_event( + url, "URL", parent=self.scan.root_event, tags=["cdn-cloudflare", "in-scope", "status-401"] + ) if url_event is not None: - await self.emit_event(url_event) + await self.emit_event(url_event) async def setup_after_prep(self, module_test): from baddns.base import BadDNS_base @@ -34,7 +37,7 @@ async def setup_after_prep(self, module_test): def set_target(self, target): return "127.0.0.1:8888" - + self.module_test = module_test self.dummy_module = self.DummyModule(module_test.scan) @@ -44,16 +47,18 @@ def set_target(self, target): respond_args = {"response_data": "The specified bucket does not exist", "status": 401} module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - await module_test.mock_dns( - {"bad.dns": {"A": ["127.0.0.1"]}} - ) + await module_test.mock_dns({"bad.dns": {"A": ["127.0.0.1"]}}) module_test.monkeypatch.setattr(BadDNS_base, "set_target", set_target) module_test.monkeypatch.setattr(WhoisManager, "dispatchWHOIS", self.dispatchWHOIS) def check(self, module_test, events): assert any( - [e.type == "FINDING" and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" in e.data["description"] for e in events] + [ + e.type == "FINDING" + and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" + in e.data["description"] + for e in events + ] ), "Failed to emit FINDING" assert any(["baddns-cname" in e.tags for e in events]), "Failed to add baddns tag" - From eaf95fbd14c15f7c8d34515cd80149bc22a87b1f Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 25 Sep 2024 00:25:16 +0000 Subject: [PATCH 117/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 9ae66ab37..e1af0d99e 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -14,7 +14,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.82.2", + "version": "3.82.4", "config": "", "only_verified": True, "concurrency": 8, From b44d6fcd8ababc43fc22f8985453bcc020ffc579 Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 26 Sep 2024 00:24:42 +0000 Subject: [PATCH 118/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index e1af0d99e..37485f59f 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -14,7 +14,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.82.4", + "version": "3.82.5", "config": "", "only_verified": True, "concurrency": 8, From 290c7165d26c55d015f0e03f627bcc22615bd1b0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 26 Sep 2024 15:57:21 -0400 Subject: [PATCH 119/254] update docstring --- bbot/scanner/scanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 73dd5a3d2..5bc4cec0b 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -1044,10 +1044,10 @@ async def dns_yara_rules(self): async def extract_in_scope_hostnames(self, s): """ - Given a string, uses yara to extract hostnames that match scan targets + Given a string, uses yara to extract hostnames matching scan targets Examples: - >>> self.scan.extract_in_scope_hostnames("http://www.evilcorp.com") + >>> await self.scan.extract_in_scope_hostnames("http://www.evilcorp.com") ... {"www.evilcorp.com"} """ matches = set() From b0472f6b4c4ebf6bb575253503257c47278f8be2 Mon Sep 17 00:00:00 2001 From: GitHub Date: Fri, 27 Sep 2024 00:25:05 +0000 Subject: [PATCH 120/254] Update nuclei --- bbot/modules/deadly/nuclei.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index b4750c55b..54bea9b82 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -15,7 +15,7 @@ class nuclei(BaseModule): } options = { - "version": "3.3.2", + "version": "3.3.3", "tags": "", "templates": "", "severity": "", From 1c316f520125b30c9ed89caf6a99b9a8a8fa2c7c Mon Sep 17 00:00:00 2001 From: GitHub Date: Fri, 27 Sep 2024 00:25:12 +0000 Subject: [PATCH 121/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 37485f59f..61aa2306d 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -14,7 +14,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.82.5", + "version": "3.82.6", "config": "", "only_verified": True, "concurrency": 8, From df920ca5c1f5852a4179fc118baf0bf29d39d964 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 27 Sep 2024 13:54:25 -0400 Subject: [PATCH 122/254] update docs --- docs/modules/list_of_modules.md | 20 +++++----- docs/modules/nuclei.md | 2 +- docs/scanning/advanced.md | 16 ++++---- docs/scanning/configuration.md | 36 ++++++++++++------ docs/scanning/events.md | 66 ++++++++++++++++----------------- docs/scanning/index.md | 48 ++++++++++++------------ docs/scanning/presets_list.md | 44 +++++++++++----------- 7 files changed, 125 insertions(+), 107 deletions(-) diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index 6c916fa3f..abed66cff 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -14,6 +14,9 @@ | bucket_google | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 | | bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | @liquidsec | 2022-07-05 | | dastardly | scan | No | Lightweight web application security scanner | active, aggressive, deadly, slow, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | @domwhewell-sage | 2023-12-11 | +| dnsbrute | scan | No | Brute-force subdomains with massdns + static wordlist | active, aggressive, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-24 | +| dnsbrute_mutations | scan | No | Brute-force subdomains with massdns + target-specific mutations | active, aggressive, slow, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-25 | +| dnscommonsrv | scan | No | Check for common SRV records | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-15 | | dotnetnuke | scan | No | Scan for critical DotNetNuke (DNN) vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE | TECHNOLOGY, VULNERABILITY | @liquidsec | 2023-11-21 | | ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | @pmueller | 2022-04-10 | | ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | @liquidsec | 2022-07-05 | @@ -37,6 +40,7 @@ | portscan | scan | No | Port scan with masscan. By default, scans top 100 ports. | active, portscan, safe | DNS_NAME, IP_ADDRESS, IP_RANGE | OPEN_TCP_PORT | @TheTechromancer | 2024-05-15 | | robots | scan | No | Look for and parse robots.txt | active, safe, web-basic | URL | URL_UNVERIFIED | @liquidsec | 2023-02-01 | | secretsdb | scan | No | Detect common secrets with secrets-patterns-db | active, safe, web-basic | HTTP_RESPONSE | FINDING | @TheTechromancer | 2023-03-17 | +| securitytxt | scan | No | Check for security.txt content | active, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME | EMAIL_ADDRESS, URL_UNVERIFIED | @colin-stubbs | 2024-05-26 | | smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | @liquidsec | 2022-07-06 | | sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | @TheTechromancer | 2022-03-30 | | telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE, URL | FINDING, VULNERABILITY | @liquidsec | 2022-04-10 | @@ -64,19 +68,16 @@ | crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-13 | | dehashed | scan | Yes | Execute queries against dehashed.com for exposed credentials | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | @SpamFaux | 2023-10-12 | | digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-25 | -| dnsbrute | scan | No | Brute-force subdomains with massdns + static wordlist | aggressive, passive, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-24 | -| dnsbrute_mutations | scan | No | Brute-force subdomains with massdns + target-specific mutations | aggressive, passive, slow, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-25 | | dnscaa | scan | No | Check for CAA records | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | @colin-stubbs | 2024-05-26 | -| dnscommonsrv | scan | No | Check for common SRV records | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-15 | | dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-03-12 | -| docker_pull | scan | No | Download images from a docker repository | passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-24 | +| docker_pull | scan | No | Download images from a docker repository | code-enum, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-24 | | dockerhub | scan | No | Search for docker repositories of discovered orgs/usernames | code-enum, passive, safe | ORG_STUB, SOCIAL | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED | @domwhewell-sage | 2024-03-12 | | emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 | | fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | -| git_clone | scan | No | Clone code github repositories | passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-08 | +| git_clone | scan | No | Clone code github repositories | code-enum, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-08 | | github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | code-enum, passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | @domwhewell-sage | 2023-12-14 | | github_org | scan | No | Query Github's API for organization and member repositories | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | @domwhewell-sage | 2023-12-14 | -| github_workflows | scan | No | Download a github repositories workflow logs | passive, safe | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-04-29 | +| github_workflows | scan | No | Download a github repositories workflow logs and workflow artifacts | code-enum, passive, safe | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-04-29 | | hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-28 | | hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | @TheTechromancer | 2022-04-25 | | internetdb | scan | No | Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities | passive, portscan, safe, subdomain-enum | DNS_NAME, IP_ADDRESS | DNS_NAME, FINDING, OPEN_TCP_PORT, TECHNOLOGY, VULNERABILITY | @TheTechromancer | 2023-12-22 | @@ -88,7 +89,8 @@ | otx | scan | No | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | | passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-08 | | pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-08-10 | -| postman | scan | No | Query Postman's API for related workspaces, collections, requests | code-enum, passive, safe, subdomain-enum | DNS_NAME | URL_UNVERIFIED | @domwhewell-sage | 2023-12-23 | +| postman | scan | No | Query Postman's API for related workspaces, collections, requests and download them | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | @domwhewell-sage | 2024-09-07 | +| postman_download | scan | No | Download workspaces, collections, requests from Postman | code-enum, passive, safe, subdomain-enum | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-09-07 | | rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | | securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-03 | | shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-03 | @@ -97,7 +99,7 @@ | social | scan | No | Look for social media links in webpages | passive, safe, social-enum | URL_UNVERIFIED | SOCIAL | @TheTechromancer | 2023-03-28 | | subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-26 | | trickest | scan | Yes | Query Trickest's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @amiremami | 2024-07-27 | -| trufflehog | scan | No | TruffleHog is a tool for finding credentials | code-enum, passive, safe | FILESYSTEM | FINDING, VULNERABILITY | @domwhewell-sage | 2024-03-12 | +| trufflehog | scan | No | TruffleHog is a tool for finding credentials | code-enum, passive, safe | CODE_REPOSITORY, FILESYSTEM | FINDING, VULNERABILITY | @domwhewell-sage | 2024-03-12 | | unstructured | scan | No | Module to extract data from files | passive, safe | FILESYSTEM | FILESYSTEM, RAW_TEXT | @domwhewell-sage | 2024-06-03 | | urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @TheTechromancer | 2022-06-09 | | viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-04 | @@ -123,7 +125,7 @@ | cloudcheck | internal | No | Tag events by cloud provider, identify cloud resources like storage buckets | | * | | | | | dnsresolve | internal | No | | | * | | | | | aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | @TheTechromancer | 2022-07-25 | -| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE | URL_UNVERIFIED, WEB_PARAMETER | @liquidsec | 2022-06-27 | +| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE, RAW_TEXT | URL_UNVERIFIED, WEB_PARAMETER | @liquidsec | 2022-06-27 | | speculate | internal | No | Derive certain event types from others by common sense | passive | AZURE_TENANT, DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, SOCIAL, STORAGE_BUCKET, URL, URL_UNVERIFIED, USERNAME | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, ORG_STUB | @liquidsec | 2022-05-03 | diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md index f0f3efed6..561c2e93c 100644 --- a/docs/modules/nuclei.md +++ b/docs/modules/nuclei.md @@ -51,7 +51,7 @@ The Nuclei module has many configuration options: | modules.nuclei.silent | bool | Don't display nuclei's banner or status messages | False | | modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | | | modules.nuclei.templates | str | template or template directory paths to include in the scan | | -| modules.nuclei.version | str | nuclei version | 3.2.0 | +| modules.nuclei.version | str | nuclei version | 3.3.2 | Most of these you probably will **NOT** want to change. In particular, we advise against changing the version of Nuclei, as it's possible the latest version won't work right with BBOT. diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index b3a142dc8..2df440e38 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -32,10 +32,12 @@ if __name__ == "__main__": ```text -usage: bbot [-h] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-p [PRESET ...]] [-c [CONFIG ...]] [-lp] - [-m MODULE [MODULE ...]] [-l] [-lmo] [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [--allow-deadly] [-n SCAN_NAME] [-v] - [-d] [-s] [--force] [-y] [--dry-run] [--current-preset] [--current-preset-full] [-o DIR] [-om MODULE [MODULE ...]] [--json] [--brief] - [--event-types EVENT_TYPES [EVENT_TYPES ...]] [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [--version] +usage: bbot [-h] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-p [PRESET ...]] + [-c [CONFIG ...]] [-lp] [-m MODULE [MODULE ...]] [-l] [-lmo] [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] + [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [--allow-deadly] [-n SCAN_NAME] [-v] [-d] [-s] [--force] [-y] [--dry-run] + [--current-preset] [--current-preset-full] [-o DIR] [-om MODULE [MODULE ...]] [--json] [--brief] + [--event-types EVENT_TYPES [EVENT_TYPES ...]] + [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [--version] [-H CUSTOM_HEADERS [CUSTOM_HEADERS ...]] [--custom-yara-rules CUSTOM_YARA_RULES] Bighuge BLS OSINT Tool @@ -61,14 +63,14 @@ Presets: Modules: -m MODULE [MODULE ...], --modules MODULE [MODULE ...] - Modules to enable. Choices: viewdns,postman,baddns_zone,dehashed,bucket_file_enum,asn,generic_ssrf,github_codesearch,columbus,azure_realm,dotnetnuke,dockerhub,credshed,passivetotal,certspotter,builtwith,otx,ipneighbor,fingerprintx,oauth,robots,dnsbrute_mutations,httpx,paramminer_headers,digitorus,gitlab,hunt,hunterio,trufflehog,ffuf,nuclei,badsecrets,git,bucket_firebase,ffuf_shortnames,urlscan,docker_pull,ip2location,subdomaincenter,telerik,pgp,zoomeye,shodan_dns,trickest,dnscommonsrv,ntlm,myssl,internetdb,emailformat,dastardly,azure_tenant,github_workflows,crt,affiliates,wayback,ajaxpro,wafw00f,iis_shortnames,sslcert,chaos,newsletters,host_header,bucket_amazon,vhost,paramminer_cookies,virustotal,rapiddns,leakix,dnsbrute,baddns,url_manipulation,code_repository,smuggler,bevigil,paramminer_getparams,unstructured,skymem,securitytrails,sitedossier,git_clone,bucket_azure,bucket_google,bypass403,wpscan,dnsdumpster,wappalyzer,dnscaa,social,hackertarget,github_org,fullhunt,filedownload,binaryedge,gowitness,anubisdb,portscan,ipstack,secretsdb,c99,censys,bucket_digitalocean + Modules to enable. Choices: ntlm,robots,dockerhub,azure_tenant,crt,dnscommonsrv,dastardly,c99,hunt,skymem,dnscaa,gowitness,postman_download,dnsbrute,newsletters,secretsdb,nuclei,columbus,oauth,viewdns,shodan_dns,emailformat,gitlab,wappalyzer,internetdb,pgp,affiliates,bucket_file_enum,url_manipulation,ipneighbor,bucket_firebase,paramminer_cookies,virustotal,securitytxt,smuggler,dnsdumpster,dnsbrute_mutations,baddns,fingerprintx,paramminer_headers,wpscan,trufflehog,iis_shortnames,baddns_zone,dehashed,dotnetnuke,passivetotal,code_repository,generic_ssrf,portscan,censys,badsecrets,ipstack,bypass403,bucket_amazon,paramminer_getparams,github_workflows,github_codesearch,sslcert,otx,bucket_azure,fullhunt,postman,ffuf_shortnames,zoomeye,subdomaincenter,leakix,github_org,chaos,host_header,docker_pull,digitorus,unstructured,wafw00f,asn,credshed,vhost,trickest,binaryedge,bucket_google,filedownload,telerik,hunterio,httpx,ip2location,urlscan,git,hackertarget,git_clone,bevigil,wayback,certspotter,builtwith,ajaxpro,myssl,anubisdb,azure_realm,ffuf,rapiddns,securitytrails,bucket_digitalocean,sitedossier,social -l, --list-modules List available modules. -lmo, --list-module-options Show all module config options -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...] Exclude these modules. -f FLAG [FLAG ...], --flags FLAG [FLAG ...] - Enable modules by flag. Choices: subdomain-hijack,web-paramminer,subdomain-enum,code-enum,cloud-enum,iis-shortnames,web-thorough,baddns,portscan,slow,social-enum,affiliates,safe,web-screenshots,deadly,report,web-basic,email-enum,active,service-enum,aggressive,passive + Enable modules by flag. Choices: slow,service-enum,baddns,subdomain-enum,deadly,web-thorough,iis-shortnames,report,affiliates,social-enum,email-enum,cloud-enum,web-basic,passive,web-screenshots,aggressive,web-paramminer,safe,subdomain-hijack,portscan,code-enum,active -lf, --list-flags List available flags. -rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...] Only enable modules with these flags (e.g. -rf passive) @@ -93,7 +95,7 @@ Output: -o DIR, --output-dir DIR Directory to output scan results -om MODULE [MODULE ...], --output-modules MODULE [MODULE ...] - Output module(s). Choices: subdomains,emails,web_report,json,txt,websocket,slack,asset_inventory,neo4j,splunk,csv,stdout,http,python,discord,teams + Output module(s). Choices: python,csv,subdomains,stdout,splunk,teams,emails,slack,http,websocket,discord,neo4j,web_report,json,asset_inventory,txt --json, -j Output scan data in JSON format --brief, -br Output only the data itself --event-types EVENT_TYPES [EVENT_TYPES ...] diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 0d786cfff..efffeaf36 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -89,10 +89,13 @@ dns: # Speed up scan by not creating any new DNS events, and only resolving A and AAAA records minimal: false # How many instances of the dns module to run concurrently - threads: 20 + threads: 25 # How many concurrent DNS resolvers to use when brute-forcing # (under the hood this is passed through directly to massdns -s) brute_threads: 1000 + # nameservers to use for DNS brute-forcing + # default is updated weekly and contains ~10K high-quality public servers + brute_nameservers: https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt # How far away from the main target to explore via DNS resolution (independent of scope.search_distance) # This is safe to change search_distance: 1 @@ -156,6 +159,11 @@ web: # Whether to verify SSL certificates ssl_verify: false +### ENGINE ### + +engine: + debug: false + # Tool dependencies deps: ffuf: @@ -262,6 +270,10 @@ Many modules accept their own configuration options. These options have the abil | modules.bucket_digitalocean.permutations | bool | Whether to try permutations | False | | modules.bucket_firebase.permutations | bool | Whether to try permutations | False | | modules.bucket_google.permutations | bool | Whether to try permutations | False | +| modules.dnsbrute.max_depth | int | How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com | 5 | +| modules.dnsbrute.wordlist | str | Subdomain wordlist URL | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt | +| modules.dnsbrute_mutations.max_mutations | int | Maximum number of target-specific mutations to try per subdomain | 100 | +| modules.dnscommonsrv.max_depth | int | The maximum subdomain depth to brute-force SRV records | 2 | | modules.ffuf.extensions | str | Optionally include a list of extensions to extend the keyword with (comma separated) | | | modules.ffuf.lines | int | take only the first N lines from the wordlist when finding directories | 5000 | | modules.ffuf.max_depth | int | the maximum directory depth to attempt to solve | 0 | @@ -310,7 +322,7 @@ Many modules accept their own configuration options. These options have the abil | modules.nuclei.silent | bool | Don't display nuclei's banner or status messages | False | | modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | | | modules.nuclei.templates | str | template or template directory paths to include in the scan | | -| modules.nuclei.version | str | nuclei version | 3.2.0 | +| modules.nuclei.version | str | nuclei version | 3.3.2 | | modules.oauth.try_all | bool | Check for OAUTH/IODC on every subdomain and URL. | False | | modules.paramminer_cookies.recycle_words | bool | Attempt to use words found during the scan on all other endpoints | False | | modules.paramminer_cookies.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True | @@ -336,6 +348,8 @@ Many modules accept their own configuration options. These options have the abil | modules.robots.include_sitemap | bool | Include 'sitemap' entries | False | | modules.secretsdb.min_confidence | int | Only use signatures with this confidence score or higher | 99 | | modules.secretsdb.signatures | str | File path or URL to YAML signatures | https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml | +| modules.securitytxt.emails | bool | emit EMAIL_ADDRESS events | True | +| modules.securitytxt.urls | bool | emit URL_UNVERIFIED events | True | | modules.sslcert.skip_non_ssl | bool | Don't try common non-SSL ports | True | | modules.sslcert.timeout | float | Socket connect timeout in seconds | 5.0 | | modules.telerik.exploit_RAU_crypto | bool | Attempt to confirm any RAU AXD detections are vulnerable | False | @@ -345,11 +359,11 @@ Many modules accept their own configuration options. These options have the abil | modules.vhost.wordlist | str | Wordlist containing subdomains | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt | | modules.wafw00f.generic_detect | bool | When no specific WAF detections are made, try to perform a generic detect | True | | modules.wpscan.api_key | str | WPScan API Key | | -| modules.wpscan.connection_timeout | int | The connection timeout in seconds (default 30) | 30 | +| modules.wpscan.connection_timeout | int | The connection timeout in seconds (default 2) | 2 | | modules.wpscan.disable_tls_checks | bool | Disables the SSL/TLS certificate verification (Default True) | True | -| modules.wpscan.enumerate | str | Enumeration Process see wpscan help documentation (default: vp,vt,tt,cb,dbe,u,m) | vp,vt,tt,cb,dbe,u,m | +| modules.wpscan.enumerate | str | Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe) | vp,vt,cb,dbe | | modules.wpscan.force | bool | Do not check if the target is running WordPress or returns a 403 | False | -| modules.wpscan.request_timeout | int | The request timeout in seconds (default 60) | 60 | +| modules.wpscan.request_timeout | int | The request timeout in seconds (default 5) | 5 | | modules.wpscan.threads | int | How many wpscan threads to spawn (default is 5) | 5 | | modules.anubisdb.limit | int | Limit the number of subdomains returned per query (increasing this may slow the scan due to garbage results from this API) | 1000 | | modules.bevigil.api_key | str | BeVigil OSINT API Key | | @@ -369,14 +383,10 @@ Many modules accept their own configuration options. These options have the abil | modules.credshed.username | str | Credshed username | | | modules.dehashed.api_key | str | DeHashed API Key | | | modules.dehashed.username | str | Email Address associated with your API key | | -| modules.dnsbrute.max_depth | int | How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com | 5 | -| modules.dnsbrute.wordlist | str | Subdomain wordlist URL | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt | -| modules.dnsbrute_mutations.max_mutations | int | Maximum number of target-specific mutations to try per subdomain | 100 | | modules.dnscaa.dns_names | bool | emit DNS_NAME events | True | | modules.dnscaa.emails | bool | emit EMAIL_ADDRESS events | True | | modules.dnscaa.in_scope_only | bool | Only check in-scope domains | True | | modules.dnscaa.urls | bool | emit URL_UNVERIFIED events | True | -| modules.dnscommonsrv.max_depth | int | The maximum subdomain depth to brute-force SRV records | 2 | | modules.docker_pull.all_tags | bool | Download all tags from each registry (Default False) | False | | modules.docker_pull.output_folder | str | Folder to download docker repositories to | | | modules.fullhunt.api_key | str | FullHunt API Key | | @@ -399,12 +409,16 @@ Many modules accept their own configuration options. These options have the abil | modules.passivetotal.api_key | str | RiskIQ API Key | | | modules.passivetotal.username | str | RiskIQ Username | | | modules.pgp.search_urls | list | PGP key servers to search |` ['https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=', 'http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search=', 'https://pgpkeys.eu/pks/lookup?search=&op=index', 'https://pgp.mit.edu/pks/lookup?search=&op=index'] `| +| modules.postman_download.api_key | str | Postman API Key | | +| modules.postman_download.output_folder | str | Folder to download postman workspaces to | | | modules.securitytrails.api_key | str | SecurityTrails API key | | | modules.shodan_dns.api_key | str | Shodan API key | | | modules.trickest.api_key | str | Trickest API key | | | modules.trufflehog.concurrency | int | Number of concurrent workers | 8 | +| modules.trufflehog.config | str | File path or URL to YAML trufflehog config | | +| modules.trufflehog.deleted_forks | bool | Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours. | False | | modules.trufflehog.only_verified | bool | Only report credentials that have been verified | True | -| modules.trufflehog.version | str | trufflehog version | 3.75.1 | +| modules.trufflehog.version | str | trufflehog version | 3.82.2 | | modules.unstructured.extensions | list | File extensions to parse | ['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'ica', 'indd', 'ini', 'key', 'pub', 'log', 'markdown', 'md', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'rdp', 'sh', 'sql', 'swp', 'sxw', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'] | | modules.unstructured.ignore_folders | list | Subfolders to ignore when crawling downloaded folders | ['.git'] | | modules.urlscan.urls | bool | Emit URLs in addition to DNS_NAMEs | False | @@ -452,7 +466,7 @@ Many modules accept their own configuration options. These options have the abil | modules.subdomains.output_file | str | Output to file | | | modules.teams.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | | modules.teams.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW | -| modules.teams.webhook_url | str | Discord webhook URL | | +| modules.teams.webhook_url | str | Teams webhook URL | | | modules.txt.output_file | str | Output to file | | | modules.web_report.css_theme_file | str | CSS theme URL for HTML output | https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css | | modules.web_report.output_file | str | Output to file | | diff --git a/docs/scanning/events.md b/docs/scanning/events.md index 9c44407fc..ed5a7cdb9 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -63,39 +63,39 @@ Below is a full list of event types along with which modules produce/consume the ## List of Event Types -| Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules | -|---------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| * | 15 | 0 | affiliates, cloudcheck, csv, discord, dnsresolve, http, json, neo4j, python, slack, splunk, stdout, teams, txt, websocket | | -| ASN | 0 | 1 | | asn | -| AZURE_TENANT | 1 | 0 | speculate | | -| CODE_REPOSITORY | 3 | 5 | docker_pull, git_clone, github_workflows | code_repository, dockerhub, github_codesearch, github_org, gitlab | -| DNS_NAME | 56 | 41 | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crt, dehashed, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, emailformat, fullhunt, github_codesearch, hackertarget, hunterio, internetdb, leakix, myssl, oauth, otx, passivetotal, pgp, portscan, postman, rapiddns, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, subdomains, trickest, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, hackertarget, hunterio, internetdb, leakix, myssl, ntlm, oauth, otx, passivetotal, rapiddns, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, trickest, urlscan, vhost, viewdns, virustotal, wayback, zoomeye | -| DNS_NAME_UNRESOLVED | 3 | 0 | baddns, speculate, subdomains | | -| EMAIL_ADDRESS | 1 | 8 | emails | credshed, dehashed, dnscaa, emailformat, hunterio, pgp, skymem, sslcert | -| FILESYSTEM | 2 | 5 | trufflehog, unstructured | docker_pull, filedownload, git_clone, github_workflows, unstructured | -| FINDING | 2 | 28 | asset_inventory, web_report | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, git, gitlab, host_header, hunt, internetdb, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, secretsdb, smuggler, speculate, telerik, trufflehog, url_manipulation, wpscan | -| GEOLOCATION | 0 | 2 | | ip2location, ipstack | -| HASHED_PASSWORD | 0 | 2 | | credshed, dehashed | -| HTTP_RESPONSE | 19 | 1 | ajaxpro, asset_inventory, badsecrets, dastardly, dotnetnuke, excavate, filedownload, gitlab, host_header, newsletters, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, speculate, telerik, wappalyzer, wpscan | httpx | -| IP_ADDRESS | 8 | 3 | asn, asset_inventory, internetdb, ip2location, ipneighbor, ipstack, portscan, speculate | asset_inventory, ipneighbor, speculate | -| IP_RANGE | 2 | 0 | portscan, speculate | | -| OPEN_TCP_PORT | 4 | 4 | asset_inventory, fingerprintx, httpx, sslcert | asset_inventory, internetdb, portscan, speculate | -| ORG_STUB | 2 | 1 | dockerhub, github_org | speculate | -| PASSWORD | 0 | 2 | | credshed, dehashed | -| PROTOCOL | 0 | 1 | | fingerprintx | -| RAW_TEXT | 0 | 1 | | unstructured | -| SOCIAL | 5 | 3 | dockerhub, github_org, gitlab, gowitness, speculate | dockerhub, gitlab, social | -| STORAGE_BUCKET | 7 | 5 | bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, speculate | bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google | -| TECHNOLOGY | 4 | 8 | asset_inventory, gitlab, web_report, wpscan | badsecrets, dotnetnuke, gitlab, gowitness, internetdb, nuclei, wappalyzer, wpscan | -| URL | 19 | 2 | ajaxpro, asset_inventory, bypass403, ffuf, generic_ssrf, git, gowitness, httpx, iis_shortnames, ntlm, nuclei, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report | gowitness, httpx | -| URL_HINT | 1 | 1 | ffuf_shortnames | iis_shortnames | -| URL_UNVERIFIED | 6 | 16 | code_repository, filedownload, httpx, oauth, social, speculate | azure_realm, bevigil, bucket_file_enum, dnscaa, dockerhub, excavate, ffuf, ffuf_shortnames, github_codesearch, gowitness, hunterio, postman, robots, urlscan, wayback, wpscan | -| USERNAME | 1 | 2 | speculate | credshed, dehashed | -| VHOST | 1 | 1 | web_report | vhost | -| VULNERABILITY | 2 | 12 | asset_inventory, web_report | ajaxpro, baddns, baddns_zone, badsecrets, dastardly, dotnetnuke, generic_ssrf, internetdb, nuclei, telerik, trufflehog, wpscan | -| WAF | 1 | 1 | asset_inventory | wafw00f | -| WEBSCREENSHOT | 0 | 1 | | gowitness | -| WEB_PARAMETER | 4 | 4 | hunt, paramminer_cookies, paramminer_getparams, paramminer_headers | excavate, paramminer_cookies, paramminer_getparams, paramminer_headers | +| Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules | +|---------------------|-----------------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| * | 15 | 0 | affiliates, cloudcheck, csv, discord, dnsresolve, http, json, neo4j, python, slack, splunk, stdout, teams, txt, websocket | | +| ASN | 0 | 1 | | asn | +| AZURE_TENANT | 1 | 0 | speculate | | +| CODE_REPOSITORY | 5 | 6 | docker_pull, git_clone, github_workflows, postman_download, trufflehog | code_repository, dockerhub, github_codesearch, github_org, gitlab, postman | +| DNS_NAME | 56 | 41 | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crt, dehashed, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, emailformat, fullhunt, github_codesearch, hackertarget, hunterio, internetdb, leakix, myssl, oauth, otx, passivetotal, pgp, portscan, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, subdomains, trickest, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, hackertarget, hunterio, internetdb, leakix, myssl, ntlm, oauth, otx, passivetotal, rapiddns, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, trickest, urlscan, vhost, viewdns, virustotal, wayback, zoomeye | +| DNS_NAME_UNRESOLVED | 3 | 0 | baddns, speculate, subdomains | | +| EMAIL_ADDRESS | 1 | 9 | emails | credshed, dehashed, dnscaa, emailformat, hunterio, pgp, securitytxt, skymem, sslcert | +| FILESYSTEM | 2 | 6 | trufflehog, unstructured | docker_pull, filedownload, git_clone, github_workflows, postman_download, unstructured | +| FINDING | 2 | 28 | asset_inventory, web_report | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, git, gitlab, host_header, hunt, internetdb, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, secretsdb, smuggler, speculate, telerik, trufflehog, url_manipulation, wpscan | +| GEOLOCATION | 0 | 2 | | ip2location, ipstack | +| HASHED_PASSWORD | 0 | 2 | | credshed, dehashed | +| HTTP_RESPONSE | 19 | 1 | ajaxpro, asset_inventory, badsecrets, dastardly, dotnetnuke, excavate, filedownload, gitlab, host_header, newsletters, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, speculate, telerik, wappalyzer, wpscan | httpx | +| IP_ADDRESS | 8 | 3 | asn, asset_inventory, internetdb, ip2location, ipneighbor, ipstack, portscan, speculate | asset_inventory, ipneighbor, speculate | +| IP_RANGE | 2 | 0 | portscan, speculate | | +| OPEN_TCP_PORT | 4 | 4 | asset_inventory, fingerprintx, httpx, sslcert | asset_inventory, internetdb, portscan, speculate | +| ORG_STUB | 3 | 1 | dockerhub, github_org, postman | speculate | +| PASSWORD | 0 | 2 | | credshed, dehashed | +| PROTOCOL | 0 | 1 | | fingerprintx | +| RAW_TEXT | 1 | 1 | excavate | unstructured | +| SOCIAL | 6 | 3 | dockerhub, github_org, gitlab, gowitness, postman, speculate | dockerhub, gitlab, social | +| STORAGE_BUCKET | 7 | 5 | bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, speculate | bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google | +| TECHNOLOGY | 4 | 8 | asset_inventory, gitlab, web_report, wpscan | badsecrets, dotnetnuke, gitlab, gowitness, internetdb, nuclei, wappalyzer, wpscan | +| URL | 19 | 2 | ajaxpro, asset_inventory, bypass403, ffuf, generic_ssrf, git, gowitness, httpx, iis_shortnames, ntlm, nuclei, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report | gowitness, httpx | +| URL_HINT | 1 | 1 | ffuf_shortnames | iis_shortnames | +| URL_UNVERIFIED | 6 | 16 | code_repository, filedownload, httpx, oauth, social, speculate | azure_realm, bevigil, bucket_file_enum, dnscaa, dockerhub, excavate, ffuf, ffuf_shortnames, github_codesearch, gowitness, hunterio, robots, securitytxt, urlscan, wayback, wpscan | +| USERNAME | 1 | 2 | speculate | credshed, dehashed | +| VHOST | 1 | 1 | web_report | vhost | +| VULNERABILITY | 2 | 12 | asset_inventory, web_report | ajaxpro, baddns, baddns_zone, badsecrets, dastardly, dotnetnuke, generic_ssrf, internetdb, nuclei, telerik, trufflehog, wpscan | +| WAF | 1 | 1 | asset_inventory | wafw00f | +| WEBSCREENSHOT | 0 | 1 | | gowitness | +| WEB_PARAMETER | 4 | 4 | hunt, paramminer_cookies, paramminer_getparams, paramminer_headers | excavate, paramminer_cookies, paramminer_getparams, paramminer_headers | ## Findings Vs. Vulnerabilities diff --git a/docs/scanning/index.md b/docs/scanning/index.md index 24069a8c1..ff3fb5825 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -107,30 +107,30 @@ A single module can have multiple flags. For example, the `securitytrails` modul ### List of Flags -| Flag | # Modules | Description | Modules | -|------------------|-------------|----------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| safe | 82 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crt, dehashed, digitorus, dnscaa, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, portscan, postman, rapiddns, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, trickest, trufflehog, unstructured, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | -| passive | 62 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crt, dehashed, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, myssl, otx, passivetotal, pgp, postman, rapiddns, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, trickest, trufflehog, unstructured, urlscan, viewdns, virustotal, wayback, zoomeye | -| subdomain-enum | 46 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, rapiddns, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, trickest, urlscan, virustotal, wayback, zoomeye | -| active | 42 | Makes active connections to target systems | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, newsletters, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, portscan, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer, wpscan | -| aggressive | 20 | Generates a large amount of network traffic | bypass403, dastardly, dnsbrute, dnsbrute_mutations, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f, wpscan | -| web-basic | 17 | Basic, non-intrusive web scan functionality | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | -| cloud-enum | 12 | Enumerates cloud resources | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth | -| web-thorough | 12 | More advanced web scanning functionality | ajaxpro, bucket_digitalocean, bypass403, dastardly, dotnetnuke, ffuf_shortnames, generic_ssrf, host_header, hunt, smuggler, telerik, url_manipulation | -| slow | 11 | May take a long time to complete | bucket_digitalocean, dastardly, dnsbrute_mutations, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | -| affiliates | 9 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, trickest, viewdns, zoomeye | -| code-enum | 8 | Find public code repositories and search them for secrets etc. | code_repository, dockerhub, git, github_codesearch, github_org, gitlab, postman, trufflehog | -| email-enum | 8 | Enumerates email addresses | dehashed, dnscaa, emailformat, emails, hunterio, pgp, skymem, sslcert | -| deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | -| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | -| baddns | 2 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone | -| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | -| portscan | 2 | Discovers open ports | internetdb, portscan | -| report | 2 | Generates a report at the end of the scan | affiliates, asn | -| social-enum | 2 | Enumerates social media | httpx, social | -| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | -| subdomain-hijack | 1 | Detects hijackable subdomains | baddns | -| web-screenshots | 1 | Takes screenshots of web pages | gowitness | +| Flag | # Modules | Description | Modules | +|------------------|-------------|----------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| safe | 84 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crt, dehashed, digitorus, dnscaa, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, portscan, postman, postman_download, rapiddns, robots, secretsdb, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, trickest, trufflehog, unstructured, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | +| passive | 60 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crt, dehashed, digitorus, dnscaa, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, myssl, otx, passivetotal, pgp, postman, postman_download, rapiddns, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, trickest, trufflehog, unstructured, urlscan, viewdns, virustotal, wayback, zoomeye | +| subdomain-enum | 48 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, trickest, urlscan, virustotal, wayback, zoomeye | +| active | 46 | Makes active connections to target systems | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dnsbrute, dnsbrute_mutations, dnscommonsrv, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, newsletters, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, portscan, robots, secretsdb, securitytxt, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer, wpscan | +| aggressive | 20 | Generates a large amount of network traffic | bypass403, dastardly, dnsbrute, dnsbrute_mutations, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f, wpscan | +| web-basic | 18 | Basic, non-intrusive web scan functionality | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, securitytxt, sslcert, wappalyzer | +| cloud-enum | 13 | Enumerates cloud resources | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth, securitytxt | +| code-enum | 12 | Find public code repositories and search them for secrets etc. | code_repository, docker_pull, dockerhub, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, postman, postman_download, trufflehog | +| web-thorough | 12 | More advanced web scanning functionality | ajaxpro, bucket_digitalocean, bypass403, dastardly, dotnetnuke, ffuf_shortnames, generic_ssrf, host_header, hunt, smuggler, telerik, url_manipulation | +| slow | 11 | May take a long time to complete | bucket_digitalocean, dastardly, dnsbrute_mutations, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | +| affiliates | 9 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, trickest, viewdns, zoomeye | +| email-enum | 8 | Enumerates email addresses | dehashed, dnscaa, emailformat, emails, hunterio, pgp, skymem, sslcert | +| deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | +| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | +| baddns | 2 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone | +| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | +| portscan | 2 | Discovers open ports | internetdb, portscan | +| report | 2 | Generates a report at the end of the scan | affiliates, asn | +| social-enum | 2 | Enumerates social media | httpx, social | +| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | +| subdomain-hijack | 1 | Detects hijackable subdomains | baddns | +| web-screenshots | 1 | Takes screenshots of web pages | gowitness | ## Dependencies diff --git a/docs/scanning/presets_list.md b/docs/scanning/presets_list.md index 6c30a817b..f63caefed 100644 --- a/docs/scanning/presets_list.md +++ b/docs/scanning/presets_list.md @@ -18,7 +18,7 @@ Enumerate cloud resources such as storage buckets, etc. -Modules: [53]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `baddns`, `bevigil`, `binaryedge`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_file_enum`, `bucket_firebase`, `bucket_google`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `columbus`, `crt`, `digitorus`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `fullhunt`, `github_codesearch`, `github_org`, `hackertarget`, `httpx`, `hunterio`, `internetdb`, `ipneighbor`, `leakix`, `myssl`, `oauth`, `otx`, `passivetotal`, `postman`, `rapiddns`, `securitytrails`, `shodan_dns`, `sitedossier`, `social`, `sslcert`, `subdomaincenter`, `trickest`, `urlscan`, `virustotal`, `wayback`, `zoomeye`") +Modules: [55]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `baddns`, `bevigil`, `binaryedge`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_file_enum`, `bucket_firebase`, `bucket_google`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `columbus`, `crt`, `digitorus`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `fullhunt`, `github_codesearch`, `github_org`, `hackertarget`, `httpx`, `hunterio`, `internetdb`, `ipneighbor`, `leakix`, `myssl`, `oauth`, `otx`, `passivetotal`, `postman_download`, `postman`, `rapiddns`, `securitytrails`, `securitytxt`, `shodan_dns`, `sitedossier`, `social`, `sslcert`, `subdomaincenter`, `trickest`, `urlscan`, `virustotal`, `wayback`, `zoomeye`") ## **code-enum** @@ -34,7 +34,7 @@ Enumerate Git repositories, Docker images, etc. -Modules: [10]("`code_repository`, `dockerhub`, `git`, `github_codesearch`, `github_org`, `gitlab`, `httpx`, `postman`, `social`, `trufflehog`") +Modules: [14]("`code_repository`, `docker_pull`, `dockerhub`, `git_clone`, `git`, `github_codesearch`, `github_org`, `github_workflows`, `gitlab`, `httpx`, `postman_download`, `postman`, `social`, `trufflehog`") ## **dirbust-heavy** @@ -216,7 +216,7 @@ Everything everywhere all at once -Modules: [75]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `baddns`, `badsecrets`, `bevigil`, `binaryedge`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_file_enum`, `bucket_firebase`, `bucket_google`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `code_repository`, `columbus`, `crt`, `dehashed`, `digitorus`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `dockerhub`, `emailformat`, `ffuf_shortnames`, `ffuf`, `filedownload`, `fullhunt`, `git`, `github_codesearch`, `github_org`, `gitlab`, `gowitness`, `hackertarget`, `httpx`, `hunterio`, `iis_shortnames`, `internetdb`, `ipneighbor`, `leakix`, `myssl`, `ntlm`, `oauth`, `otx`, `paramminer_cookies`, `paramminer_getparams`, `paramminer_headers`, `passivetotal`, `pgp`, `postman`, `rapiddns`, `robots`, `secretsdb`, `securitytrails`, `shodan_dns`, `sitedossier`, `skymem`, `social`, `sslcert`, `subdomaincenter`, `trickest`, `trufflehog`, `urlscan`, `virustotal`, `wappalyzer`, `wayback`, `zoomeye`") +Modules: [80]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `baddns`, `badsecrets`, `bevigil`, `binaryedge`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_file_enum`, `bucket_firebase`, `bucket_google`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `code_repository`, `columbus`, `crt`, `dehashed`, `digitorus`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `docker_pull`, `dockerhub`, `emailformat`, `ffuf_shortnames`, `ffuf`, `filedownload`, `fullhunt`, `git_clone`, `git`, `github_codesearch`, `github_org`, `github_workflows`, `gitlab`, `gowitness`, `hackertarget`, `httpx`, `hunterio`, `iis_shortnames`, `internetdb`, `ipneighbor`, `leakix`, `myssl`, `ntlm`, `oauth`, `otx`, `paramminer_cookies`, `paramminer_getparams`, `paramminer_headers`, `passivetotal`, `pgp`, `postman_download`, `postman`, `rapiddns`, `robots`, `secretsdb`, `securitytrails`, `securitytxt`, `shodan_dns`, `sitedossier`, `skymem`, `social`, `sslcert`, `subdomaincenter`, `trickest`, `trufflehog`, `urlscan`, `virustotal`, `wappalyzer`, `wayback`, `zoomeye`") ## **paramminer** @@ -299,7 +299,7 @@ Enumerate subdomains via APIs, brute-force -Modules: [46]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `bevigil`, `binaryedge`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `columbus`, `crt`, `digitorus`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `fullhunt`, `github_codesearch`, `github_org`, `hackertarget`, `httpx`, `hunterio`, `internetdb`, `ipneighbor`, `leakix`, `myssl`, `oauth`, `otx`, `passivetotal`, `postman`, `rapiddns`, `securitytrails`, `shodan_dns`, `sitedossier`, `social`, `sslcert`, `subdomaincenter`, `trickest`, `urlscan`, `virustotal`, `wayback`, `zoomeye`") +Modules: [48]("`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_zone`, `bevigil`, `binaryedge`, `builtwith`, `c99`, `censys`, `certspotter`, `chaos`, `columbus`, `crt`, `digitorus`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `fullhunt`, `github_codesearch`, `github_org`, `hackertarget`, `httpx`, `hunterio`, `internetdb`, `ipneighbor`, `leakix`, `myssl`, `oauth`, `otx`, `passivetotal`, `postman_download`, `postman`, `rapiddns`, `securitytrails`, `securitytxt`, `shodan_dns`, `sitedossier`, `social`, `sslcert`, `subdomaincenter`, `trickest`, `urlscan`, `virustotal`, `wayback`, `zoomeye`") ## **web-basic** @@ -318,7 +318,7 @@ Quick web scan -Modules: [18]("`azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_azure`, `bucket_firebase`, `bucket_google`, `ffuf_shortnames`, `filedownload`, `git`, `httpx`, `iis_shortnames`, `ntlm`, `oauth`, `robots`, `secretsdb`, `sslcert`, `wappalyzer`") +Modules: [19]("`azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_azure`, `bucket_firebase`, `bucket_google`, `ffuf_shortnames`, `filedownload`, `git`, `httpx`, `iis_shortnames`, `ntlm`, `oauth`, `robots`, `secretsdb`, `securitytxt`, `sslcert`, `wappalyzer`") ## **web-screenshots** @@ -364,7 +364,7 @@ Aggressive web scan -Modules: [29]("`ajaxpro`, `azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_firebase`, `bucket_google`, `bypass403`, `dastardly`, `dotnetnuke`, `ffuf_shortnames`, `filedownload`, `generic_ssrf`, `git`, `host_header`, `httpx`, `hunt`, `iis_shortnames`, `ntlm`, `oauth`, `robots`, `secretsdb`, `smuggler`, `sslcert`, `telerik`, `url_manipulation`, `wappalyzer`") +Modules: [30]("`ajaxpro`, `azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_azure`, `bucket_digitalocean`, `bucket_firebase`, `bucket_google`, `bypass403`, `dastardly`, `dotnetnuke`, `ffuf_shortnames`, `filedownload`, `generic_ssrf`, `git`, `host_header`, `httpx`, `hunt`, `iis_shortnames`, `ntlm`, `oauth`, `robots`, `secretsdb`, `securitytxt`, `smuggler`, `sslcert`, `telerik`, `url_manipulation`, `wappalyzer`") ## Table of Default Presets @@ -372,20 +372,20 @@ Modules: [29]("`ajaxpro`, `azure_realm`, `baddns`, `badsecrets`, `bucket_amazon` Here is a the same data, but in a table: -| Preset | Category | Description | # Modules | Modules | -|-----------------|------------|--------------------------------------------------------------------------|-------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| cloud-enum | | Enumerate cloud resources such as storage buckets, etc. | 53 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, rapiddns, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, trickest, urlscan, virustotal, wayback, zoomeye | -| code-enum | | Enumerate Git repositories, Docker images, etc. | 10 | code_repository, dockerhub, git, github_codesearch, github_org, gitlab, httpx, postman, social, trufflehog | -| dirbust-heavy | web | Recursive web directory brute-force (aggressive) | 5 | ffuf, ffuf_shortnames, httpx, iis_shortnames, wayback | -| dirbust-light | web | Basic web directory brute-force (surface-level directories only) | 4 | ffuf, ffuf_shortnames, httpx, iis_shortnames | -| dotnet-audit | web | Comprehensive scan for all IIS/.NET specific modules and module settings | 8 | ajaxpro, badsecrets, dotnetnuke, ffuf, ffuf_shortnames, httpx, iis_shortnames, telerik | -| email-enum | | Enumerate email addresses from APIs, web crawling, etc. | 7 | dehashed, dnscaa, emailformat, hunterio, pgp, skymem, sslcert | -| iis-shortnames | web | Recursively enumerate IIS shortnames | 3 | ffuf_shortnames, httpx, iis_shortnames | -| kitchen-sink | | Everything everywhere all at once | 75 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, crt, dehashed, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dockerhub, emailformat, ffuf, ffuf_shortnames, filedownload, fullhunt, git, github_codesearch, github_org, gitlab, gowitness, hackertarget, httpx, hunterio, iis_shortnames, internetdb, ipneighbor, leakix, myssl, ntlm, oauth, otx, paramminer_cookies, paramminer_getparams, paramminer_headers, passivetotal, pgp, postman, rapiddns, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, trickest, trufflehog, urlscan, virustotal, wappalyzer, wayback, zoomeye | -| paramminer | web | Discover new web parameters via brute-force | 4 | httpx, paramminer_cookies, paramminer_getparams, paramminer_headers | -| spider | | Recursive web spider | 1 | httpx | -| subdomain-enum | | Enumerate subdomains via APIs, brute-force | 46 | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, rapiddns, securitytrails, shodan_dns, sitedossier, social, sslcert, subdomaincenter, trickest, urlscan, virustotal, wayback, zoomeye | -| web-basic | | Quick web scan | 18 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, ffuf_shortnames, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer | -| web-screenshots | | Take screenshots of webpages | 3 | gowitness, httpx, social | -| web-thorough | | Aggressive web scan | 29 | ajaxpro, azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer | +| Preset | Category | Description | # Modules | Modules | +|-----------------|------------|--------------------------------------------------------------------------|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| cloud-enum | | Enumerate cloud resources such as storage buckets, etc. | 55 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, social, sslcert, subdomaincenter, trickest, urlscan, virustotal, wayback, zoomeye | +| code-enum | | Enumerate Git repositories, Docker images, etc. | 14 | code_repository, docker_pull, dockerhub, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, httpx, postman, postman_download, social, trufflehog | +| dirbust-heavy | web | Recursive web directory brute-force (aggressive) | 5 | ffuf, ffuf_shortnames, httpx, iis_shortnames, wayback | +| dirbust-light | web | Basic web directory brute-force (surface-level directories only) | 4 | ffuf, ffuf_shortnames, httpx, iis_shortnames | +| dotnet-audit | web | Comprehensive scan for all IIS/.NET specific modules and module settings | 8 | ajaxpro, badsecrets, dotnetnuke, ffuf, ffuf_shortnames, httpx, iis_shortnames, telerik | +| email-enum | | Enumerate email addresses from APIs, web crawling, etc. | 7 | dehashed, dnscaa, emailformat, hunterio, pgp, skymem, sslcert | +| iis-shortnames | web | Recursively enumerate IIS shortnames | 3 | ffuf_shortnames, httpx, iis_shortnames | +| kitchen-sink | | Everything everywhere all at once | 80 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, crt, dehashed, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, ffuf, ffuf_shortnames, filedownload, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunterio, iis_shortnames, internetdb, ipneighbor, leakix, myssl, ntlm, oauth, otx, paramminer_cookies, paramminer_getparams, paramminer_headers, passivetotal, pgp, postman, postman_download, rapiddns, robots, secretsdb, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, trickest, trufflehog, urlscan, virustotal, wappalyzer, wayback, zoomeye | +| paramminer | web | Discover new web parameters via brute-force | 4 | httpx, paramminer_cookies, paramminer_getparams, paramminer_headers | +| spider | | Recursive web spider | 1 | httpx | +| subdomain-enum | | Enumerate subdomains via APIs, brute-force | 48 | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, social, sslcert, subdomaincenter, trickest, urlscan, virustotal, wayback, zoomeye | +| web-basic | | Quick web scan | 19 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, ffuf_shortnames, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, securitytxt, sslcert, wappalyzer | +| web-screenshots | | Take screenshots of webpages | 3 | gowitness, httpx, social | +| web-thorough | | Aggressive web scan | 30 | ajaxpro, azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, ntlm, oauth, robots, secretsdb, securitytxt, smuggler, sslcert, telerik, url_manipulation, wappalyzer | From e478c8ca3fdf065a95f4deb1d5833b0882df5983 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 27 Sep 2024 13:56:19 -0400 Subject: [PATCH 123/254] update docs only on non-published branches --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5bbd3bf4e..4ec67d070 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,7 +58,7 @@ jobs: update_docs: needs: test runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable') + if: github.event_name == 'push' && (github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/stable') steps: - uses: actions/checkout@v3 with: From a076f7c91281474e910cd17b2cca2293c4044b1c Mon Sep 17 00:00:00 2001 From: GitHub Date: Sun, 29 Sep 2024 00:28:03 +0000 Subject: [PATCH 124/254] Update nuclei --- bbot/modules/deadly/nuclei.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 54bea9b82..506db6f0e 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -15,7 +15,7 @@ class nuclei(BaseModule): } options = { - "version": "3.3.3", + "version": "3.3.4", "tags": "", "templates": "", "severity": "", From 71516f8e70229a9eae497fe4e61ebb2ae3e3e905 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 29 Sep 2024 14:16:32 -0400 Subject: [PATCH 125/254] bump baddns version --- bbot/modules/baddns.py | 2 +- bbot/modules/baddns_direct.py | 2 +- bbot/modules/baddns_zone.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index d8ce14e46..59c7bdc14 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -22,7 +22,7 @@ class baddns(BaseModule): "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", } module_threads = 8 - deps_pip = ["baddns~=1.1.839"] + deps_pip = ["baddns~=1.1.846"] def select_modules(self): selected_submodules = [] diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index e575d2bd4..c5212d508 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -19,7 +19,7 @@ class baddns_direct(BaseModule): "custom_nameservers": "Force BadDNS to use a list of custom nameservers", } module_threads = 8 - deps_pip = ["baddns~=1.1.815"] + deps_pip = ["baddns~=1.1.846"] scope_distance_modifier = 1 diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index 16eb7c1ae..ee92aeec4 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -16,7 +16,7 @@ class baddns_zone(baddns_module): "only_high_confidence": "Do not emit low-confidence or generic detections", } module_threads = 8 - deps_pip = ["baddns~=1.1.839"] + deps_pip = ["baddns~=1.1.846"] def set_modules(self): self.enabled_submodules = ["NSEC", "zonetransfer"] From 977be2732932622fccda7afd22a6279235386e57 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 29 Sep 2024 14:21:18 -0400 Subject: [PATCH 126/254] baddns presets adjust --- bbot/presets/baddns-thorough.yml | 2 +- bbot/presets/kitchen-sink.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/presets/baddns-thorough.yml b/bbot/presets/baddns-thorough.yml index e2299b841..8afeebd3d 100644 --- a/bbot/presets/baddns-thorough.yml +++ b/bbot/presets/baddns-thorough.yml @@ -1,4 +1,4 @@ -description: Run all baddns and baddns_zone submodules. +description: Run all baddns modules and submodules. modules: diff --git a/bbot/presets/kitchen-sink.yml b/bbot/presets/kitchen-sink.yml index 624d597d2..43057bf44 100644 --- a/bbot/presets/kitchen-sink.yml +++ b/bbot/presets/kitchen-sink.yml @@ -10,6 +10,7 @@ include: - paramminer - dirbust-light - web-screenshots + - baddns-thorough config: modules: From baefedb8fc82efbfb8b087d53cd4d3767d89cf25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 05:03:20 +0000 Subject: [PATCH 127/254] Bump mkdocs-material from 9.5.36 to 9.5.39 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.36 to 9.5.39. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.36...9.5.39) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 27b7cc72c..f3238f135 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1243,13 +1243,13 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.36" +version = "9.5.39" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.36-py3-none-any.whl", hash = "sha256:36734c1fd9404bea74236242ba3359b267fc930c7233b9fd086b0898825d0ac9"}, - {file = "mkdocs_material-9.5.36.tar.gz", hash = "sha256:140456f761320f72b399effc073fa3f8aac744c77b0970797c201cae2f6c967f"}, + {file = "mkdocs_material-9.5.39-py3-none-any.whl", hash = "sha256:0f2f68c8db89523cb4a59705cd01b4acd62b2f71218ccb67e1e004e560410d2b"}, + {file = "mkdocs_material-9.5.39.tar.gz", hash = "sha256:25faa06142afa38549d2b781d475a86fb61de93189f532b88e69bf11e5e5c3be"}, ] [package.dependencies] From 64241b8091d8fbf7b7b698518dcdcb075815a376 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 30 Sep 2024 09:58:00 +0100 Subject: [PATCH 128/254] Added in scope validator using the helper & a workspace that is out-of-scope in the test that will fail validation --- bbot/modules/postman_download.py | 57 ++---- .../test_module_postman_download.py | 174 ++++++++++++++++-- 2 files changed, 173 insertions(+), 58 deletions(-) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 0d3df214c..fc6bed1fb 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -49,7 +49,7 @@ async def handle_event(self, event): workspace = data["workspace"] environments = data["environments"] collections = data["collections"] - in_scope = self.validate_workspace(workspace, environments, collections) + in_scope = await self.validate_workspace(workspace, environments, collections) if in_scope: workspace_path = self.save_workspace(workspace, environments, collections) if workspace_path: @@ -182,55 +182,20 @@ async def get_collection(self, collection_id): collection = json.get("collection", {}) return collection - def validate_string(self, v): - if ( - isinstance(v, str) - and ( - self.helpers.is_dns_name(v, include_local=False) or self.helpers.is_url(v) or self.helpers.is_email(v) - ) - and self.scan.in_scope(v) - ): - return True - return False - - def unpack_and_validate(self, data, workspace_name): - for k, v in data.items(): - if isinstance(v, dict): - if self.unpack_and_validate(v, workspace_name): - self.verbose( - f'Found in-scope key "{k}": "{v}" for workspace {workspace_name}, it appears to be in-scope' - ) + async def validate_workspace(self, workspace, environments, collections): + name = workspace.get("name", "") + for dict in [workspace, *environments, *collections]: + for email in await self.helpers.re.extract_emails(str(dict)): + if self.scan.in_scope(email): + self.verbose(f'Found in-scope email: "{email}" for workspace {name}, it appears to be in-scope') return True - elif isinstance(v, list): - for item in v: - if isinstance(item, dict): - if self.unpack_and_validate(item, workspace_name): - self.verbose( - f'Found in-scope key "{k}": "{item}" for workspace {workspace_name}, it appears to be in-scope' - ) - return True - elif self.validate_string(item): + for host in await self.scan.extract_in_scope_hostnames(str(dict)): + if self.scan.in_scope(host): + if self.scan.in_scope(host): self.verbose( - f'Found in-scope key "{k}": "{item}" for workspace {workspace_name}, it appears to be in-scope' + f'Found in-scope hostname: "{host}" for workspace {name}, it appears to be in-scope' ) return True - elif self.validate_string(v): - self.verbose( - f'Found in-scope key "{k}": "{v}" for workspace {workspace_name}, it appears to be in-scope' - ) - return True - return False - - def validate_workspace(self, workspace, environments, collections): - name = workspace.get("name", "") - if self.unpack_and_validate(workspace, name): - return True - for environment in environments: - if self.unpack_and_validate(environment, name): - return True - for collection in collections: - if self.unpack_and_validate(collection, name): - return True return False def save_workspace(self, workspace, environments, collections): diff --git a/bbot/test/test_step_2/module_tests/test_module_postman_download.py b/bbot/test/test_step_2/module_tests/test_module_postman_download.py index 8ac0175d3..5c731eef2 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman_download.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman_download.py @@ -97,14 +97,65 @@ async def setup_after_prep(self, module_test): "documentType": "workspace", }, "highlight": {"summary": "BLS BBOT api test."}, - } + }, + { + "score": 611.41156, + "normalizedScore": 23, + "document": { + "watcherCount": 6, + "apiCount": 0, + "forkCount": 0, + "isblacklisted": "false", + "createdAt": "2021-06-15T14:03:51", + "publishertype": "team", + "publisherHandle": "testteam", + "id": "11498add-357d-4bc5-a008-0a2d44fb8829", + "slug": "testing-bbot-api", + "updatedAt": "2024-07-30T11:00:35", + "entityType": "workspace", + "visibilityStatus": "public", + "forkcount": "0", + "tags": [], + "createdat": "2021-06-15T14:03:51", + "forkLabel": "", + "publisherName": "testteam", + "name": "Test BlackLanternSecurity API Team Workspace", + "dependencyCount": 7, + "collectionCount": 6, + "warehouse__updated_at": "2024-07-30 11:00:00", + "privateNetworkFolders": [], + "isPublisherVerified": False, + "publisherType": "team", + "curatedInList": [], + "creatorId": "6900157", + "description": "", + "forklabel": "", + "publisherId": "299401", + "publisherLogo": "", + "popularity": 5, + "isPublic": True, + "categories": [], + "universaltags": "", + "views": 5788, + "summary": "Private test of BBOTs public API", + "memberCount": 2, + "isBlacklisted": False, + "publisherid": "299401", + "isPrivateNetworkEntity": False, + "isDomainNonTrivial": True, + "privateNetworkMeta": "", + "updatedat": "2021-10-20T16:19:29", + "documentType": "workspace", + }, + "highlight": {"summary": "Private test of BBOTs Public API"}, + }, ], "meta": { "queryText": "blacklanternsecurity", "total": { "collection": 0, "request": 0, - "workspace": 1, + "workspace": 2, "api": 0, "team": 0, "user": 0, @@ -113,7 +164,7 @@ async def setup_after_prep(self, module_test): "privateNetworkFolder": 0, }, "state": "AQ4", - "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None}, + "spellCorrection": {"count": {"all": 2, "workspace": 2}, "correctedQueryText": None}, "featureFlags": { "enabledPublicResultCuration": True, "boostByPopularity": True, @@ -153,6 +204,36 @@ async def setup_after_prep(self, module_test): ], }, ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=testteam&slug=testing-bbot-api"}', + json={ + "meta": {"model": "workspace", "action": "find", "nextCursor": ""}, + "data": [ + { + "id": "a4dfe981-2593-4f0b-b4c3-5145e8640f7d", + "name": "Test BlackLanternSecurity API Team Workspace", + "description": None, + "summary": "Private test of BBOTs public API", + "createdBy": "299401", + "updatedBy": "299401", + "team": None, + "createdAt": "2021-10-20T16:19:29", + "updatedAt": "2021-10-20T16:19:29", + "visibilityStatus": "public", + "profileInfo": { + "slug": "bbot-public", + "profileType": "team", + "profileId": "000000", + "publicHandle": "https://www.postman.com/testteam", + "publicImageURL": "", + "publicName": "testteam", + "isVerified": False, + }, + } + ], + }, + ) module_test.httpx_mock.add_response( url="https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", json={ @@ -184,6 +265,31 @@ async def setup_after_prep(self, module_test): } }, ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/workspaces/a4dfe981-2593-4f0b-b4c3-5145e8640f7d", + json={ + "workspace": { + "id": "a4dfe981-2593-4f0b-b4c3-5145e8640f7d", + "name": "Test BlackLanternSecurity API Team Workspace", + "type": "personal", + "description": None, + "visibility": "public", + "createdBy": "00000000", + "updatedBy": "00000000", + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-17T08:57:16.000Z", + "collections": [ + { + "id": "f46bebfd-420a-4adf-97d1-6fb5a02cf7fc", + "name": "BBOT Public", + "uid": "10197090-f46bebfd-420a-4adf-97d1-6fb5a02cf7fc", + }, + ], + "environments": [], + "apis": [], + } + }, + ) module_test.httpx_mock.add_response( url="https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals", json={ @@ -206,6 +312,22 @@ async def setup_after_prep(self, module_test): }, }, ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/workspace/a4dfe981-2593-4f0b-b4c3-5145e8640f7d/globals", + json={ + "model_id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "meta": {"model": "globals", "action": "find"}, + "data": { + "workspace": "a4dfe981-2593-4f0b-b4c3-5145e8640f7d", + "lastUpdatedBy": "00000000", + "lastRevision": 1637239113000, + "id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "values": [], + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-18T12:38:33.000Z", + }, + }, + ) module_test.httpx_mock.add_response( url="https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", json={ @@ -261,17 +383,45 @@ async def setup_after_prep(self, module_test): } }, ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/collections/10197090-f46bebfd-420a-4adf-97d1-6fb5a02cf7fc", + json={ + "collection": { + "info": { + "_postman_id": "f46bebfd-420a-4adf-97d1-6fb5a02cf7fc", + "name": "BBOT Public", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "updatedAt": "2021-11-17T07:13:16.000Z", + "createdAt": "2021-11-17T07:13:15.000Z", + "lastUpdatedBy": "00000000", + "uid": "00000000-f46bebfd-420a-4adf-97d1-6fb5a02cf7fc", + }, + "item": [ + { + "name": "Out of Scope API request", + "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + "protocolProfileBehavior": {"disableBodyPruning": True}, + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": '{"username": "test", "password": "Test"}', + }, + "url": {"raw": "https://www.outofscope.com", "host": ["www.outofscope.com"]}, + "description": "", + }, + "response": [], + "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + }, + ], + } + }, + ) def check(self, module_test, events): - assert 1 == len( - [ - e - for e in events - if e.type == "CODE_REPOSITORY" - and "postman" in e.tags - and e.data["url"] == "https://www.postman.com/blacklanternsecurity/bbot-public" - and e.scope_distance == 1 - ] + assert 2 == len( + [e for e in events if e.type == "CODE_REPOSITORY" and "postman" in e.tags and e.scope_distance == 1] ), "Failed to find blacklanternsecurity postman workspace" assert 1 == len( [ From b00c7d01d1981ff1ae27c71594f4a561c5de1bf8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 30 Sep 2024 09:03:07 -0400 Subject: [PATCH 129/254] fix docs autopublish --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 732c9c44e..0d33c2931 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -83,7 +83,6 @@ jobs: author_email: info@blacklanternsecurity.com message: "Refresh module docs" publish_docs: - needs: update_docs runs-on: ubuntu-latest if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') steps: From 3cd739cefe2db63fcf68930c2ccd440e3b6c2746 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 30 Sep 2024 14:24:38 +0100 Subject: [PATCH 130/254] Simplify entire workspace into single string --- bbot/modules/postman_download.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index fc6bed1fb..a2a6ded31 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -184,18 +184,16 @@ async def get_collection(self, collection_id): async def validate_workspace(self, workspace, environments, collections): name = workspace.get("name", "") - for dict in [workspace, *environments, *collections]: - for email in await self.helpers.re.extract_emails(str(dict)): - if self.scan.in_scope(email): - self.verbose(f'Found in-scope email: "{email}" for workspace {name}, it appears to be in-scope') - return True - for host in await self.scan.extract_in_scope_hostnames(str(dict)): + full_wks = str([workspace, environments, collections]) + for email in await self.helpers.re.extract_emails(full_wks): + if self.scan.in_scope(email): + self.verbose(f'Found in-scope email: "{email}" for workspace {name}, it appears to be in-scope') + return True + for host in await self.scan.extract_in_scope_hostnames(full_wks): + if self.scan.in_scope(host): if self.scan.in_scope(host): - if self.scan.in_scope(host): - self.verbose( - f'Found in-scope hostname: "{host}" for workspace {name}, it appears to be in-scope' - ) - return True + self.verbose(f'Found in-scope hostname: "{host}" for workspace {name}, it appears to be in-scope') + return True return False def save_workspace(self, workspace, environments, collections): From 92a68f2460732851b136a5eda637f24fbf8efb30 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 30 Sep 2024 09:50:27 -0400 Subject: [PATCH 131/254] extract hostnames from whitelist only, not target --- bbot/modules/internal/excavate.py | 1 + bbot/scanner/scanner.py | 12 +++++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index f09931dc7..d2c65414e 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -828,6 +828,7 @@ async def setup(self): yara.set_config(max_match_data=yara_max_match_data) yara_rules_combined = "\n".join(self.yara_rules_dict.values()) try: + self.info(f"Compiling {len(self.yara_rules_dict):,} YARA rules") self.yara_rules = yara.compile(source=yara_rules_combined) except yara.SyntaxError as e: self.debug(yara_rules_combined) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 9a7860d60..19ea7106a 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -1004,15 +1004,13 @@ def dns_strings(self): A list of DNS hostname strings generated from the scan target """ if self._dns_strings is None: - dns_targets = set(t.host for t in self.target if t.host and isinstance(t.host, str)) dns_whitelist = set(t.host for t in self.whitelist if t.host and isinstance(t.host, str)) - dns_targets.update(dns_whitelist) - dns_targets = sorted(dns_targets, key=len) - dns_targets_set = set() + dns_whitelist = sorted(dns_whitelist, key=len) + dns_whitelist_set = set() dns_strings = [] - for t in dns_targets: - if not any(x in dns_targets_set for x in self.helpers.domain_parents(t, include_self=True)): - dns_targets_set.add(t) + for t in dns_whitelist: + if not any(x in dns_whitelist_set for x in self.helpers.domain_parents(t, include_self=True)): + dns_whitelist_set.add(t) dns_strings.append(t) self._dns_strings = dns_strings return self._dns_strings From 78c2d1b6edfc3d68849db1a4038c52a1308d1b75 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 30 Sep 2024 10:08:17 -0400 Subject: [PATCH 132/254] needs test --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d33c2931..02cb65923 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -83,6 +83,7 @@ jobs: author_email: info@blacklanternsecurity.com message: "Refresh module docs" publish_docs: + needs: test runs-on: ubuntu-latest if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') steps: From 525f63b863b044414beb51c38cbbefbc33cfb71d Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 30 Sep 2024 10:27:13 -0400 Subject: [PATCH 133/254] revise releases page --- docs/release_history.md | 380 +++------------------------------------- 1 file changed, 28 insertions(+), 352 deletions(-) diff --git a/docs/release_history.md b/docs/release_history.md index 9de5ef6ca..f93f7d785 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,365 +1,41 @@ -## v1.1.7 -May 15th, 2024 +### 2.1.0 +- https://github.com/blacklanternsecurity/bbot/pull/1724 -### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/1037 -- https://github.com/blacklanternsecurity/bbot/pull/1122 -- https://github.com/blacklanternsecurity/bbot/pull/1176 -- https://github.com/blacklanternsecurity/bbot/pull/1164 -- https://github.com/blacklanternsecurity/bbot/pull/1169 -- https://github.com/blacklanternsecurity/bbot/pull/1175 -- https://github.com/blacklanternsecurity/bbot/pull/1209 -- https://github.com/blacklanternsecurity/bbot/pull/1335 -- https://github.com/blacklanternsecurity/bbot/pull/1380 +### 2.0.1 - Aug 29, 2024 +- https://github.com/blacklanternsecurity/bbot/pull/1650 -### Improvements -- https://github.com/blacklanternsecurity/bbot/pull/1132 -- https://github.com/blacklanternsecurity/bbot/pull/1156 -- https://github.com/blacklanternsecurity/bbot/pull/1160 -- https://github.com/blacklanternsecurity/bbot/pull/1162 -- https://github.com/blacklanternsecurity/bbot/pull/1165 -- https://github.com/blacklanternsecurity/bbot/pull/1179 -- https://github.com/blacklanternsecurity/bbot/pull/1182 -- https://github.com/blacklanternsecurity/bbot/pull/1185 -- https://github.com/blacklanternsecurity/bbot/pull/1186 -- https://github.com/blacklanternsecurity/bbot/pull/1197 -- https://github.com/blacklanternsecurity/bbot/pull/1198 -- https://github.com/blacklanternsecurity/bbot/pull/1205 -- https://github.com/blacklanternsecurity/bbot/pull/1217 -- https://github.com/blacklanternsecurity/bbot/pull/1233 -- https://github.com/blacklanternsecurity/bbot/pull/1283 -- https://github.com/blacklanternsecurity/bbot/pull/1288 -- https://github.com/blacklanternsecurity/bbot/pull/1296 -- https://github.com/blacklanternsecurity/bbot/pull/1306 -- https://github.com/blacklanternsecurity/bbot/pull/1313 -- https://github.com/blacklanternsecurity/bbot/pull/1349 -- https://github.com/blacklanternsecurity/bbot/pull/1343 +### 2.0.0 - Aug 9, 2024 +- https://github.com/blacklanternsecurity/bbot/pull/1424 -### Bugfixes -- https://github.com/blacklanternsecurity/bbot/pull/1136 -- https://github.com/blacklanternsecurity/bbot/pull/1140 -- https://github.com/blacklanternsecurity/bbot/pull/1152 -- https://github.com/blacklanternsecurity/bbot/pull/1154 -- https://github.com/blacklanternsecurity/bbot/pull/1181 -- https://github.com/blacklanternsecurity/bbot/pull/1247 -- https://github.com/blacklanternsecurity/bbot/pull/1249 -- https://github.com/blacklanternsecurity/bbot/pull/1273 -- https://github.com/blacklanternsecurity/bbot/pull/1277 -- https://github.com/blacklanternsecurity/bbot/pull/1287 -- https://github.com/blacklanternsecurity/bbot/pull/1298 -- https://github.com/blacklanternsecurity/bbot/pull/1308 -- https://github.com/blacklanternsecurity/bbot/pull/1336 -- https://github.com/blacklanternsecurity/bbot/pull/1348 +### 1.1.8 - May 29, 2024 +- https://github.com/blacklanternsecurity/bbot/pull/1382 -## v1.1.6 -February 21, 2024 +### 1.1.7 - May 15, 2024 +- https://github.com/blacklanternsecurity/bbot/pull/1119 -### Improvements -- https://github.com/blacklanternsecurity/bbot/pull/1001 -- https://github.com/blacklanternsecurity/bbot/pull/1006 -- https://github.com/blacklanternsecurity/bbot/pull/1010 -- https://github.com/blacklanternsecurity/bbot/pull/1013 -- https://github.com/blacklanternsecurity/bbot/pull/1014 -- https://github.com/blacklanternsecurity/bbot/pull/1015 -- https://github.com/blacklanternsecurity/bbot/pull/1032 -- https://github.com/blacklanternsecurity/bbot/pull/1043 -- https://github.com/blacklanternsecurity/bbot/pull/1047 -- https://github.com/blacklanternsecurity/bbot/pull/1048 -- https://github.com/blacklanternsecurity/bbot/pull/1049 -- https://github.com/blacklanternsecurity/bbot/pull/1051 -- https://github.com/blacklanternsecurity/bbot/pull/1065 -- https://github.com/blacklanternsecurity/bbot/pull/1070 -- https://github.com/blacklanternsecurity/bbot/pull/1076 -- https://github.com/blacklanternsecurity/bbot/pull/1077 -- https://github.com/blacklanternsecurity/bbot/pull/1095 -- https://github.com/blacklanternsecurity/bbot/pull/1101 -- https://github.com/blacklanternsecurity/bbot/pull/1103 +### 1.1.6 - Feb 21, 2024 +- https://github.com/blacklanternsecurity/bbot/pull/1002 -### Bigfixes -- https://github.com/blacklanternsecurity/bbot/pull/1005 -- https://github.com/blacklanternsecurity/bbot/pull/1022 -- https://github.com/blacklanternsecurity/bbot/pull/1030 -- https://github.com/blacklanternsecurity/bbot/pull/1033 -- https://github.com/blacklanternsecurity/bbot/pull/1034 -- https://github.com/blacklanternsecurity/bbot/pull/1042 -- https://github.com/blacklanternsecurity/bbot/pull/1066 -- https://github.com/blacklanternsecurity/bbot/pull/1067 -- https://github.com/blacklanternsecurity/bbot/pull/1073 -- https://github.com/blacklanternsecurity/bbot/pull/1086 -- https://github.com/blacklanternsecurity/bbot/pull/1089 -- https://github.com/blacklanternsecurity/bbot/pull/1094 -- https://github.com/blacklanternsecurity/bbot/pull/1098 +### 1.1.5 - Jan 15, 2024 +- https://github.com/blacklanternsecurity/bbot/pull/996 -### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/1072 -- https://github.com/blacklanternsecurity/bbot/pull/1091 +### 1.1.4 - Jan 11, 2024 +- https://github.com/blacklanternsecurity/bbot/pull/837 +### 1.1.3 - Nov 4, 2023 +- https://github.com/blacklanternsecurity/bbot/pull/823 -## v1.1.5 -January 29, 2024 +### 1.1.2 - Nov 3, 2023 +- https://github.com/blacklanternsecurity/bbot/pull/777 -## Improvements -- https://github.com/blacklanternsecurity/bbot/pull/1001 -- https://github.com/blacklanternsecurity/bbot/pull/1006 -- https://github.com/blacklanternsecurity/bbot/pull/1010 -- https://github.com/blacklanternsecurity/bbot/pull/1013 -- https://github.com/blacklanternsecurity/bbot/pull/1014 -- https://github.com/blacklanternsecurity/bbot/pull/1015 -- https://github.com/blacklanternsecurity/bbot/pull/1032 -- https://github.com/blacklanternsecurity/bbot/pull/1040 +### 1.1.1 - Oct 11, 2023 +- https://github.com/blacklanternsecurity/bbot/pull/668 -## Bugfixes -- https://github.com/blacklanternsecurity/bbot/pull/1005 -- https://github.com/blacklanternsecurity/bbot/pull/1022 -- https://github.com/blacklanternsecurity/bbot/pull/1030 -- https://github.com/blacklanternsecurity/bbot/pull/1033 -- https://github.com/blacklanternsecurity/bbot/pull/1034 +### 1.1.0 - Aug 4, 2023 +- https://github.com/blacklanternsecurity/bbot/pull/598 +### 1.0.5 - Mar 10, 2023 +- https://github.com/blacklanternsecurity/bbot/pull/352 -## v1.1.4 -January 11, 2024 - -### Improvements -- https://github.com/blacklanternsecurity/bbot/pull/835 -- https://github.com/blacklanternsecurity/bbot/pull/833 -- https://github.com/blacklanternsecurity/bbot/pull/836 -- https://github.com/blacklanternsecurity/bbot/pull/816 -- https://github.com/blacklanternsecurity/bbot/pull/772 -- https://github.com/blacklanternsecurity/bbot/pull/841 -- https://github.com/blacklanternsecurity/bbot/pull/842 -- https://github.com/blacklanternsecurity/bbot/pull/843 -- https://github.com/blacklanternsecurity/bbot/pull/846 -- https://github.com/blacklanternsecurity/bbot/pull/850 -- https://github.com/blacklanternsecurity/bbot/pull/856 -- https://github.com/blacklanternsecurity/bbot/pull/857 -- https://github.com/blacklanternsecurity/bbot/pull/859 -- https://github.com/blacklanternsecurity/bbot/pull/870 -- https://github.com/blacklanternsecurity/bbot/pull/879 -- https://github.com/blacklanternsecurity/bbot/pull/875 -- https://github.com/blacklanternsecurity/bbot/pull/889 -- https://github.com/blacklanternsecurity/bbot/pull/918 -- https://github.com/blacklanternsecurity/bbot/pull/919 -- https://github.com/blacklanternsecurity/bbot/pull/937 -- https://github.com/blacklanternsecurity/bbot/pull/938 -- https://github.com/blacklanternsecurity/bbot/pull/945 -- https://github.com/blacklanternsecurity/bbot/pull/949 -- https://github.com/blacklanternsecurity/bbot/pull/957 -- https://github.com/blacklanternsecurity/bbot/pull/963 -- https://github.com/blacklanternsecurity/bbot/pull/968 -- https://github.com/blacklanternsecurity/bbot/pull/980 -- https://github.com/blacklanternsecurity/bbot/pull/979 -- https://github.com/blacklanternsecurity/bbot/pull/978 -- https://github.com/blacklanternsecurity/bbot/pull/970 -- https://github.com/blacklanternsecurity/bbot/pull/969 -- https://github.com/blacklanternsecurity/bbot/pull/980 -- https://github.com/blacklanternsecurity/bbot/pull/983 -- https://github.com/blacklanternsecurity/bbot/pull/984 -- https://github.com/blacklanternsecurity/bbot/pull/985 -- https://github.com/blacklanternsecurity/bbot/pull/986 -- https://github.com/blacklanternsecurity/bbot/pull/971 -- https://github.com/blacklanternsecurity/bbot/pull/987 -- https://github.com/blacklanternsecurity/bbot/pull/988 -- https://github.com/blacklanternsecurity/bbot/pull/989 -- https://github.com/blacklanternsecurity/bbot/pull/990 -- https://github.com/blacklanternsecurity/bbot/pull/991 - -### Bugfixes -- https://github.com/blacklanternsecurity/bbot/pull/832 -- https://github.com/blacklanternsecurity/bbot/pull/831 -- https://github.com/blacklanternsecurity/bbot/pull/845 -- https://github.com/blacklanternsecurity/bbot/pull/855 -- https://github.com/blacklanternsecurity/bbot/pull/862 -- https://github.com/blacklanternsecurity/bbot/pull/871 -- https://github.com/blacklanternsecurity/bbot/pull/873 -- https://github.com/blacklanternsecurity/bbot/pull/927 -- https://github.com/blacklanternsecurity/bbot/pull/935 -- https://github.com/blacklanternsecurity/bbot/pull/936 -- https://github.com/blacklanternsecurity/bbot/pull/939 -- https://github.com/blacklanternsecurity/bbot/pull/943 -- https://github.com/blacklanternsecurity/bbot/pull/946 -- https://github.com/blacklanternsecurity/bbot/pull/955 -- https://github.com/blacklanternsecurity/bbot/pull/956 -- https://github.com/blacklanternsecurity/bbot/pull/959 -- https://github.com/blacklanternsecurity/bbot/pull/965 -- https://github.com/blacklanternsecurity/bbot/pull/973 - -### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/806 -- https://github.com/blacklanternsecurity/bbot/pull/896 -- https://github.com/blacklanternsecurity/bbot/pull/909 -- https://github.com/blacklanternsecurity/bbot/pull/902 -- https://github.com/blacklanternsecurity/bbot/pull/911 -- https://github.com/blacklanternsecurity/bbot/pull/923 -- https://github.com/blacklanternsecurity/bbot/pull/929 - - -## v1.1.2 -October 24, 2023 - -### Improvements -- https://github.com/blacklanternsecurity/bbot/pull/776 -- https://github.com/blacklanternsecurity/bbot/pull/783 -- https://github.com/blacklanternsecurity/bbot/pull/790 -- https://github.com/blacklanternsecurity/bbot/pull/797 -- https://github.com/blacklanternsecurity/bbot/pull/798 -- https://github.com/blacklanternsecurity/bbot/pull/799 -- https://github.com/blacklanternsecurity/bbot/pull/801 - -### Bugfixes -- https://github.com/blacklanternsecurity/bbot/pull/780 -- https://github.com/blacklanternsecurity/bbot/pull/787 -- https://github.com/blacklanternsecurity/bbot/pull/788 -- https://github.com/blacklanternsecurity/bbot/pull/791 - -### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/774 - -## v1.1.1 -October 11, 2023 - -Includes webhook output modules - Discord, Slack, and Teams! - -![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/72e3e940-a41a-4c7a-952e-49a1d7cae526) - -### Improvements -- https://github.com/blacklanternsecurity/bbot/pull/677 -- https://github.com/blacklanternsecurity/bbot/pull/674 -- https://github.com/blacklanternsecurity/bbot/pull/683 -- https://github.com/blacklanternsecurity/bbot/pull/740 -- https://github.com/blacklanternsecurity/bbot/pull/743 -- https://github.com/blacklanternsecurity/bbot/pull/748 -- https://github.com/blacklanternsecurity/bbot/pull/749 -- https://github.com/blacklanternsecurity/bbot/pull/751 -- https://github.com/blacklanternsecurity/bbot/pull/692 - -### Bugfixes -- https://github.com/blacklanternsecurity/bbot/pull/691 -- https://github.com/blacklanternsecurity/bbot/pull/684 -- https://github.com/blacklanternsecurity/bbot/pull/669 -- https://github.com/blacklanternsecurity/bbot/pull/664 -- https://github.com/blacklanternsecurity/bbot/pull/737 -- https://github.com/blacklanternsecurity/bbot/pull/741 -- https://github.com/blacklanternsecurity/bbot/pull/744 -- https://github.com/blacklanternsecurity/bbot/issues/760 -- https://github.com/blacklanternsecurity/bbot/issues/759 -- https://github.com/blacklanternsecurity/bbot/issues/758 -- https://github.com/blacklanternsecurity/bbot/pull/764 -- https://github.com/blacklanternsecurity/bbot/pull/773 - -### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/689 -- https://github.com/blacklanternsecurity/bbot/pull/665 -- https://github.com/blacklanternsecurity/bbot/pull/663 - -## v1.1.0 -August 4, 2023 - -**New Features**: - -- Complete Asyncification -- Documentation + auto-updating pipelines -- Ability to list flags and their descriptions with `-lf` -- Fine-grained rate-limiting for HTTP and DNS - -**Improvements / Fixes**: - -- Better tests (one for each individual module, 91% test coverage) -- New and improved paramminer modules -- Misc bugfixes - -**New Modules**: - -- Git (detects exposed .git folder on websites) -- [Subdomain Center](https://www.subdomain.center/) (subdomain enumeration) -- [Columbus API](https://columbus.elmasy.com/) (subdomain enumeration) -- MySSL (subdomain enumeration) -- Sitedossier (subdomain enumeration) -- Digitorus (subdomain enumeration) -- Nmap (port scanner, more reliable than naabu) - - Naabu has been removed due to reliability issues -- NSEC (DNSSEC zone-walking for subdomain enumeration) -- OAUTH (Enumerates OAUTH / OpenID-Connect, detects sprayable endpoints) -- Azure Realm (Detects Managed/Federated Azure Tenants) -- Subdomains output module - - -## v1.0.5 -March 10, 2023 - -**New Modules**: - -- [Badsecrets](https://github.com/blacklanternsecurity/badsecrets) ([blacklist3r](https://github.com/NotSoSecure/Blacklist3r) but better!) -- Subdomain Hijacking (uses [can-i-take-over-xyz](https://github.com/EdOverflow/can-i-take-over-xyz)) -- [WafW00f](https://github.com/EnableSecurity/wafw00f) -- [Fingerprintx](https://github.com/praetorian-inc/fingerprintx) -- [Masscan](https://github.com/robertdavidgraham/masscan) -- Robots.txt -- Web Report -- IIS shortnames (Pure Python rewrite) - -**New Features**: - -- Automatic tagging of cloud resources (with [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck)) -- Significant performance increases -- Bug fixes -- Better tests + code coverage -- Support for punycode (non-ascii) domains -- Better support for non-64-bit systems -- Enter key now toggles verbosity during scan - - -## v1.0.4 -December 15, 2022 - -**New Modules**: - -- Storage buckets: - - Azure - - GCP - - AWS - - DigitalOcean -- ipstack (geolocation) -- BeVigil -- ASN (rewrite) - -**New Features**: - -- Colored vulnerabilities on CLI -- Log full nuclei output -- Various bugfixes -- Better handling of: - - DNS wildcards - - Infinite DNS-record chains - - Infinite HTTP redirects -- Improved module tests - -## v1.0.3 -October 12, 2022 - -**Changes**: - -- Tag URL events with their corresponding IP address -- Automatic docker hub publishing -- Added `retries` option for httpx module -- Added `asset_inventory` output module -- Improvements to nuclei module -- Avoid unnecessary failed sudo attempts during dependency install -- Improved Python API -- Added AnubisDB module -- Various bugfixes -- Add examples to `--help` output -- Reduce annoying warnings on free API modules -- Update iis_shortnames .jar dependency -- Updated documentation to explain targets, whitelists, blacklists -- Added help for module-specific options -- Added warning if unable to validate public DNS servers (for massdns) -- Various performance optimizations -- Various bugfixes -- Fix Pypi auto-publishing -- Added bug report template -- Added examples in README -- Improved wildcard detection -- Added DNS retry functionality -- Improved excavate hostname extraction -- Added command-line option for installing all dependencies -- Improved gowitness dependency install, improved tests +### 1.0.5 - Mar 10, 2023 +- https://github.com/blacklanternsecurity/bbot/pull/352 \ No newline at end of file From 8d9479be1ddf315fb99b16fc3e0316bd3504e18a Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 30 Sep 2024 17:03:17 +0100 Subject: [PATCH 134/254] As in-scope hostnames only are returned change to if statement so the whole dictionary is displayed --- bbot/modules/postman_download.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index a2a6ded31..32182e847 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -185,15 +185,12 @@ async def get_collection(self, collection_id): async def validate_workspace(self, workspace, environments, collections): name = workspace.get("name", "") full_wks = str([workspace, environments, collections]) - for email in await self.helpers.re.extract_emails(full_wks): - if self.scan.in_scope(email): - self.verbose(f'Found in-scope email: "{email}" for workspace {name}, it appears to be in-scope') - return True - for host in await self.scan.extract_in_scope_hostnames(full_wks): - if self.scan.in_scope(host): - if self.scan.in_scope(host): - self.verbose(f'Found in-scope hostname: "{host}" for workspace {name}, it appears to be in-scope') - return True + in_scope_hosts = await self.scan.extract_in_scope_hostnames(full_wks) + if in_scope_hosts: + self.verbose( + f'Found in-scope hostname(s): "{in_scope_hosts}" in workspace {name}, it appears to be in-scope' + ) + return True return False def save_workspace(self, workspace, environments, collections): From 2b935e5b9d42c45c0c1aed384d6db31d1652a349 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 30 Sep 2024 14:29:41 -0400 Subject: [PATCH 135/254] adding cleanup calls --- bbot/modules/baddns.py | 8 ++++++-- bbot/modules/baddns_direct.py | 4 +++- bbot/modules/baddns_zone.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 59c7bdc14..4b9f977c2 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -22,7 +22,7 @@ class baddns(BaseModule): "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", } module_threads = 8 - deps_pip = ["baddns~=1.1.846"] + deps_pip = ["baddns~=1.1.855"] def select_modules(self): selected_submodules = [] @@ -75,7 +75,9 @@ async def handle_event(self, event): tasks.append((module_instance, asyncio.create_task(module_instance.dispatch()))) for module_instance, task in tasks: - if await task: + + complete_task = await task + if complete_task: results = module_instance.analyze() if results and len(results) > 0: for r in results: @@ -120,3 +122,5 @@ async def handle_event(self, event): tags=[f"baddns-{module_instance.name.lower()}"], context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {{event.data}}', ) + + await module_instance.cleanup() diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index c5212d508..8ba6e79b2 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -19,7 +19,7 @@ class baddns_direct(BaseModule): "custom_nameservers": "Force BadDNS to use a list of custom nameservers", } module_threads = 8 - deps_pip = ["baddns~=1.1.846"] + deps_pip = ["baddns~=1.1.855"] scope_distance_modifier = 1 @@ -51,6 +51,7 @@ async def handle_event(self, event): CNAME_direct_instance = CNAME_direct_module(event.host, **kwargs) if await CNAME_direct_instance.dispatch(): + results = CNAME_direct_instance.analyze() if results and len(results) > 0: for r in results: @@ -68,6 +69,7 @@ async def handle_event(self, event): tags=[f"baddns-{CNAME_direct_module.name.lower()}"], context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}', ) + await CNAME_direct_instance.cleanup() async def filter_event(self, event): if event.type == "STORAGE_BUCKET": diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index ee92aeec4..64f795845 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -16,7 +16,7 @@ class baddns_zone(baddns_module): "only_high_confidence": "Do not emit low-confidence or generic detections", } module_threads = 8 - deps_pip = ["baddns~=1.1.846"] + deps_pip = ["baddns~=1.1.855"] def set_modules(self): self.enabled_submodules = ["NSEC", "zonetransfer"] From 7a021f954d12f0cace12b1e19852577dd54d7e16 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 30 Sep 2024 15:32:15 -0400 Subject: [PATCH 136/254] more efficient task handling --- bbot/modules/baddns.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 4b9f977c2..84a5b6fed 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -71,13 +71,19 @@ async def handle_event(self, event): kwargs["raw_query_retry_wait"] = 0 module_instance = ModuleClass(event.data, **kwargs) + task = asyncio.create_task(module_instance.dispatch()) + tasks.append((module_instance, task)) - tasks.append((module_instance, asyncio.create_task(module_instance.dispatch()))) + async for completed_task in self.helpers.as_completed([task for _, task in tasks]): - for module_instance, task in tasks: + module_instance = next((m for m, t in tasks if t == completed_task), None) + try: + task_result = await completed_task + except Exception as e: + self.hugewarning(f"Task for {module_instance} raised an error: {e}") + task_result = None - complete_task = await task - if complete_task: + if task_result: results = module_instance.analyze() if results and len(results) > 0: for r in results: @@ -122,5 +128,4 @@ async def handle_event(self, event): tags=[f"baddns-{module_instance.name.lower()}"], context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {{event.data}}', ) - await module_instance.cleanup() From 99d93c5f17849a79d95e0a659607d529d859d6b3 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 30 Sep 2024 20:51:58 +0100 Subject: [PATCH 137/254] Change warning to verbose --- bbot/modules/postman_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 32182e847..fa0e13a85 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -62,7 +62,7 @@ async def handle_event(self, event): context=f"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}", ) else: - self.warning( + self.verbose( f"Failed to validate {repo_url} is in our scope as it does not contain any in-scope dns_names / emails, skipping download" ) From 4eb472e36db9835dc33ab527befa00f342a90526 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Tue, 1 Oct 2024 10:34:30 +0100 Subject: [PATCH 138/254] Trufflehog now accepts postman workspaces --- bbot/modules/trufflehog.py | 5 + .../module_tests/test_module_trufflehog.py | 326 +++++++++++++++++- 2 files changed, 320 insertions(+), 11 deletions(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 61aa2306d..db3497ac9 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -99,6 +99,8 @@ async def handle_event(self, event): module = "git" elif "docker" in event.tags: module = "docker" + elif "postman" in event.tags: + module = "postman" else: module = "filesystem" if event.type == "CODE_REPOSITORY": @@ -164,6 +166,9 @@ async def execute_trufflehog(self, module, path): elif module == "docker": command.append("docker") command.append("--image=file://" + path) + elif module == "postman": + command.append("postman") + command.append("--workspace-paths=" + path) elif module == "filesystem": command.append("filesystem") command.append(path) diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py index 68285a001..a838ad6ab 100644 --- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py +++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py @@ -9,6 +9,7 @@ class TestTrufflehog(ModuleTestBase): + config_overrides = {"modules": {"postman_download": {"api_key": "asdf"}}} modules_overrides = [ "github_org", "speculate", @@ -17,6 +18,8 @@ class TestTrufflehog(ModuleTestBase): "github_workflows", "dockerhub", "docker_pull", + "postman", + "postman_download", "trufflehog", ] @@ -24,6 +27,37 @@ class TestTrufflehog(ModuleTestBase): async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response(url="https://api.github.com/zen") + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/me", + json={ + "user": { + "id": 000000, + "username": "test_key", + "email": "blacklanternsecurity@test.com", + "fullName": "Test Key", + "avatar": "", + "isPublic": True, + "teamId": 0, + "teamDomain": "", + "roles": ["user"], + }, + "operations": [ + {"name": "api_object_usage", "limit": 3, "usage": 0, "overage": 0}, + {"name": "collection_run_limit", "limit": 25, "usage": 0, "overage": 0}, + {"name": "file_storage_limit", "limit": 20, "usage": 0, "overage": 0}, + {"name": "flow_count", "limit": 5, "usage": 0, "overage": 0}, + {"name": "flow_requests", "limit": 5000, "usage": 0, "overage": 0}, + {"name": "performance_test_limit", "limit": 25, "usage": 0, "overage": 0}, + {"name": "postbot_calls", "limit": 50, "usage": 0, "overage": 0}, + {"name": "reusable_packages", "limit": 3, "usage": 0, "overage": 0}, + {"name": "test_data_retrieval", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "test_data_storage", "limit": 10, "usage": 0, "overage": 0}, + {"name": "mock_usage", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "monitor_request_runs", "limit": 1000, "usage": 0, "overage": 0}, + {"name": "api_usage", "limit": 1000, "usage": 0, "overage": 0}, + ], + }, + ) module_test.httpx_mock.add_response( url="https://api.github.com/orgs/blacklanternsecurity", json={ @@ -813,6 +847,248 @@ async def setup_before_prep(self, module_test): ) async def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "search", "method": "POST", "path": "/search-all", "body": {"queryIndices": ["collaboration.workspace"], "queryText": "blacklanternsecurity", "size": 100, "from": 0, "clientTraceId": "", "requestOrigin": "srp", "mergeEntities": "true", "nonNestedRequests": "true", "domain": "public"}}', + json={ + "data": [ + { + "score": 611.41156, + "normalizedScore": 23, + "document": { + "watcherCount": 6, + "apiCount": 0, + "forkCount": 0, + "isblacklisted": "false", + "createdAt": "2021-06-15T14:03:51", + "publishertype": "team", + "publisherHandle": "blacklanternsecurity", + "id": "11498add-357d-4bc5-a008-0a2d44fb8829", + "slug": "bbot-public", + "updatedAt": "2024-07-30T11:00:35", + "entityType": "workspace", + "visibilityStatus": "public", + "forkcount": "0", + "tags": [], + "createdat": "2021-06-15T14:03:51", + "forkLabel": "", + "publisherName": "blacklanternsecurity", + "name": "BlackLanternSecurity BBOT [Public]", + "dependencyCount": 7, + "collectionCount": 6, + "warehouse__updated_at": "2024-07-30 11:00:00", + "privateNetworkFolders": [], + "isPublisherVerified": False, + "publisherType": "team", + "curatedInList": [], + "creatorId": "6900157", + "description": "", + "forklabel": "", + "publisherId": "299401", + "publisherLogo": "", + "popularity": 5, + "isPublic": True, + "categories": [], + "universaltags": "", + "views": 5788, + "summary": "BLS public workspaces.", + "memberCount": 2, + "isBlacklisted": False, + "publisherid": "299401", + "isPrivateNetworkEntity": False, + "isDomainNonTrivial": True, + "privateNetworkMeta": "", + "updatedat": "2021-10-20T16:19:29", + "documentType": "workspace", + }, + "highlight": {"summary": "BLS BBOT api test."}, + }, + ], + "meta": { + "queryText": "blacklanternsecurity", + "total": { + "collection": 0, + "request": 0, + "workspace": 1, + "api": 0, + "team": 0, + "user": 0, + "flow": 0, + "apiDefinition": 0, + "privateNetworkFolder": 0, + }, + "state": "AQ4", + "spellCorrection": {"count": {"all": 1, "workspace": 1}, "correctedQueryText": None}, + "featureFlags": { + "enabledPublicResultCuration": True, + "boostByPopularity": True, + "reRankPostNormalization": True, + "enableUrlBarHostNameSearch": True, + }, + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/ws/proxy", + match_content=b'{"service": "workspaces", "method": "GET", "path": "/workspaces?handle=blacklanternsecurity&slug=bbot-public"}', + json={ + "meta": {"model": "workspace", "action": "find", "nextCursor": ""}, + "data": [ + { + "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "name": "BlackLanternSecurity BBOT [Public]", + "description": None, + "summary": "BLS public workspaces.", + "createdBy": "299401", + "updatedBy": "299401", + "team": None, + "createdAt": "2021-10-20T16:19:29", + "updatedAt": "2021-10-20T16:19:29", + "visibilityStatus": "public", + "profileInfo": { + "slug": "bbot-public", + "profileType": "team", + "profileId": "000000", + "publicHandle": "https://www.postman.com/blacklanternsecurity", + "publicImageURL": "", + "publicName": "BlackLanternSecurity", + "isVerified": False, + }, + } + ], + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + json={ + "workspace": { + "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "name": "BlackLanternSecurity BBOT [Public]", + "type": "personal", + "description": None, + "visibility": "public", + "createdBy": "00000000", + "updatedBy": "00000000", + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-17T08:57:16.000Z", + "collections": [ + { + "id": "2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + "name": "BBOT Public", + "uid": "10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + }, + ], + "environments": [ + { + "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", + "name": "BBOT Test", + "uid": "10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + } + ], + "apis": [], + } + }, + ) + module_test.httpx_mock.add_response( + url="https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals", + json={ + "model_id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "meta": {"model": "globals", "action": "find"}, + "data": { + "workspace": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + "lastUpdatedBy": "00000000", + "lastRevision": 1637239113000, + "id": "8be7574b-219f-49e0-8d25-da447a882e4e", + "values": [ + { + "key": "endpoint_url", + "value": "https://api.blacklanternsecurity.com/", + "enabled": True, + }, + ], + "createdAt": "2021-11-17T06:09:01.000Z", + "updatedAt": "2021-11-18T12:38:33.000Z", + }, + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + json={ + "environment": { + "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", + "name": "BBOT Test", + "owner": "00000000", + "createdAt": "2021-11-17T06:29:54.000Z", + "updatedAt": "2021-11-23T07:06:53.000Z", + "values": [ + { + "key": "temp_session_endpoint", + "value": "https://api.blacklanternsecurity.com/", + "enabled": True, + }, + ], + "isPublic": True, + } + }, + ) + module_test.httpx_mock.add_response( + url="https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + json={ + "collection": { + "info": { + "_postman_id": "62b91565-d2e2-4bcd-8248-4dba2e3452f0", + "name": "BBOT Public", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "updatedAt": "2021-11-17T07:13:16.000Z", + "createdAt": "2021-11-17T07:13:15.000Z", + "lastUpdatedBy": "00000000", + "uid": "172983-62b91565-d2e2-4bcd-8248-4dba2e3452f0", + }, + "item": [ + { + "name": "Generate API Session", + "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + "protocolProfileBehavior": {"disableBodyPruning": True}, + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": '{"username": "test", "password": "Test"}', + }, + "url": { + "raw": "https://admin:admin@the-internet.herokuapp.com/basic_auth", + "host": ["https://admin:admin@the-internet.herokuapp.com/basic_auth"], + }, + "description": "", + }, + "response": [], + "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + }, + { + "name": "Generate API Session", + "id": "c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + "protocolProfileBehavior": {"disableBodyPruning": True}, + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": '{"username": "test", "password": "Test"}', + }, + "url": { + "raw": "https://admin:admin@internal.host.com", + "host": ["https://admin:admin@internal.host.com"], + }, + "description": "", + }, + "response": [], + "uid": "10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1", + }, + ], + } + }, + ) temp_path = Path("/tmp/.bbot_test") temp_repo_path = temp_path / "test_keys" shutil.rmtree(temp_repo_path, ignore_errors=True) @@ -850,21 +1126,25 @@ def check(self, module_test, events): e for e in events if e.type == "VULNERABILITY" - and (e.data["host"] == "hub.docker.com" or e.data["host"] == "github.com") + and ( + e.data["host"] == "hub.docker.com" + or e.data["host"] == "github.com" + or e.data["host"] == "www.postman.com" + ) and "Verified Secret Found." in e.data["description"] and "Raw result: [https://admin:admin@the-internet.herokuapp.com]" in e.data["description"] and "RawV2 result: [https://admin:admin@the-internet.herokuapp.com/basic_auth]" in e.data["description"] ] - # Trufflehog should find 3 verifiable secrets, 1 from the github, 1 from the workflow log and 1 from the docker image. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder - assert 3 == len(vuln_events), "Failed to find secret in events" + # Trufflehog should find 4 verifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder + assert 4 == len(vuln_events), "Failed to find secret in events" github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].parent folder = Path(github_repo_event.data["path"]) assert folder.is_dir(), "Destination folder doesn't exist" with open(folder / "keys.txt") as f: content = f.read() assert content == self.file_content, "File content doesn't match" - filesystem_events = [e.parent for e in vuln_events if "bbot" in e.data["description"]] - assert len(filesystem_events) == 3 + filesystem_events = [e.parent for e in vuln_events] + assert len(filesystem_events) == 4 assert all([e.type == "FILESYSTEM" for e in filesystem_events]) assert 1 == len( [ @@ -889,30 +1169,44 @@ def check(self, module_test, events): and Path(e.data["path"]).is_file() ] ), "Docker image file does not exist" + assert 1 == len( + [ + e + for e in filesystem_events + if e.data["path"].endswith( + "/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip" + ) + and Path(e.data["path"]).is_file() + ] + ), "Failed to find blacklanternsecurity postman workspace" class TestTrufflehog_NonVerified(TestTrufflehog): - config_overrides = {"modules": {"trufflehog": {"only_verified": False}}} + config_overrides = {"modules": {"trufflehog": {"only_verified": False}, "postman_download": {"api_key": "asdf"}}} def check(self, module_test, events): finding_events = [ e for e in events if e.type == e.type == "FINDING" - and (e.data["host"] == "hub.docker.com" or e.data["host"] == "github.com") + and ( + e.data["host"] == "hub.docker.com" + or e.data["host"] == "github.com" + or e.data["host"] == "www.postman.com" + ) and "Potential Secret Found." in e.data["description"] and "Raw result: [https://admin:admin@internal.host.com]" in e.data["description"] ] - # Trufflehog should find 3 unverifiable secrets, 1 from the github, 1 from the workflow log and 1 from the docker image. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder - assert 3 == len(finding_events), "Failed to find secret in events" + # Trufflehog should find 4 unverifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder + assert 4 == len(finding_events), "Failed to find secret in events" github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].parent folder = Path(github_repo_event.data["path"]) assert folder.is_dir(), "Destination folder doesn't exist" with open(folder / "keys.txt") as f: content = f.read() assert content == self.file_content, "File content doesn't match" - filesystem_events = [e.parent for e in finding_events if "bbot" in e.data["description"]] - assert len(filesystem_events) == 3 + filesystem_events = [e.parent for e in finding_events] + assert len(filesystem_events) == 4 assert all([e.type == "FILESYSTEM" for e in filesystem_events]) assert 1 == len( [ @@ -937,3 +1231,13 @@ def check(self, module_test, events): and Path(e.data["path"]).is_file() ] ), "Docker image file does not exist" + assert 1 == len( + [ + e + for e in filesystem_events + if e.data["path"].endswith( + "/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip" + ) + and Path(e.data["path"]).is_file() + ] + ), "Failed to find blacklanternsecurity postman workspace" From 58486c556ada38a95ec41d6b541268d1690c8b01 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 30 Sep 2024 12:12:59 -0400 Subject: [PATCH 139/254] support multiple API keys, WIP --- README.md | 8 +- bbot/core/helpers/web/web.py | 61 -------- bbot/modules/base.py | 138 +++++++++++++++--- bbot/modules/bevigil.py | 9 +- bbot/modules/binaryedge.py | 9 +- bbot/modules/templates/subdomain_enum.py | 8 +- .../module_tests/test_module_bevigil.py | 27 ++++ .../module_tests/test_module_binaryedge.py | 2 + docs/scanning/presets.md | 5 +- 9 files changed, 170 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index ad2ad61d0..a36af3cc3 100644 --- a/README.md +++ b/README.md @@ -303,13 +303,17 @@ For more information, see [Targets](https://www.blacklanternsecurity.com/bbot/St Similar to Amass or Subfinder, BBOT supports API keys for various third-party services such as SecurityTrails, etc. -The standard way to do this is to enter your API keys in **`~/.config/bbot/bbot.yml`**: +The standard way to do this is to enter your API keys in **`~/.config/bbot/bbot.yml`**. Note that multiple API keys are allowed: ```yaml modules: shodan_dns: api_key: 4f41243847da693a4f356c0486114bc6 c99: - api_key: 21a270d5f59c9b05813a72bb41707266 + # multiple API keys + api_key: + - 21a270d5f59c9b05813a72bb41707266 + - ea8f243d9885cf8ce9876a580224fd3c + - 5bc6ed268ab6488270e496d3183a1a27 virustotal: api_key: dd5f0eee2e4a99b71a939bded450b246 securitytrails: diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py index 5ed7e781e..a49748008 100644 --- a/bbot/core/helpers/web/web.py +++ b/bbot/core/helpers/web/web.py @@ -1,7 +1,6 @@ import re import logging import warnings -import traceback from pathlib import Path from bs4 import BeautifulSoup @@ -288,66 +287,6 @@ async def wordlist(self, path, lines=None, **kwargs): f.write(line) return truncated_filename - async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs): - """ - An asynchronous generator function for iterating through paginated API data. - - This function continuously makes requests to a specified API URL, incrementing the page number - or applying a custom pagination function, and yields the received data one page at a time. - It is well-suited for APIs that provide paginated results. - - Args: - url (str): The initial API URL. Can contain placeholders for 'page', 'page_size', and 'offset'. - page_size (int, optional): The number of items per page. Defaults to 100. - json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True. - next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None. - **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function. - - Yields: - dict or httpx.Response: If 'json' is True, yields a dictionary containing the parsed JSON data. Otherwise, yields the raw HTTP response. - - Note: - The loop will continue indefinitely unless manually stopped. Make sure to break out of the loop once the last page has been received. - - Examples: - >>> agen = api_page_iter('https://api.example.com/data?page={page}&page_size={page_size}') - >>> try: - >>> async for page in agen: - >>> subdomains = page["subdomains"] - >>> self.hugesuccess(subdomains) - >>> if not subdomains: - >>> break - >>> finally: - >>> agen.aclose() - """ - page = 1 - offset = 0 - result = None - while 1: - if result and callable(next_key): - try: - new_url = next_key(result) - except Exception as e: - log.debug(f"Failed to extract next page of results from {url}: {e}") - log.debug(traceback.format_exc()) - else: - new_url = url.format(page=page, page_size=page_size, offset=offset) - result = await self.request(new_url, **requests_kwargs) - if result is None: - log.verbose(f"api_page_iter() got no response for {url}") - break - try: - if json: - result = result.json() - yield result - except Exception: - log.warning(f'Error in api_page_iter() for url: "{new_url}"') - log.trace(traceback.format_exc()) - break - finally: - offset += page_size - page += 1 - async def curl(self, *args, **kwargs): """ An asynchronous function that runs a cURL command with specified arguments and options. diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 42426c2ac..127049585 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -103,7 +103,7 @@ class BaseModule: _module_threads = 1 _batch_size = 1 batch_wait = 10 - failed_request_abort_threshold = 5 + _failed_request_abort_threshold = 5 default_discovery_context = "{module} discovered {event.type}: {event.data}" @@ -148,6 +148,8 @@ def __init__(self, scan): # string constant self._custom_filter_criteria_msg = "it did not meet custom filter criteria" + self._api_keys = [] + # track number of failures (for .request_with_fail_count()) self._request_failures = 0 @@ -310,6 +312,24 @@ async def require_api_key(self): else: return None, "No API key set" + @property + def api_key(self): + if self._api_keys: + return self._api_keys[0] + + @api_key.setter + def api_key(self, api_keys): + if isinstance(api_keys, str): + api_keys = [api_keys] + self._api_keys = list(api_keys) + + def cycle_api_key(self): + self._api_keys.insert(0, self._api_keys.pop()) + + @property + def failed_request_abort_threshold(self): + return max(self._failed_request_abort_threshold, len(self._api_keys)) + async def ping(self): """Asynchronously checks the health of the configured API. @@ -1065,32 +1085,108 @@ async def run_process_live(self, *args, **kwargs): async for line in self.helpers.run_live(*args, **kwargs): yield line - async def request_with_fail_count(self, *args, **kwargs): - """Asynchronously perform an HTTP request while keeping track of consecutive failures. + def prepare_api_request(self, url, kwargs): + """ + Prepare an API request by adding the necessary authentication - header, bearer token, etc. + """ + if self.api_key: + url = url.format(api_key=self.api_key) + if not "headers" in kwargs: + kwargs["headers"] = {} + kwargs["headers"]["Authorization"] = f"Bearer {self.api_key}" + return url, kwargs - This function wraps the `self.helpers.request` method, incrementing a failure counter if - the request returns None. When the failure counter exceeds `self.failed_request_abort_threshold`, - the module is set to an error state. + async def api_request(self, *args, **kwargs): + """ + Makes an HTTP request while automatically cycling API keys + """ + url = args[0] if args else kwargs.pop("url", "") - Args: - *args: Positional arguments to pass to `self.helpers.request`. - **kwargs: Keyword arguments to pass to `self.helpers.request`. + # loop until we have a successful request + while 1: + if not "headers" in kwargs: + kwargs["headers"] = {} + new_url, kwargs = self.prepare_api_request(url, kwargs) + kwargs["url"] = new_url - Returns: - Any: The response object or None if the request failed. + self.critical(kwargs) + r = await self.helpers.request(**kwargs) + success = False if r is None else r.is_success - Raises: - None: Sets the module to an error state when the failure threshold is reached. - """ - r = await self.helpers.request(*args, **kwargs) - if r is None: - self._request_failures += 1 - else: - self._request_failures = 0 - if self._request_failures >= self.failed_request_abort_threshold: - self.set_error_state(f"Setting error state due to {self._request_failures:,} failed HTTP requests") + if success: + self._request_failures = 0 + break + else: + # if request failed, cycle API keys and try again + self._request_failures += 1 + if self._request_failures >= self.failed_request_abort_threshold: + self.set_error_state(f"Setting error state due to {self._request_failures:,} failed HTTP requests") + break + else: + self.cycle_api_key() + continue return r + async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs): + """ + An asynchronous generator function for iterating through paginated API data. + + This function continuously makes requests to a specified API URL, incrementing the page number + or applying a custom pagination function, and yields the received data one page at a time. + It is well-suited for APIs that provide paginated results. + + Args: + url (str): The initial API URL. Can contain placeholders for 'page', 'page_size', and 'offset'. + page_size (int, optional): The number of items per page. Defaults to 100. + json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True. + next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None. + **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function. + + Yields: + dict or httpx.Response: If 'json' is True, yields a dictionary containing the parsed JSON data. Otherwise, yields the raw HTTP response. + + Note: + The loop will continue indefinitely unless manually stopped. Make sure to break out of the loop once the last page has been received. + + Examples: + >>> agen = api_page_iter('https://api.example.com/data?page={page}&page_size={page_size}') + >>> try: + >>> async for page in agen: + >>> subdomains = page["subdomains"] + >>> self.hugesuccess(subdomains) + >>> if not subdomains: + >>> break + >>> finally: + >>> agen.aclose() + """ + page = 1 + offset = 0 + result = None + while 1: + if result and callable(next_key): + try: + new_url = next_key(result) + except Exception as e: + self.debug(f"Failed to extract next page of results from {url}: {e}") + self.debug(traceback.format_exc()) + else: + new_url = url.format(page=page, page_size=page_size, offset=offset) + result = await self.api_request(new_url, **requests_kwargs) + if result is None: + self.verbose(f"api_page_iter() got no response for {url}") + break + try: + if json: + result = result.json() + yield result + except Exception: + self.warning(f'Error in api_page_iter() for url: "{new_url}"') + self.trace(traceback.format_exc()) + break + finally: + offset += page_size + page += 1 + @property def preset(self): return self.scan.preset diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index 3926d90b3..50e891811 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -22,10 +22,13 @@ class bevigil(subdomain_enum_apikey): async def setup(self): self.api_key = self.config.get("api_key", "") - self.headers = {"X-Access-Token": self.api_key} self.urls = self.config.get("urls", False) return await super().setup() + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["X-Access-Token"] = self.api_key + return url, kwargs + async def ping(self): pass @@ -54,11 +57,11 @@ async def handle_event(self, event): async def request_subdomains(self, query): url = f"{self.base_url}/{self.helpers.quote(query)}/subdomains/" - return await self.request_with_fail_count(url, headers=self.headers) + return await self.api_request(url) async def request_urls(self, query): url = f"{self.base_url}/{self.helpers.quote(query)}/urls/" - return await self.request_with_fail_count(url, headers=self.headers) + return await self.api_request(url) def parse_subdomains(self, r, query=None): results = set() diff --git a/bbot/modules/binaryedge.py b/bbot/modules/binaryedge.py index 3948e893c..e9f6224b6 100644 --- a/bbot/modules/binaryedge.py +++ b/bbot/modules/binaryedge.py @@ -21,18 +21,21 @@ class binaryedge(subdomain_enum_apikey): async def setup(self): self.max_records = self.config.get("max_records", 1000) - self.headers = {"X-Key": self.config.get("api_key", "")} return await super().setup() + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["X-Key"] = self.api_key + return url, kwargs + async def ping(self): url = f"{self.base_url}/user/subscription" - j = (await self.request_with_fail_count(url, headers=self.headers)).json() + j = (await self.api_request(url)).json() assert j.get("requests_left", 0) > 0 async def request_url(self, query): # todo: host query (certs + services) url = f"{self.base_url}/query/domains/subdomain/{self.helpers.quote(query)}" - return await self.request_with_fail_count(url, headers=self.headers) + return await self.api_request(url) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index 4f0608fb9..e161636b2 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -63,7 +63,7 @@ async def handle_event(self, event): async def request_url(self, query): url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}" - return await self.request_with_fail_count(url) + return await self.api_request(url) def make_query(self, event): query = event.data @@ -78,11 +78,7 @@ def make_query(self, event): if self.scan.in_scope(p): query = p break - try: - return ".".join([s for s in query.split(".") if s != "_wildcard"]) - except Exception: - self.critical(query) - raise + return ".".join([s for s in query.split(".") if s != "_wildcard"]) def parse_results(self, r, query=None): json = r.json() diff --git a/bbot/test/test_step_2/module_tests/test_module_bevigil.py b/bbot/test/test_step_2/module_tests/test_module_bevigil.py index e8ab13d7a..328e213d2 100644 --- a/bbot/test/test_step_2/module_tests/test_module_bevigil.py +++ b/bbot/test/test_step_2/module_tests/test_module_bevigil.py @@ -1,12 +1,16 @@ +import random + from .base import ModuleTestBase class TestBeVigil(ModuleTestBase): + module_name = "bevigil" config_overrides = {"modules": {"bevigil": {"api_key": "asdf", "urls": True}}} async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( url=f"https://osint.bevigil.com/api/blacklanternsecurity.com/subdomains/", + match_headers={"X-Access-Token": "asdf"}, json={ "domain": "blacklanternsecurity.com", "subdomains": [ @@ -22,3 +26,26 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" assert any(e.data == "https://asdf.blacklanternsecurity.com/" for e in events), "Failed to detect url" + + +class TestBeVigilMultiKey(TestBeVigil): + api_keys = ["1234", "4321", "asdf", "fdsa"] + random.shuffle(api_keys) + config_overrides = {"modules": {"bevigil": {"api_key": api_keys, "urls": True}}} + + async def setup_after_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://osint.bevigil.com/api/blacklanternsecurity.com/subdomains/", + match_headers={"X-Access-Token": "fdsa"}, + json={ + "domain": "blacklanternsecurity.com", + "subdomains": [ + "asdf.blacklanternsecurity.com", + ], + }, + ) + module_test.httpx_mock.add_response( + match_headers={"X-Access-Token": "asdf"}, + url=f"https://osint.bevigil.com/api/blacklanternsecurity.com/urls/", + json={"domain": "blacklanternsecurity.com", "urls": ["https://asdf.blacklanternsecurity.com"]}, + ) diff --git a/bbot/test/test_step_2/module_tests/test_module_binaryedge.py b/bbot/test/test_step_2/module_tests/test_module_binaryedge.py index 505c62376..95b4ae7a7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_binaryedge.py +++ b/bbot/test/test_step_2/module_tests/test_module_binaryedge.py @@ -7,6 +7,7 @@ class TestBinaryEdge(ModuleTestBase): async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( url=f"https://api.binaryedge.io/v2/query/domains/subdomain/blacklanternsecurity.com", + match_headers={"X-Key": "asdf"}, json={ "query": "blacklanternsecurity.com", "page": 1, @@ -19,6 +20,7 @@ async def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url=f"https://api.binaryedge.io/v2/user/subscription", + match_headers={"X-Key": "asdf"}, json={ "subscription": {"name": "Free"}, "end_date": "2023-06-17", diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md index d0c16c4f1..f68a62dc3 100644 --- a/docs/scanning/presets.md +++ b/docs/scanning/presets.md @@ -77,7 +77,10 @@ config: securitytrails: api_key: 21a270d5f59c9b05813a72bb41707266 virustotal: - api_key: 4f41243847da693a4f356c0486114bc6 + # multiple API keys are allowed + api_key: + - 4f41243847da693a4f356c0486114bc6 + - 5bc6ed268ab6488270e496d3183a1a27 ``` To execute your custom preset, you do: From 1e87e6d4437f33f058c864459f0e27ffd103d8d1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 1 Oct 2024 09:33:01 -0400 Subject: [PATCH 140/254] fix conflict --- bbot/modules/anubisdb.py | 2 +- bbot/modules/base.py | 37 +++++++++++++------ bbot/modules/builtwith.py | 8 ++-- bbot/modules/c99.py | 8 ++-- bbot/modules/certspotter.py | 2 +- bbot/modules/chaos.py | 8 +++- bbot/modules/columbus.py | 2 +- bbot/modules/crt.py | 2 +- bbot/modules/dnsdumpster.py | 4 +- bbot/modules/emailformat.py | 2 +- bbot/modules/fullhunt.py | 9 +++-- bbot/modules/github_codesearch.py | 2 +- bbot/modules/github_org.py | 8 ++-- bbot/modules/github_workflows.py | 6 +-- bbot/modules/gitlab.py | 7 +--- bbot/modules/hackertarget.py | 2 +- bbot/modules/hunterio.py | 9 ++--- bbot/modules/internetdb.py | 2 +- bbot/modules/ip2location.py | 6 +-- bbot/modules/ipstack.py | 8 ++-- bbot/modules/leakix.py | 9 ++++- bbot/modules/myssl.py | 2 +- bbot/modules/otx.py | 2 +- bbot/modules/passivetotal.py | 4 +- bbot/modules/postman_download.py | 15 ++++---- bbot/modules/rapiddns.py | 2 +- bbot/modules/securitytrails.py | 8 ++-- bbot/modules/shodan_dns.py | 4 +- bbot/modules/skymem.py | 4 +- bbot/modules/templates/github.py | 19 +++++++--- bbot/modules/templates/shodan.py | 16 +++++--- bbot/modules/trickest.py | 9 +++-- bbot/modules/virustotal.py | 13 +++---- bbot/modules/zoomeye.py | 8 ++-- .../module_tests/test_module_gitlab.py | 13 +++++-- .../test_module_postman_download.py | 4 ++ 36 files changed, 156 insertions(+), 110 deletions(-) diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py index f95adde0b..b456365e5 100644 --- a/bbot/modules/anubisdb.py +++ b/bbot/modules/anubisdb.py @@ -20,7 +20,7 @@ class anubisdb(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}/{self.helpers.quote(query)}" - return await self.request_with_fail_count(url) + return await self.api_request(url) def abort_if_pre(self, hostname): """ diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 127049585..e98e630ac 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -63,7 +63,7 @@ class BaseModule: batch_wait (int): Seconds to wait before force-submitting a batch. Default is 10. - failed_request_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `request_with_fail_count()` is used. Default is 5. + failed_request_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `api_request()` is used. Default is 5. _preserve_graph (bool): When set to True, accept events that may be duplicates but are necessary for construction of complete graph. Typically only enabled for output modules that need to maintain full chains of events, e.g. `neo4j` and `json`. Default is False. @@ -103,7 +103,10 @@ class BaseModule: _module_threads = 1 _batch_size = 1 batch_wait = 10 + # disable the module after this many failed requests in a row _failed_request_abort_threshold = 5 + # sleep for this many seconds after being rate limited + _429_sleep_interval = 30 default_discovery_context = "{module} discovered {event.type}: {event.data}" @@ -150,7 +153,7 @@ def __init__(self, scan): self._api_keys = [] - # track number of failures (for .request_with_fail_count()) + # track number of failures (for .api_request()) self._request_failures = 0 self._tasks = [] @@ -324,7 +327,8 @@ def api_key(self, api_keys): self._api_keys = list(api_keys) def cycle_api_key(self): - self._api_keys.insert(0, self._api_keys.pop()) + if self._api_keys: + self._api_keys.insert(0, self._api_keys.pop()) @property def failed_request_abort_threshold(self): @@ -338,7 +342,7 @@ async def ping(self): Example Usage: In your implementation, if the API has a "/ping" endpoint: async def ping(self): - r = await self.request_with_fail_count(f"{self.base_url}/ping") + r = await self.api_request(f"{self.base_url}/ping") resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content @@ -1098,7 +1102,10 @@ def prepare_api_request(self, url, kwargs): async def api_request(self, *args, **kwargs): """ - Makes an HTTP request while automatically cycling API keys + Makes an HTTP request while automatically: + - avoiding rate limits (sleep/retry) + - cycling API keys + - cancelling after too many failed attempts """ url = args[0] if args else kwargs.pop("url", "") @@ -1109,22 +1116,30 @@ async def api_request(self, *args, **kwargs): new_url, kwargs = self.prepare_api_request(url, kwargs) kwargs["url"] = new_url - self.critical(kwargs) r = await self.helpers.request(**kwargs) success = False if r is None else r.is_success if success: self._request_failures = 0 - break else: - # if request failed, cycle API keys and try again + status_code = getattr(r, "status_code", 0) self._request_failures += 1 if self._request_failures >= self.failed_request_abort_threshold: self.set_error_state(f"Setting error state due to {self._request_failures:,} failed HTTP requests") - break else: - self.cycle_api_key() - continue + # sleep for a bit if we're being rate limited + if status_code == 429: + self.verbose( + f"Sleeping for {self._429_sleep_interval:,} seconds due to rate limit (HTTP status: 429)" + ) + await asyncio.sleep(self._429_sleep_interval) + continue + elif self._api_keys: + # if request failed, cycle API keys and try again + self.cycle_api_key() + continue + break + return r async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **requests_kwargs): diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index e51bb5db8..76f8a4e39 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -59,12 +59,12 @@ async def handle_event(self, event): ) async def request_domains(self, query): - url = f"{self.base_url}/v20/api.json?KEY={self.api_key}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes" - return await self.request_with_fail_count(url) + url = f"{self.base_url}/v20/api.json?KEY={{api_key}}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes" + return await self.api_request(url) async def request_redirects(self, query): - url = f"{self.base_url}/redirect1/api.json?KEY={self.api_key}&LOOKUP={query}" - return await self.request_with_fail_count(url) + url = f"{self.base_url}/redirect1/api.json?KEY={{api_key}}&LOOKUP={query}" + return await self.api_request(url) def parse_domains(self, r, query): """ diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 8db5775cb..062b5523e 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -17,13 +17,13 @@ class c99(subdomain_enum_apikey): base_url = "https://api.c99.nl" async def ping(self): - url = f"{self.base_url}/randomnumber?key={self.api_key}&between=1,100&json" - response = await self.request_with_fail_count(url) + url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json" + response = await self.api_request(url) assert response.json()["success"] == True async def request_url(self, query): - url = f"{self.base_url}/subdomainfinder?key={self.api_key}&domain={self.helpers.quote(query)}&json" - return await self.request_with_fail_count(url) + url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json" + return await self.api_request(url) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/certspotter.py b/bbot/modules/certspotter.py index 9d7e00d87..d4d770365 100644 --- a/bbot/modules/certspotter.py +++ b/bbot/modules/certspotter.py @@ -15,7 +15,7 @@ class certspotter(subdomain_enum): def request_url(self, query): url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names" - return self.request_with_fail_count(url, timeout=self.http_timeout + 30) + return self.api_request(url, timeout=self.http_timeout + 30) def parse_results(self, r, query): json = r.json() diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py index e98fedd84..ecc960690 100644 --- a/bbot/modules/chaos.py +++ b/bbot/modules/chaos.py @@ -18,13 +18,17 @@ class chaos(subdomain_enum_apikey): async def ping(self): url = f"{self.base_url}/example.com" - response = await self.request_with_fail_count(url, headers={"Authorization": self.api_key}) + response = await self.api_request(url) assert response.json()["domain"] == "example.com" + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["Authorization"] = self.api_key + return url, kwargs + async def request_url(self, query): _, domain = self.helpers.split_domain(query) url = f"{self.base_url}/{domain}/subdomains" - return await self.request_with_fail_count(url, headers={"Authorization": self.api_key}) + return await self.api_request(url) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/columbus.py b/bbot/modules/columbus.py index 4c09f3a5d..6e3e9ce0b 100644 --- a/bbot/modules/columbus.py +++ b/bbot/modules/columbus.py @@ -15,7 +15,7 @@ class columbus(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}/{self.helpers.quote(query)}?days=365" - return await self.request_with_fail_count(url) + return await self.api_request(url) def parse_results(self, r, query): results = set() diff --git a/bbot/modules/crt.py b/bbot/modules/crt.py index 27d088a0d..441dbbb9b 100644 --- a/bbot/modules/crt.py +++ b/bbot/modules/crt.py @@ -21,7 +21,7 @@ async def setup(self): async def request_url(self, query): params = {"q": f"%.{query}", "output": "json"} url = self.helpers.add_get_params(self.base_url, params).geturl() - return await self.request_with_fail_count(url, timeout=self.http_timeout + 30) + return await self.api_request(url, timeout=self.http_timeout + 30) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/dnsdumpster.py b/bbot/modules/dnsdumpster.py index a1d981565..ab36b493e 100644 --- a/bbot/modules/dnsdumpster.py +++ b/bbot/modules/dnsdumpster.py @@ -18,7 +18,7 @@ class dnsdumpster(subdomain_enum): async def query(self, domain): ret = [] # first, get the CSRF tokens - res1 = await self.request_with_fail_count(self.base_url) + res1 = await self.api_request(self.base_url) status_code = getattr(res1, "status_code", 0) if status_code in [429]: self.verbose(f'Too many requests "{status_code}"') @@ -62,7 +62,7 @@ async def query(self, domain): # Otherwise, do the needful subdomains = set() - res2 = await self.request_with_fail_count( + res2 = await self.api_request( f"{self.base_url}/", method="POST", cookies={"csrftoken": csrftoken}, diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py index c7161070a..67e1a3806 100644 --- a/bbot/modules/emailformat.py +++ b/bbot/modules/emailformat.py @@ -18,7 +18,7 @@ class emailformat(BaseModule): async def handle_event(self, event): _, query = self.helpers.split_domain(event.data) url = f"{self.base_url}/d/{self.helpers.quote(query)}/" - r = await self.request_with_fail_count(url) + r = await self.api_request(url) if not r: return for email in await self.helpers.re.extract_emails(r.text): diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py index 90f83bcfc..5736053e3 100644 --- a/bbot/modules/fullhunt.py +++ b/bbot/modules/fullhunt.py @@ -18,18 +18,21 @@ class fullhunt(subdomain_enum_apikey): async def setup(self): self.api_key = self.config.get("api_key", "") - self.headers = {"x-api-key": self.api_key} return await super().setup() async def ping(self): url = f"{self.base_url}/auth/status" - j = (await self.request_with_fail_count(url, headers=self.headers)).json() + j = (await self.api_request(url)).json() remaining = j["user_credits"]["remaining_credits"] assert remaining > 0, "No credits remaining" + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["x-api-key"] = self.api_key + return url, kwargs + async def request_url(self, query): url = f"{self.base_url}/domain/{self.helpers.quote(query)}/subdomains" - response = await self.request_with_fail_count(url, headers=self.headers) + response = await self.api_request(url) return response def parse_results(self, r, query): diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py index cca7e3cff..39e1ee7b4 100644 --- a/bbot/modules/github_codesearch.py +++ b/bbot/modules/github_codesearch.py @@ -42,7 +42,7 @@ async def handle_event(self, event): async def query(self, query): repos = {} url = f"{self.base_url}/search/code?per_page=100&type=Code&q={self.helpers.quote(query)}&page=" + "{page}" - agen = self.helpers.api_page_iter(url, headers=self.headers, json=False) + agen = self.api_page_iter(url, headers=self.headers, json=False) num_results = 0 try: async for r in agen: diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py index 90fba82b8..5b8571874 100644 --- a/bbot/modules/github_org.py +++ b/bbot/modules/github_org.py @@ -104,7 +104,7 @@ async def handle_event(self, event): async def query_org_repos(self, query): repos = [] url = f"{self.base_url}/orgs/{self.helpers.quote(query)}/repos?per_page=100&page=" + "{page}" - agen = self.helpers.api_page_iter(url, headers=self.headers, json=False) + agen = self.api_page_iter(url, json=False) try: async for r in agen: if r is None: @@ -132,7 +132,7 @@ async def query_org_repos(self, query): async def query_org_members(self, query): members = [] url = f"{self.base_url}/orgs/{self.helpers.quote(query)}/members?per_page=100&page=" + "{page}" - agen = self.helpers.api_page_iter(url, headers=self.headers, json=False) + agen = self.api_page_iter(url, json=False) try: async for r in agen: if r is None: @@ -160,7 +160,7 @@ async def query_org_members(self, query): async def query_user_repos(self, query): repos = [] url = f"{self.base_url}/users/{self.helpers.quote(query)}/repos?per_page=100&page=" + "{page}" - agen = self.helpers.api_page_iter(url, headers=self.headers, json=False) + agen = self.api_page_iter(url, json=False) try: async for r in agen: if r is None: @@ -189,7 +189,7 @@ async def validate_org(self, org): is_org = False in_scope = False url = f"{self.base_url}/orgs/{org}" - r = await self.helpers.request(url, headers=self.headers) + r = await self.api_request(url) if r is None: return is_org, in_scope status_code = getattr(r, "status_code", 0) diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py index df46f155c..369b33742 100644 --- a/bbot/modules/github_workflows.py +++ b/bbot/modules/github_workflows.py @@ -88,7 +88,7 @@ async def handle_event(self, event): async def get_workflows(self, owner, repo): workflows = [] url = f"{self.base_url}/repos/{owner}/{repo}/actions/workflows?per_page=100&page=" + "{page}" - agen = self.helpers.api_page_iter(url, headers=self.headers, json=False) + agen = self.api_page_iter(url, json=False) try: async for r in agen: if r is None: @@ -115,7 +115,7 @@ async def get_workflows(self, owner, repo): async def get_workflow_runs(self, owner, repo, workflow_id): runs = [] url = f"{self.base_url}/repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs?status=success&per_page={self.num_logs}" - r = await self.helpers.request(url, headers=self.headers) + r = await self.api_request(url) if r is None: return runs status_code = getattr(r, "status_code", 0) @@ -176,7 +176,7 @@ async def download_run_logs(self, owner, repo, run_id): async def get_run_artifacts(self, owner, repo, run_id): artifacts = [] url = f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts" - r = await self.helpers.request(url, headers=self.headers) + r = await self.api_request(url) if r is None: return artifacts status_code = getattr(r, "status_code", 0) diff --git a/bbot/modules/gitlab.py b/bbot/modules/gitlab.py index dcdc841b5..e1ba3850e 100644 --- a/bbot/modules/gitlab.py +++ b/bbot/modules/gitlab.py @@ -16,10 +16,7 @@ class gitlab(BaseModule): scope_distance_modifier = 2 async def setup(self): - self.headers = {} - self.api_key = self.config.get("api_key", "") - if self.api_key: - self.headers.update({"Authorization": f"Bearer {self.api_key}"}) + await self.require_api_key() return True async def filter_event(self, event): @@ -111,7 +108,7 @@ async def handle_groups_url(self, groups_url, event): await self.handle_namespace(group, event) async def gitlab_json_request(self, url): - response = await self.helpers.request(url, headers=self.headers) + response = await self.api_request(url) if response is not None: try: json = response.json() diff --git a/bbot/modules/hackertarget.py b/bbot/modules/hackertarget.py index aee94ccd3..adfa54458 100644 --- a/bbot/modules/hackertarget.py +++ b/bbot/modules/hackertarget.py @@ -15,7 +15,7 @@ class hackertarget(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}/hostsearch/?q={self.helpers.quote(query)}" - response = await self.request_with_fail_count(url) + response = await self.api_request(url) return response def parse_results(self, r, query): diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index 7396df481..f5a275b41 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -18,8 +18,8 @@ class hunterio(subdomain_enum_apikey): limit = 100 async def ping(self): - url = f"{self.base_url}/account?api_key={self.api_key}" - r = await self.helpers.request(url) + url = f"{self.base_url}/account?api_key={{api_key}}" + r = await self.api_request(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content @@ -56,10 +56,9 @@ async def handle_event(self, event): async def query(self, query): emails = [] url = ( - f"{self.base_url}/domain-search?domain={query}&api_key={self.api_key}" - + "&limit={page_size}&offset={offset}" + f"{self.base_url}/domain-search?domain={query}&api_key={{api_key}}" + "&limit={page_size}&offset={offset}" ) - agen = self.helpers.api_page_iter(url, page_size=self.limit) + agen = self.api_page_iter(url, page_size=self.limit) try: async for j in agen: new_emails = j.get("data", {}).get("emails", []) diff --git a/bbot/modules/internetdb.py b/bbot/modules/internetdb.py index bfafec810..55b613f16 100644 --- a/bbot/modules/internetdb.py +++ b/bbot/modules/internetdb.py @@ -64,7 +64,7 @@ async def handle_event(self, event): if ip is None: return url = f"{self.base_url}/{ip}" - r = await self.request_with_fail_count(url) + r = await self.api_request(url) if r is None: self.debug(f"No response for {event.data}") return diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index af7dd5d94..4118d3471 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -32,12 +32,12 @@ async def setup(self): async def ping(self): url = self.build_url("8.8.8.8") - r = await self.request_with_fail_count(url) + r = await self.api_request(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content def build_url(self, data): - url = f"{self.base_url}/?key={self.api_key}&ip={data}&format=json&source=bbot" + url = f"{self.base_url}/?key={{api_key}}&ip={data}&format=json&source=bbot" if self.lang: url = f"{url}&lang={self.lang}" return url @@ -45,7 +45,7 @@ def build_url(self, data): async def handle_event(self, event): try: url = self.build_url(event.data) - result = await self.request_with_fail_count(url) + result = await self.api_request(url) if result: geo_data = result.json() if not geo_data: diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index 115a620ba..f3caf77f0 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -28,15 +28,15 @@ async def setup(self): return await self.require_api_key() async def ping(self): - url = f"{self.base_url}/check?access_key={self.api_key}" - r = await self.request_with_fail_count(url) + url = f"{self.base_url}/check?access_key={{api_key}}" + r = await self.api_request(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content async def handle_event(self, event): try: - url = f"{self.base_url}/{event.data}?access_key={self.api_key}" - result = await self.request_with_fail_count(url) + url = f"{self.base_url}/{event.data}?access_key={{api_key}}" + result = await self.api_request(url) if result: geo_data = result.json() if not geo_data: diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index 4bc88d3c7..bb63abd80 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -25,15 +25,20 @@ async def setup(self): return await self.require_api_key() return ret + def prepare_api_request(self, url, kwargs): + if self.api_key: + kwargs["headers"]["api-key"] = self.api_key + return url, kwargs + async def ping(self): url = f"{self.base_url}/host/1.2.3.4.5" - r = await self.helpers.request(url, headers=self.headers) + r = await self.helpers.request(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) != 401, resp_content async def request_url(self, query): url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}" - response = await self.request_with_fail_count(url, headers=self.headers) + response = await self.api_request(url) return response def parse_results(self, r, query=None): diff --git a/bbot/modules/myssl.py b/bbot/modules/myssl.py index 047ced928..5c4a8021b 100644 --- a/bbot/modules/myssl.py +++ b/bbot/modules/myssl.py @@ -15,7 +15,7 @@ class myssl(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}?domain={self.helpers.quote(query)}" - return await self.request_with_fail_count(url) + return await self.api_request(url) def parse_results(self, r, query): results = set() diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py index 5a2319e3d..01b65eff5 100644 --- a/bbot/modules/otx.py +++ b/bbot/modules/otx.py @@ -15,7 +15,7 @@ class otx(subdomain_enum): def request_url(self, query): url = f"{self.base_url}/api/v1/indicators/domain/{self.helpers.quote(query)}/passive_dns" - return self.request_with_fail_count(url) + return self.api_request(url) def parse_results(self, r, query): j = r.json() diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py index e22dfc3e5..eb895b0ea 100644 --- a/bbot/modules/passivetotal.py +++ b/bbot/modules/passivetotal.py @@ -24,7 +24,7 @@ async def setup(self): async def ping(self): url = f"{self.base_url}/account/quota" - j = (await self.request_with_fail_count(url, auth=self.auth)).json() + j = (await self.api_request(url, auth=self.auth)).json() limit = j["user"]["limits"]["search_api"] used = j["user"]["counts"]["search_api"] assert used < limit, "No quota remaining" @@ -35,7 +35,7 @@ async def abort_if(self, event): async def request_url(self, query): url = f"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}" - return await self.request_with_fail_count(url, auth=self.auth) + return await self.api_request(url, auth=self.auth) def parse_results(self, r, query): for subdomain in r.json().get("subdomains", []): diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index fa0e13a85..6222e41c0 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -18,9 +18,6 @@ class postman_download(postman): scope_distance_modifier = 2 async def setup(self): - self.api_key = self.config.get("api_key", "") - self.authorization_headers = {"X-Api-Key": self.api_key} - output_folder = self.config.get("output_folder") if output_folder: self.output_dir = Path(output_folder) / "postman_workspaces" @@ -29,9 +26,13 @@ async def setup(self): self.helpers.mkdir(self.output_dir) return await self.require_api_key() + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["X-Api-Key"] = self.api_key + return url, kwargs + async def ping(self): url = f"{self.api_url}/me" - response = await self.helpers.request(url, headers=self.authorization_headers) + response = await self.api_request(url) assert getattr(response, "status_code", 0) == 200, response.text async def filter_event(self, event): @@ -125,7 +126,7 @@ async def request_workspace(self, id): async def get_workspace(self, workspace_id): workspace = {} workspace_url = f"{self.api_url}/workspaces/{workspace_id}" - r = await self.helpers.request(workspace_url, headers=self.authorization_headers) + r = await self.api_request(workspace_url) if r is None: return workspace status_code = getattr(r, "status_code", 0) @@ -155,7 +156,7 @@ async def get_globals(self, workspace_id): async def get_environment(self, environment_id): environment = {} environment_url = f"{self.api_url}/environments/{environment_id}" - r = await self.helpers.request(environment_url, headers=self.authorization_headers) + r = await self.api_request(environment_url) if r is None: return environment status_code = getattr(r, "status_code", 0) @@ -170,7 +171,7 @@ async def get_environment(self, environment_id): async def get_collection(self, collection_id): collection = {} collection_url = f"{self.api_url}/collections/{collection_id}" - r = await self.helpers.request(collection_url, headers=self.authorization_headers) + r = await self.api_request(collection_url) if r is None: return collection status_code = getattr(r, "status_code", 0) diff --git a/bbot/modules/rapiddns.py b/bbot/modules/rapiddns.py index 934beb829..ad680131a 100644 --- a/bbot/modules/rapiddns.py +++ b/bbot/modules/rapiddns.py @@ -15,7 +15,7 @@ class rapiddns(subdomain_enum): async def request_url(self, query): url = f"{self.base_url}/subdomain/{self.helpers.quote(query)}?full=1#result" - response = await self.request_with_fail_count(url, timeout=self.http_timeout + 10) + response = await self.api_request(url, timeout=self.http_timeout + 10) return response def parse_results(self, r, query): diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index a91db2912..8a24ed1b6 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -21,14 +21,14 @@ async def setup(self): return await super().setup() async def ping(self): - url = f"{self.base_url}/ping?apikey={self.api_key}" - r = await self.request_with_fail_count(url) + url = f"{self.base_url}/ping?apikey={{api_key}}" + r = await self.api_request(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content async def request_url(self, query): - url = f"{self.base_url}/domain/{query}/subdomains?apikey={self.api_key}" - response = await self.request_with_fail_count(url) + url = f"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}" + response = await self.api_request(url) return response def parse_results(self, r, query): diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index 2336e0243..9814ccb5c 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -17,8 +17,8 @@ class shodan_dns(shodan): base_url = "https://api.shodan.io" async def request_url(self, query): - url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={self.api_key}" - response = await self.request_with_fail_count(url) + url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}" + response = await self.api_request(url) return response def parse_results(self, r, query): diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py index 4aac709b3..ce1075ae9 100644 --- a/bbot/modules/skymem.py +++ b/bbot/modules/skymem.py @@ -24,7 +24,7 @@ async def handle_event(self, event): _, query = self.helpers.split_domain(event.data) # get first page url = f"{self.base_url}/srch?q={self.helpers.quote(query)}" - r = await self.request_with_fail_count(url) + r = await self.api_request(url) if not r: return responses = [r] @@ -34,7 +34,7 @@ async def handle_event(self, event): if domain_ids: domain_id = domain_ids[0] for page in range(2, 22): - r2 = await self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}") + r2 = await self.api_request(f"{self.base_url}/domain/{domain_id}?p={page}") if not r2: continue responses.append(r2) diff --git a/bbot/modules/templates/github.py b/bbot/modules/templates/github.py index 4169647ce..a5eb17ee6 100644 --- a/bbot/modules/templates/github.py +++ b/bbot/modules/templates/github.py @@ -10,20 +10,27 @@ class github(BaseModule): _qsize = 1 base_url = "https://api.github.com" + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["Authorization"] = f"token {self.api_key}" + return url, kwargs + async def setup(self): await super().setup() - self.api_key = None self.headers = {} + api_keys = set() for module_name in ("github", "github_codesearch", "github_org", "git_clone"): module_config = self.scan.config.get("modules", {}).get(module_name, {}) api_key = module_config.get("api_key", "") - if api_key: - self.api_key = api_key - self.headers = {"Authorization": f"token {self.api_key}"} - break - if not self.api_key: + if isinstance(api_key, str): + api_key = [api_key] + for key in api_key: + key = key.strip() + if key: + api_keys.update(key) + if not api_keys: if self.auth_required: return None, "No API key set" + self.api_key = api_keys try: await self.ping() self.hugesuccess(f"API is ready") diff --git a/bbot/modules/templates/shodan.py b/bbot/modules/templates/shodan.py index f439bfd9d..42b3e5afe 100644 --- a/bbot/modules/templates/shodan.py +++ b/bbot/modules/templates/shodan.py @@ -9,16 +9,20 @@ class shodan(subdomain_enum): async def setup(self): await super().setup() - self.api_key = None + api_keys = set() for module_name in ("shodan", "shodan_dns", "shodan_port"): module_config = self.scan.config.get("modules", {}).get(module_name, {}) api_key = module_config.get("api_key", "") - if api_key: - self.api_key = api_key - break - if not self.api_key: + if isinstance(api_key, str): + api_key = [api_key] + for key in api_key: + key = key.strip() + if key: + api_keys.add(key) + if not api_keys: if self.auth_required: return None, "No API key set" + self.api_key = api_keys try: await self.ping() self.hugesuccess(f"API is ready") @@ -29,6 +33,6 @@ async def setup(self): async def ping(self): url = f"{self.base_url}/api-info?key={self.api_key}" - r = await self.request_with_fail_count(url) + r = await self.api_request(url) resp_content = getattr(r, "text", "") assert getattr(r, "status_code", 0) == 200, resp_content diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py index e1f39550f..b786d8048 100644 --- a/bbot/modules/trickest.py +++ b/bbot/modules/trickest.py @@ -22,10 +22,13 @@ class Trickest(subdomain_enum_apikey): dataset_id = "a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc" page_size = 50 + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["Authorization"] = f"Token {self.api_key}" + async def ping(self): - self.headers = {"Authorization": f"Token {self.api_key}"} + url = f"{self.base_url}/dataset" - response = await self.helpers.request(url, headers=self.headers) + response = await self.api_request(url) status_code = getattr(response, "status_code", 0) if status_code != 200: response_text = getattr(response, "text", "no response from server") @@ -54,7 +57,7 @@ async def query(self, query): url = f"{self.base_url}/view?q=hostname%20~%20%22.{self.helpers.quote(query)}%22" url += f"&dataset_id={self.dataset_id}" url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname" - agen = self.helpers.api_page_iter(url, headers=self.headers, page_size=self.page_size) + agen = self.api_page_iter(url, page_size=self.page_size) try: async for response in agen: subdomains = self.parse_results(response) diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index 6f0ba5e82..87e82766d 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -16,15 +16,14 @@ class virustotal(subdomain_enum_apikey): base_url = "https://www.virustotal.com/api/v3" - async def setup(self): - self.api_key = self.config.get("api_key", "") - self.headers = {"x-apikey": self.api_key} - return await super().setup() - async def ping(self): # virustotal does not have a ping function return + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["x-apikey"] = self.api_key + return url, kwargs + def parse_results(self, r, query): results = set() text = getattr(r, "text", "") @@ -37,9 +36,7 @@ def parse_results(self, r, query): async def query(self, query): results = set() url = f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains" - agen = self.helpers.api_page_iter( - url, json=False, headers=self.headers, next_key=lambda r: r.json().get("links", {}).get("next", "") - ) + agen = self.api_page_iter(url, json=False, next_key=lambda r: r.json().get("links", {}).get("next", "")) try: async for response in agen: r = self.parse_results(response, query) diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py index fc4cfbfee..2bf7789d5 100644 --- a/bbot/modules/zoomeye.py +++ b/bbot/modules/zoomeye.py @@ -22,13 +22,15 @@ class zoomeye(subdomain_enum_apikey): async def setup(self): self.max_pages = self.config.get("max_pages", 20) - self.headers = {"API-KEY": self.config.get("api_key", "")} self.include_related = self.config.get("include_related", False) return await super().setup() + def prepare_api_request(self, url, kwargs): + kwargs["headers"]["API-KEY"] = self.api_key + async def ping(self): url = f"{self.base_url}/resources-info" - r = await self.helpers.request(url, headers=self.headers) + r = await self.api_request(url) assert int(r.json()["quota_info"]["remain_total_quota"]) > 0, "No quota remaining" async def handle_event(self, event): @@ -54,7 +56,7 @@ async def query(self, query): query_type = 0 if self.include_related else 1 url = f"{self.base_url}/domain/search?q={self.helpers.quote(query)}&type={query_type}&page=" + "{page}" i = 0 - agen = self.helpers.api_page_iter(url, headers=self.headers) + agen = self.api_page_iter(url) try: async for j in agen: r = list(self.parse_results(j)) diff --git a/bbot/test/test_step_2/module_tests/test_module_gitlab.py b/bbot/test/test_step_2/module_tests/test_module_gitlab.py index 0d60d9a11..6d593adf6 100644 --- a/bbot/test/test_step_2/module_tests/test_module_gitlab.py +++ b/bbot/test/test_step_2/module_tests/test_module_gitlab.py @@ -4,10 +4,13 @@ class TestGitlab(ModuleTestBase): targets = ["http://127.0.0.1:8888"] modules_overrides = ["gitlab", "httpx"] + config_overrides = {"modules": {"gitlab": {"api_key": "asdf"}}} async def setup_before_prep(self, module_test): module_test.httpserver.expect_request("/").respond_with_data(headers={"X-Gitlab-Meta": "asdf"}) - module_test.httpserver.expect_request("/api/v4/projects", query_string="simple=true").respond_with_json( + module_test.httpserver.expect_request( + "/api/v4/projects", query_string="simple=true", headers={"Authorization": "Bearer asdf"} + ).respond_with_json( [ { "id": 33, @@ -41,7 +44,9 @@ async def setup_before_prep(self, module_test): }, ], ) - module_test.httpserver.expect_request("/api/v4/groups", query_string="simple=true").respond_with_json( + module_test.httpserver.expect_request( + "/api/v4/groups", query_string="simple=true", headers={"Authorization": "Bearer asdf"} + ).respond_with_json( [ { "id": 9, @@ -84,7 +89,7 @@ async def setup_before_prep(self, module_test): ] ) module_test.httpserver.expect_request( - "/api/v4/groups/bbotgroup/projects", query_string="simple=true" + "/api/v4/groups/bbotgroup/projects", query_string="simple=true", headers={"Authorization": "Bearer asdf"} ).respond_with_json( [ { @@ -120,7 +125,7 @@ async def setup_before_prep(self, module_test): ] ) module_test.httpserver.expect_request( - "/api/v4/users/bbotgroup/projects", query_string="simple=true" + "/api/v4/users/bbotgroup/projects", query_string="simple=true", headers={"Authorization": "Bearer asdf"} ).respond_with_json( [ { diff --git a/bbot/test/test_step_2/module_tests/test_module_postman_download.py b/bbot/test/test_step_2/module_tests/test_module_postman_download.py index 5c731eef2..f937ebc20 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman_download.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman_download.py @@ -8,6 +8,7 @@ class TestPostman_Download(ModuleTestBase): async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( url="https://api.getpostman.com/me", + match_headers={"X-Api-Key": "asdf"}, json={ "user": { "id": 000000, @@ -236,6 +237,7 @@ async def setup_after_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", + match_headers={"X-Api-Key": "asdf"}, json={ "workspace": { "id": "3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b", @@ -330,6 +332,7 @@ async def setup_after_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089", + match_headers={"X-Api-Key": "asdf"}, json={ "environment": { "id": "f770f816-9c6a-40f7-bde3-c0855d2a1089", @@ -350,6 +353,7 @@ async def setup_after_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f", + match_headers={"X-Api-Key": "asdf"}, json={ "collection": { "info": { From 87fed35e003203513ed6c330f019bfa6f23fcf1b Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 30 Sep 2024 15:41:26 -0400 Subject: [PATCH 141/254] log messages --- bbot/modules/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index e98e630ac..023cf0966 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -328,7 +328,10 @@ def api_key(self, api_keys): def cycle_api_key(self): if self._api_keys: + self.verbose(f"Cycling API key") self._api_keys.insert(0, self._api_keys.pop()) + else: + self.debug(f"No extra API keys to cycle") @property def failed_request_abort_threshold(self): From d953f705c590be0441f7133a482400d0886af59c Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 1 Oct 2024 09:51:26 -0400 Subject: [PATCH 142/254] fix race condition in tests --- bbot/test/test_step_1/test_manager_scope_accuracy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index f860f1746..1d1f8d3ca 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -114,8 +114,9 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) await scan.helpers.dns._mock_dns(_dns_mock) if scan_callback is not None: scan_callback(scan) + output_events = [e async for e in scan.async_start()] return ( - [e async for e in scan.async_start()], + output_events, dummy_module.events, dummy_module_nodupes.events, dummy_graph_output_module.events, From 39d9a077b102f4fb9341f2ab20a5f0ba63615f21 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 10:16:23 -0400 Subject: [PATCH 143/254] bumping baddns version --- bbot/modules/baddns.py | 4 ++-- bbot/modules/baddns_direct.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 84a5b6fed..3e3a0dcb5 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -22,7 +22,7 @@ class baddns(BaseModule): "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", } module_threads = 8 - deps_pip = ["baddns~=1.1.855"] + deps_pip = ["baddns~=1.1.862"] def select_modules(self): selected_submodules = [] @@ -80,7 +80,7 @@ async def handle_event(self, event): try: task_result = await completed_task except Exception as e: - self.hugewarning(f"Task for {module_instance} raised an error: {e}") + self.warning(f"Task for {module_instance} raised an error: {e}") task_result = None if task_result: diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 8ba6e79b2..cc4272d69 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -19,7 +19,7 @@ class baddns_direct(BaseModule): "custom_nameservers": "Force BadDNS to use a list of custom nameservers", } module_threads = 8 - deps_pip = ["baddns~=1.1.855"] + deps_pip = ["baddns~=1.1.862"] scope_distance_modifier = 1 From 8a863a6a644af2a5d0216740be67cf455738bf7a Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 1 Oct 2024 10:56:28 -0400 Subject: [PATCH 144/254] api page iter --- bbot/modules/dehashed.py | 2 +- bbot/modules/dockerhub.py | 2 +- bbot/test/test_step_1/test_web.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 8913ca403..d8ef7d003 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -90,7 +90,7 @@ async def query(self, domain): url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}" page = 0 num_entries = 0 - agen = self.helpers.api_page_iter(url=url, auth=self.auth, headers=self.headers, json=False) + agen = self.api_page_iter(url=url, auth=self.auth, headers=self.headers, json=False) async for result in agen: result_json = {} with suppress(Exception): diff --git a/bbot/modules/dockerhub.py b/bbot/modules/dockerhub.py index c9c206d7a..b64b88705 100644 --- a/bbot/modules/dockerhub.py +++ b/bbot/modules/dockerhub.py @@ -64,7 +64,7 @@ async def handle_social(self, event): async def get_repos(self, username): repos = [] url = f"{self.api_url}/repositories/{username}?page_size=25&page=" + "{page}" - agen = self.helpers.api_page_iter(url, json=False) + agen = self.api_page_iter(url, json=False) try: async for r in agen: if r is None: diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index f282b1d62..c4e02ccb8 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -252,7 +252,7 @@ async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock): uri=f"{base_path}/3", query_string={"page_size": "100", "offset": "200"} ).respond_with_data("page3") results = [] - agen = scan1.helpers.api_page_iter(template_url) + agen = scan1.api_page_iter(template_url) try: async for result in agen: if result and result.text.startswith("page"): @@ -262,7 +262,7 @@ async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock): finally: await agen.aclose() assert not results - agen = scan1.helpers.api_page_iter(template_url, json=False) + agen = scan1.api_page_iter(template_url, json=False) try: async for result in agen: if result and result.text.startswith("page"): From 786105b200f2bb842621ef8012e13010e54b62a5 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 11:02:40 -0400 Subject: [PATCH 145/254] baddns_zone update version --- bbot/modules/baddns_zone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index 64f795845..26b427451 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -16,7 +16,7 @@ class baddns_zone(baddns_module): "only_high_confidence": "Do not emit low-confidence or generic detections", } module_threads = 8 - deps_pip = ["baddns~=1.1.855"] + deps_pip = ["baddns~=1.1.862"] def set_modules(self): self.enabled_submodules = ["NSEC", "zonetransfer"] From 87e83c02427a0721d7badffad2031cc94c1e68af Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 11:02:51 -0400 Subject: [PATCH 146/254] update poetry.lock --- poetry.lock | 663 ++++++++++++++++++++++++++++------------------------ 1 file changed, 353 insertions(+), 310 deletions(-) diff --git a/poetry.lock b/poetry.lock index f3238f135..75b27e134 100644 --- a/poetry.lock +++ b/poetry.lock @@ -74,13 +74,13 @@ files = [ [[package]] name = "anyio" -version = "4.4.0" +version = "4.6.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"}, - {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"}, + {file = "anyio-4.6.0-py3-none-any.whl", hash = "sha256:c7d2e9d63e31599eeb636c8c5c03a7e108d73b345f064f1c19fdc87b79036a9a"}, + {file = "anyio-4.6.0.tar.gz", hash = "sha256:137b4559cbb034c477165047febb6ff83f390fc3b20bf181c1fc0a728cb8beeb"}, ] [package.dependencies] @@ -90,19 +90,19 @@ sniffio = ">=1.1" typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} [package.extras] -doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] -trio = ["trio (>=0.23)"] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.21.0b1)"] +trio = ["trio (>=0.26.1)"] [[package]] name = "babel" -version = "2.15.0" +version = "2.16.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" files = [ - {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, - {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, + {file = "babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b"}, + {file = "babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"}, ] [package.extras] @@ -188,74 +188,89 @@ files = [ [[package]] name = "certifi" -version = "2024.6.2" +version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, - {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] name = "cffi" -version = "1.16.0" +version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" files = [ - {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, - {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, - {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, - {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, - {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, - {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, - {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, - {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, - {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, - {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, - {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, - {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, - {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, - {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, - {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, + {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, + {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, + {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, + {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, + {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, + {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, + {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, + {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, + {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, + {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, + {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] [package.dependencies] @@ -415,63 +430,83 @@ files = [ [[package]] name = "coverage" -version = "7.6.0" +version = "7.6.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dff044f661f59dace805eedb4a7404c573b6ff0cdba4a524141bc63d7be5c7fd"}, - {file = "coverage-7.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8659fd33ee9e6ca03950cfdcdf271d645cf681609153f218826dd9805ab585c"}, - {file = "coverage-7.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7792f0ab20df8071d669d929c75c97fecfa6bcab82c10ee4adb91c7a54055463"}, - {file = "coverage-7.6.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d4b3cd1ca7cd73d229487fa5caca9e4bc1f0bca96526b922d61053ea751fe791"}, - {file = "coverage-7.6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7e128f85c0b419907d1f38e616c4f1e9f1d1b37a7949f44df9a73d5da5cd53c"}, - {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a94925102c89247530ae1dab7dc02c690942566f22e189cbd53579b0693c0783"}, - {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:dcd070b5b585b50e6617e8972f3fbbee786afca71b1936ac06257f7e178f00f6"}, - {file = "coverage-7.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d50a252b23b9b4dfeefc1f663c568a221092cbaded20a05a11665d0dbec9b8fb"}, - {file = "coverage-7.6.0-cp310-cp310-win32.whl", hash = "sha256:0e7b27d04131c46e6894f23a4ae186a6a2207209a05df5b6ad4caee6d54a222c"}, - {file = "coverage-7.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:54dece71673b3187c86226c3ca793c5f891f9fc3d8aa183f2e3653da18566169"}, - {file = "coverage-7.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7b525ab52ce18c57ae232ba6f7010297a87ced82a2383b1afd238849c1ff933"}, - {file = "coverage-7.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bea27c4269234e06f621f3fac3925f56ff34bc14521484b8f66a580aacc2e7d"}, - {file = "coverage-7.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed8d1d1821ba5fc88d4a4f45387b65de52382fa3ef1f0115a4f7a20cdfab0e94"}, - {file = "coverage-7.6.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01c322ef2bbe15057bc4bf132b525b7e3f7206f071799eb8aa6ad1940bcf5fb1"}, - {file = "coverage-7.6.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03cafe82c1b32b770a29fd6de923625ccac3185a54a5e66606da26d105f37dac"}, - {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0d1b923fc4a40c5832be4f35a5dab0e5ff89cddf83bb4174499e02ea089daf57"}, - {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4b03741e70fb811d1a9a1d75355cf391f274ed85847f4b78e35459899f57af4d"}, - {file = "coverage-7.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a73d18625f6a8a1cbb11eadc1d03929f9510f4131879288e3f7922097a429f63"}, - {file = "coverage-7.6.0-cp311-cp311-win32.whl", hash = "sha256:65fa405b837060db569a61ec368b74688f429b32fa47a8929a7a2f9b47183713"}, - {file = "coverage-7.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:6379688fb4cfa921ae349c76eb1a9ab26b65f32b03d46bb0eed841fd4cb6afb1"}, - {file = "coverage-7.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f7db0b6ae1f96ae41afe626095149ecd1b212b424626175a6633c2999eaad45b"}, - {file = "coverage-7.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bbdf9a72403110a3bdae77948b8011f644571311c2fb35ee15f0f10a8fc082e8"}, - {file = "coverage-7.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc44bf0315268e253bf563f3560e6c004efe38f76db03a1558274a6e04bf5d5"}, - {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da8549d17489cd52f85a9829d0e1d91059359b3c54a26f28bec2c5d369524807"}, - {file = "coverage-7.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0086cd4fc71b7d485ac93ca4239c8f75732c2ae3ba83f6be1c9be59d9e2c6382"}, - {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1fad32ee9b27350687035cb5fdf9145bc9cf0a094a9577d43e909948ebcfa27b"}, - {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:044a0985a4f25b335882b0966625270a8d9db3d3409ddc49a4eb00b0ef5e8cee"}, - {file = "coverage-7.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:76d5f82213aa78098b9b964ea89de4617e70e0d43e97900c2778a50856dac605"}, - {file = "coverage-7.6.0-cp312-cp312-win32.whl", hash = "sha256:3c59105f8d58ce500f348c5b56163a4113a440dad6daa2294b5052a10db866da"}, - {file = "coverage-7.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca5d79cfdae420a1d52bf177de4bc2289c321d6c961ae321503b2ca59c17ae67"}, - {file = "coverage-7.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d39bd10f0ae453554798b125d2f39884290c480f56e8a02ba7a6ed552005243b"}, - {file = "coverage-7.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:beb08e8508e53a568811016e59f3234d29c2583f6b6e28572f0954a6b4f7e03d"}, - {file = "coverage-7.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2e16f4cd2bc4d88ba30ca2d3bbf2f21f00f382cf4e1ce3b1ddc96c634bc48ca"}, - {file = "coverage-7.6.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6616d1c9bf1e3faea78711ee42a8b972367d82ceae233ec0ac61cc7fec09fa6b"}, - {file = "coverage-7.6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad4567d6c334c46046d1c4c20024de2a1c3abc626817ae21ae3da600f5779b44"}, - {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d17c6a415d68cfe1091d3296ba5749d3d8696e42c37fca5d4860c5bf7b729f03"}, - {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9146579352d7b5f6412735d0f203bbd8d00113a680b66565e205bc605ef81bc6"}, - {file = "coverage-7.6.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:cdab02a0a941af190df8782aafc591ef3ad08824f97850b015c8c6a8b3877b0b"}, - {file = "coverage-7.6.0-cp38-cp38-win32.whl", hash = "sha256:df423f351b162a702c053d5dddc0fc0ef9a9e27ea3f449781ace5f906b664428"}, - {file = "coverage-7.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:f2501d60d7497fd55e391f423f965bbe9e650e9ffc3c627d5f0ac516026000b8"}, - {file = "coverage-7.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7221f9ac9dad9492cecab6f676b3eaf9185141539d5c9689d13fd6b0d7de840c"}, - {file = "coverage-7.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ddaaa91bfc4477d2871442bbf30a125e8fe6b05da8a0015507bfbf4718228ab2"}, - {file = "coverage-7.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4cbe651f3904e28f3a55d6f371203049034b4ddbce65a54527a3f189ca3b390"}, - {file = "coverage-7.6.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:831b476d79408ab6ccfadaaf199906c833f02fdb32c9ab907b1d4aa0713cfa3b"}, - {file = "coverage-7.6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46c3d091059ad0b9c59d1034de74a7f36dcfa7f6d3bde782c49deb42438f2450"}, - {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4d5fae0a22dc86259dee66f2cc6c1d3e490c4a1214d7daa2a93d07491c5c04b6"}, - {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:07ed352205574aad067482e53dd606926afebcb5590653121063fbf4e2175166"}, - {file = "coverage-7.6.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:49c76cdfa13015c4560702574bad67f0e15ca5a2872c6a125f6327ead2b731dd"}, - {file = "coverage-7.6.0-cp39-cp39-win32.whl", hash = "sha256:482855914928c8175735a2a59c8dc5806cf7d8f032e4820d52e845d1f731dca2"}, - {file = "coverage-7.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:543ef9179bc55edfd895154a51792b01c017c87af0ebaae092720152e19e42ca"}, - {file = "coverage-7.6.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:6fe885135c8a479d3e37a7aae61cbd3a0fb2deccb4dda3c25f92a49189f766d6"}, - {file = "coverage-7.6.0.tar.gz", hash = "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51"}, + {file = "coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16"}, + {file = "coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959"}, + {file = "coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232"}, + {file = "coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0"}, + {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, + {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, + {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, + {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, + {file = "coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778"}, + {file = "coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d"}, + {file = "coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5"}, + {file = "coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb"}, + {file = "coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106"}, + {file = "coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155"}, + {file = "coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a"}, + {file = "coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129"}, + {file = "coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e"}, + {file = "coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3"}, + {file = "coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f"}, + {file = "coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657"}, + {file = "coverage-7.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0"}, + {file = "coverage-7.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989"}, + {file = "coverage-7.6.1-cp38-cp38-win32.whl", hash = "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7"}, + {file = "coverage-7.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8"}, + {file = "coverage-7.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255"}, + {file = "coverage-7.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36"}, + {file = "coverage-7.6.1-cp39-cp39-win32.whl", hash = "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c"}, + {file = "coverage-7.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca"}, + {file = "coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df"}, + {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, ] [package.dependencies] @@ -482,43 +517,38 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "42.0.8" +version = "43.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, - {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, - {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, - {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, - {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, - {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, - {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, + {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, + {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, + {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, + {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, + {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, + {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, + {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, ] [package.dependencies] @@ -531,7 +561,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -596,13 +626,13 @@ files = [ [[package]] name = "dunamai" -version = "1.21.1" +version = "1.22.0" description = "Dynamic version generation" optional = false python-versions = ">=3.5" files = [ - {file = "dunamai-1.21.1-py3-none-any.whl", hash = "sha256:fe303541463648b8197c495decf62cd8f15234fb6d891a5f295015e452f656c8"}, - {file = "dunamai-1.21.1.tar.gz", hash = "sha256:d7fea28ad2faf20a6ca5ec121e5c68e55eec6b8ada23d9c387e4e7a574cc559f"}, + {file = "dunamai-1.22.0-py3-none-any.whl", hash = "sha256:eab3894b31e145bd028a74b13491c57db01986a7510482c9b5fff3b4e53d77b7"}, + {file = "dunamai-1.22.0.tar.gz", hash = "sha256:375a0b21309336f0d8b6bbaea3e038c36f462318c68795166e31f9873fdad676"}, ] [package.dependencies] @@ -610,13 +640,13 @@ packaging = ">=20.9" [[package]] name = "exceptiongroup" -version = "1.2.1" +version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] [package.extras] @@ -624,34 +654,34 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.14.0" +version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.14.0-py3-none-any.whl", hash = "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f"}, - {file = "filelock-3.14.0.tar.gz", hash = "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"}, + {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, + {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] [package.extras] -docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] -typing = ["typing-extensions (>=4.8)"] +docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] +typing = ["typing-extensions (>=4.12.2)"] [[package]] name = "flake8" -version = "7.0.0" +version = "7.1.1" description = "the modular source code checker: pep8 pyflakes and co" optional = false python-versions = ">=3.8.1" files = [ - {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, - {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, + {file = "flake8-7.1.1-py2.py3-none-any.whl", hash = "sha256:597477df7860daa5aa0fdd84bf5208a043ab96b8e96ab708770ae0364dd03213"}, + {file = "flake8-7.1.1.tar.gz", hash = "sha256:049d058491e228e03e67b390f311bbf88fce2dbaa8fa673e7aea87b7198b8d38"}, ] [package.dependencies] mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.11.0,<2.12.0" +pycodestyle = ">=2.12.0,<2.13.0" pyflakes = ">=3.2.0,<3.3.0" [[package]] @@ -673,13 +703,13 @@ dev = ["flake8", "markdown", "twine", "wheel"] [[package]] name = "griffe" -version = "1.1.0" +version = "1.3.2" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." optional = false python-versions = ">=3.8" files = [ - {file = "griffe-1.1.0-py3-none-any.whl", hash = "sha256:38ccc5721571c95ae427123074cf0dc0d36bce7c9701ab2ada9fe0566ff50c10"}, - {file = "griffe-1.1.0.tar.gz", hash = "sha256:c6328cbdec0d449549c1cc332f59227cd5603f903479d73e4425d828b782ffc3"}, + {file = "griffe-1.3.2-py3-none-any.whl", hash = "sha256:2e34b5e46507d615915c8e6288bb1a2234bd35dee44d01e40a2bc2f25bd4d10c"}, + {file = "griffe-1.3.2.tar.gz", hash = "sha256:1ec50335aa507ed2445f2dd45a15c9fa3a45f52c9527e880571dfc61912fd60c"}, ] [package.dependencies] @@ -744,13 +774,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "identify" -version = "2.5.36" +version = "2.6.1" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"}, - {file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"}, + {file = "identify-2.6.1-py2.py3-none-any.whl", hash = "sha256:53863bcac7caf8d2ed85bd20312ea5dcfc22226800f6d6881f232d861db5a8f0"}, + {file = "identify-2.6.1.tar.gz", hash = "sha256:91478c5fb7c3aac5ff7bf9b4344f803843dc586832d5f110d672b19aa1984c98"}, ] [package.extras] @@ -849,18 +879,17 @@ files = [ [[package]] name = "livereload" -version = "2.6.3" +version = "2.7.0" description = "Python LiveReload is an awesome tool for web developers" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "livereload-2.6.3-py2.py3-none-any.whl", hash = "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4"}, - {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, + {file = "livereload-2.7.0-py3-none-any.whl", hash = "sha256:19bee55aff51d5ade6ede0dc709189a0f904d3b906d3ea71641ed548acff3246"}, + {file = "livereload-2.7.0.tar.gz", hash = "sha256:f4ba199ef93248902841e298670eebfe1aa9e148e19b343bc57dbf1b74de0513"}, ] [package.dependencies] -six = "*" -tornado = {version = "*", markers = "python_version > \"2.7\""} +tornado = "*" [[package]] name = "lockfile" @@ -1029,13 +1058,13 @@ source = ["Cython (>=3.0.11)"] [[package]] name = "markdown" -version = "3.6" +version = "3.7" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.8" files = [ - {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"}, - {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, + {file = "Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803"}, + {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, ] [package.dependencies] @@ -1502,14 +1531,19 @@ files = [ [[package]] name = "paginate" -version = "0.5.6" +version = "0.5.7" description = "Divides large result sets into pages for easier browsing" optional = false python-versions = "*" files = [ - {file = "paginate-0.5.6.tar.gz", hash = "sha256:5e6007b6a9398177a7e1648d04fdd9f8c9766a1a945bceac82f1929e8c78af2d"}, + {file = "paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591"}, + {file = "paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945"}, ] +[package.extras] +dev = ["pytest", "tox"] +lint = ["black"] + [[package]] name = "pathspec" version = "0.12.1" @@ -1537,19 +1571,19 @@ ptyprocess = ">=0.5" [[package]] name = "platformdirs" -version = "4.2.2" +version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, - {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, ] [package.extras] -docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] -type = ["mypy (>=1.8)"] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] [[package]] name = "pluggy" @@ -1587,13 +1621,13 @@ plugin = ["poetry (>=1.2.0,<2.0.0)"] [[package]] name = "pre-commit" -version = "3.7.1" +version = "3.8.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" files = [ - {file = "pre_commit-3.7.1-py2.py3-none-any.whl", hash = "sha256:fae36fd1d7ad7d6a5a1c0b0d5adb2ed1a3bda5a21bf6c3e5372073d7a11cd4c5"}, - {file = "pre_commit-3.7.1.tar.gz", hash = "sha256:8ca3ad567bc78a4972a3f1a477e94a79d4597e8140a6e0b651c5e33899c3654a"}, + {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, + {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, ] [package.dependencies] @@ -1644,13 +1678,13 @@ files = [ [[package]] name = "pycodestyle" -version = "2.11.1" +version = "2.12.1" description = "Python style guide checker" optional = false python-versions = ">=3.8" files = [ - {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, - {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, + {file = "pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3"}, + {file = "pycodestyle-2.12.1.tar.gz", hash = "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521"}, ] [[package]] @@ -1873,13 +1907,13 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pymdown-extensions" -version = "10.8.1" +version = "10.11.1" description = "Extension pack for Python Markdown." optional = false python-versions = ">=3.8" files = [ - {file = "pymdown_extensions-10.8.1-py3-none-any.whl", hash = "sha256:f938326115884f48c6059c67377c46cf631c733ef3629b6eed1349989d1b30cb"}, - {file = "pymdown_extensions-10.8.1.tar.gz", hash = "sha256:3ab1db5c9e21728dabf75192d71471f8e50f216627e9a1fa9535ecb0231b9940"}, + {file = "pymdown_extensions-10.11.1-py3-none-any.whl", hash = "sha256:a2b28f5786e041f19cb5bb30a1c2c853668a7099da8e3dd822a5ad05f2e855e3"}, + {file = "pymdown_extensions-10.11.1.tar.gz", hash = "sha256:a8836e955851542fa2625d04d59fdf97125ca001377478ed5618e04f9183a59a"}, ] [package.dependencies] @@ -1891,13 +1925,13 @@ extra = ["pygments (>=2.12)"] [[package]] name = "pyparsing" -version = "3.1.2" +version = "3.1.4" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, - {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, + {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, + {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, ] [package.extras] @@ -2076,62 +2110,64 @@ six = ">=1.5" [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] [[package]] @@ -2537,18 +2573,23 @@ test = ["pytest"] [[package]] name = "setuptools" -version = "70.0.0" +version = "75.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, - {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, + {file = "setuptools-75.1.0-py3-none-any.whl", hash = "sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2"}, + {file = "setuptools-75.1.0.tar.gz", hash = "sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] [[package]] name = "six" @@ -2585,13 +2626,13 @@ files = [ [[package]] name = "soupsieve" -version = "2.5" +version = "2.6" description = "A modern CSS selector implementation for Beautiful Soup." optional = false python-versions = ">=3.8" files = [ - {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, - {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, ] [[package]] @@ -2642,13 +2683,13 @@ files = [ [[package]] name = "tomlkit" -version = "0.12.5" +version = "0.13.2" description = "Style preserving TOML library" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tomlkit-0.12.5-py3-none-any.whl", hash = "sha256:af914f5a9c59ed9d0762c7b64d3b5d5df007448eb9cd2edc8a46b1eafead172f"}, - {file = "tomlkit-0.12.5.tar.gz", hash = "sha256:eef34fba39834d4d6b73c9ba7f3e4d1c417a4e56f89a7e96e090dd0d24b8fb3c"}, + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] [[package]] @@ -2726,13 +2767,13 @@ test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] [[package]] name = "virtualenv" -version = "20.26.2" +version = "20.26.6" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.26.2-py3-none-any.whl", hash = "sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b"}, - {file = "virtualenv-20.26.2.tar.gz", hash = "sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c"}, + {file = "virtualenv-20.26.6-py3-none-any.whl", hash = "sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2"}, + {file = "virtualenv-20.26.6.tar.gz", hash = "sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48"}, ] [package.dependencies] @@ -2746,43 +2787,41 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [[package]] name = "watchdog" -version = "4.0.1" +version = "5.0.3" description = "Filesystem events monitoring" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "watchdog-4.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:da2dfdaa8006eb6a71051795856bedd97e5b03e57da96f98e375682c48850645"}, - {file = "watchdog-4.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e93f451f2dfa433d97765ca2634628b789b49ba8b504fdde5837cdcf25fdb53b"}, - {file = "watchdog-4.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ef0107bbb6a55f5be727cfc2ef945d5676b97bffb8425650dadbb184be9f9a2b"}, - {file = "watchdog-4.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:17e32f147d8bf9657e0922c0940bcde863b894cd871dbb694beb6704cfbd2fb5"}, - {file = "watchdog-4.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03e70d2df2258fb6cb0e95bbdbe06c16e608af94a3ffbd2b90c3f1e83eb10767"}, - {file = "watchdog-4.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:123587af84260c991dc5f62a6e7ef3d1c57dfddc99faacee508c71d287248459"}, - {file = "watchdog-4.0.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:093b23e6906a8b97051191a4a0c73a77ecc958121d42346274c6af6520dec175"}, - {file = "watchdog-4.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:611be3904f9843f0529c35a3ff3fd617449463cb4b73b1633950b3d97fa4bfb7"}, - {file = "watchdog-4.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:62c613ad689ddcb11707f030e722fa929f322ef7e4f18f5335d2b73c61a85c28"}, - {file = "watchdog-4.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d4925e4bf7b9bddd1c3de13c9b8a2cdb89a468f640e66fbfabaf735bd85b3e35"}, - {file = "watchdog-4.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cad0bbd66cd59fc474b4a4376bc5ac3fc698723510cbb64091c2a793b18654db"}, - {file = "watchdog-4.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a3c2c317a8fb53e5b3d25790553796105501a235343f5d2bf23bb8649c2c8709"}, - {file = "watchdog-4.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c9904904b6564d4ee8a1ed820db76185a3c96e05560c776c79a6ce5ab71888ba"}, - {file = "watchdog-4.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:667f3c579e813fcbad1b784db7a1aaa96524bed53437e119f6a2f5de4db04235"}, - {file = "watchdog-4.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d10a681c9a1d5a77e75c48a3b8e1a9f2ae2928eda463e8d33660437705659682"}, - {file = "watchdog-4.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0144c0ea9997b92615af1d94afc0c217e07ce2c14912c7b1a5731776329fcfc7"}, - {file = "watchdog-4.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:998d2be6976a0ee3a81fb8e2777900c28641fb5bfbd0c84717d89bca0addcdc5"}, - {file = "watchdog-4.0.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e7921319fe4430b11278d924ef66d4daa469fafb1da679a2e48c935fa27af193"}, - {file = "watchdog-4.0.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:f0de0f284248ab40188f23380b03b59126d1479cd59940f2a34f8852db710625"}, - {file = "watchdog-4.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bca36be5707e81b9e6ce3208d92d95540d4ca244c006b61511753583c81c70dd"}, - {file = "watchdog-4.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ab998f567ebdf6b1da7dc1e5accfaa7c6992244629c0fdaef062f43249bd8dee"}, - {file = "watchdog-4.0.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:dddba7ca1c807045323b6af4ff80f5ddc4d654c8bce8317dde1bd96b128ed253"}, - {file = "watchdog-4.0.1-py3-none-manylinux2014_armv7l.whl", hash = "sha256:4513ec234c68b14d4161440e07f995f231be21a09329051e67a2118a7a612d2d"}, - {file = "watchdog-4.0.1-py3-none-manylinux2014_i686.whl", hash = "sha256:4107ac5ab936a63952dea2a46a734a23230aa2f6f9db1291bf171dac3ebd53c6"}, - {file = "watchdog-4.0.1-py3-none-manylinux2014_ppc64.whl", hash = "sha256:6e8c70d2cd745daec2a08734d9f63092b793ad97612470a0ee4cbb8f5f705c57"}, - {file = "watchdog-4.0.1-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:f27279d060e2ab24c0aa98363ff906d2386aa6c4dc2f1a374655d4e02a6c5e5e"}, - {file = "watchdog-4.0.1-py3-none-manylinux2014_s390x.whl", hash = "sha256:f8affdf3c0f0466e69f5b3917cdd042f89c8c63aebdb9f7c078996f607cdb0f5"}, - {file = "watchdog-4.0.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ac7041b385f04c047fcc2951dc001671dee1b7e0615cde772e84b01fbf68ee84"}, - {file = "watchdog-4.0.1-py3-none-win32.whl", hash = "sha256:206afc3d964f9a233e6ad34618ec60b9837d0582b500b63687e34011e15bb429"}, - {file = "watchdog-4.0.1-py3-none-win_amd64.whl", hash = "sha256:7577b3c43e5909623149f76b099ac49a1a01ca4e167d1785c76eb52fa585745a"}, - {file = "watchdog-4.0.1-py3-none-win_ia64.whl", hash = "sha256:d7b9f5f3299e8dd230880b6c55504a1f69cf1e4316275d1b215ebdd8187ec88d"}, - {file = "watchdog-4.0.1.tar.gz", hash = "sha256:eebaacf674fa25511e8867028d281e602ee6500045b57f43b08778082f7f8b44"}, + {file = "watchdog-5.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:85527b882f3facda0579bce9d743ff7f10c3e1e0db0a0d0e28170a7d0e5ce2ea"}, + {file = "watchdog-5.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:53adf73dcdc0ef04f7735066b4a57a4cd3e49ef135daae41d77395f0b5b692cb"}, + {file = "watchdog-5.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e25adddab85f674acac303cf1f5835951345a56c5f7f582987d266679979c75b"}, + {file = "watchdog-5.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f01f4a3565a387080dc49bdd1fefe4ecc77f894991b88ef927edbfa45eb10818"}, + {file = "watchdog-5.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:91b522adc25614cdeaf91f7897800b82c13b4b8ac68a42ca959f992f6990c490"}, + {file = "watchdog-5.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d52db5beb5e476e6853da2e2d24dbbbed6797b449c8bf7ea118a4ee0d2c9040e"}, + {file = "watchdog-5.0.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94d11b07c64f63f49876e0ab8042ae034674c8653bfcdaa8c4b32e71cfff87e8"}, + {file = "watchdog-5.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:349c9488e1d85d0a58e8cb14222d2c51cbc801ce11ac3936ab4c3af986536926"}, + {file = "watchdog-5.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:53a3f10b62c2d569e260f96e8d966463dec1a50fa4f1b22aec69e3f91025060e"}, + {file = "watchdog-5.0.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:950f531ec6e03696a2414b6308f5c6ff9dab7821a768c9d5788b1314e9a46ca7"}, + {file = "watchdog-5.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae6deb336cba5d71476caa029ceb6e88047fc1dc74b62b7c4012639c0b563906"}, + {file = "watchdog-5.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1021223c08ba8d2d38d71ec1704496471ffd7be42cfb26b87cd5059323a389a1"}, + {file = "watchdog-5.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:752fb40efc7cc8d88ebc332b8f4bcbe2b5cc7e881bccfeb8e25054c00c994ee3"}, + {file = "watchdog-5.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a2e8f3f955d68471fa37b0e3add18500790d129cc7efe89971b8a4cc6fdeb0b2"}, + {file = "watchdog-5.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b8ca4d854adcf480bdfd80f46fdd6fb49f91dd020ae11c89b3a79e19454ec627"}, + {file = "watchdog-5.0.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:90a67d7857adb1d985aca232cc9905dd5bc4803ed85cfcdcfcf707e52049eda7"}, + {file = "watchdog-5.0.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:720ef9d3a4f9ca575a780af283c8fd3a0674b307651c1976714745090da5a9e8"}, + {file = "watchdog-5.0.3-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:223160bb359281bb8e31c8f1068bf71a6b16a8ad3d9524ca6f523ac666bb6a1e"}, + {file = "watchdog-5.0.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:560135542c91eaa74247a2e8430cf83c4342b29e8ad4f520ae14f0c8a19cfb5b"}, + {file = "watchdog-5.0.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:dd021efa85970bd4824acacbb922066159d0f9e546389a4743d56919b6758b91"}, + {file = "watchdog-5.0.3-py3-none-manylinux2014_armv7l.whl", hash = "sha256:78864cc8f23dbee55be34cc1494632a7ba30263951b5b2e8fc8286b95845f82c"}, + {file = "watchdog-5.0.3-py3-none-manylinux2014_i686.whl", hash = "sha256:1e9679245e3ea6498494b3028b90c7b25dbb2abe65c7d07423ecfc2d6218ff7c"}, + {file = "watchdog-5.0.3-py3-none-manylinux2014_ppc64.whl", hash = "sha256:9413384f26b5d050b6978e6fcd0c1e7f0539be7a4f1a885061473c5deaa57221"}, + {file = "watchdog-5.0.3-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:294b7a598974b8e2c6123d19ef15de9abcd282b0fbbdbc4d23dfa812959a9e05"}, + {file = "watchdog-5.0.3-py3-none-manylinux2014_s390x.whl", hash = "sha256:26dd201857d702bdf9d78c273cafcab5871dd29343748524695cecffa44a8d97"}, + {file = "watchdog-5.0.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:0f9332243355643d567697c3e3fa07330a1d1abf981611654a1f2bf2175612b7"}, + {file = "watchdog-5.0.3-py3-none-win32.whl", hash = "sha256:c66f80ee5b602a9c7ab66e3c9f36026590a0902db3aea414d59a2f55188c1f49"}, + {file = "watchdog-5.0.3-py3-none-win_amd64.whl", hash = "sha256:f00b4cf737f568be9665563347a910f8bdc76f88c2970121c86243c8cfdf90e9"}, + {file = "watchdog-5.0.3-py3-none-win_ia64.whl", hash = "sha256:49f4d36cb315c25ea0d946e018c01bb028048023b9e103d3d3943f58e109dd45"}, + {file = "watchdog-5.0.3.tar.gz", hash = "sha256:108f42a7f0345042a854d4d0ad0834b741d421330d5f575b81cb27b883500176"}, ] [package.extras] @@ -3012,18 +3051,22 @@ files = [ [[package]] name = "zipp" -version = "3.19.2" +version = "3.20.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, + {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, + {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] [package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [metadata] lock-version = "2.0" From e67b7b224decf7890c617ace9e7ee37bd4755cfd Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 11:11:23 -0400 Subject: [PATCH 147/254] bumping baddns version (again) --- bbot/modules/baddns.py | 2 +- bbot/modules/baddns_direct.py | 2 +- bbot/modules/baddns_zone.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 3e3a0dcb5..2a5892f25 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -22,7 +22,7 @@ class baddns(BaseModule): "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", } module_threads = 8 - deps_pip = ["baddns~=1.1.862"] + deps_pip = ["baddns~=1.1.864"] def select_modules(self): selected_submodules = [] diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index cc4272d69..e60c2bbb6 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -19,7 +19,7 @@ class baddns_direct(BaseModule): "custom_nameservers": "Force BadDNS to use a list of custom nameservers", } module_threads = 8 - deps_pip = ["baddns~=1.1.862"] + deps_pip = ["baddns~=1.1.864"] scope_distance_modifier = 1 diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index 26b427451..ca3ab39b0 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -16,7 +16,7 @@ class baddns_zone(baddns_module): "only_high_confidence": "Do not emit low-confidence or generic detections", } module_threads = 8 - deps_pip = ["baddns~=1.1.862"] + deps_pip = ["baddns~=1.1.864"] def set_modules(self): self.enabled_submodules = ["NSEC", "zonetransfer"] From d66dd1aad97c8c820134d4b202c30c699a9f48b7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 1 Oct 2024 12:11:56 -0400 Subject: [PATCH 148/254] fixing tests --- bbot/core/helpers/misc.py | 12 ++++ bbot/modules/base.py | 2 +- bbot/test/test_step_1/test_helpers.py | 4 ++ bbot/test/test_step_1/test_web.py | 9 ++- .../module_tests/test_module_hunterio.py | 70 ++++++++++++++++++- 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 0c4e26ca9..f08444cd4 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2788,3 +2788,15 @@ def top_tcp_ports(n, as_string=False): if as_string: return ",".join([str(s) for s in top_ports]) return top_ports + + +class SafeDict(dict): + def __missing__(self, key): + return "{" + key + "}" + + +def safe_format(s, **kwargs): + """ + Format string while ignoring unused keys (prevents KeyError) + """ + return s.format_map(SafeDict(kwargs)) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 023cf0966..3a9793e20 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1188,7 +1188,7 @@ async def api_page_iter(self, url, page_size=100, json=True, next_key=None, **re self.debug(f"Failed to extract next page of results from {url}: {e}") self.debug(traceback.format_exc()) else: - new_url = url.format(page=page, page_size=page_size, offset=offset) + new_url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset) result = await self.api_request(new_url, **requests_kwargs) if result is None: self.verbose(f"api_page_iter() got no response for {url}") diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 85dd4d29a..8bb62917a 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -424,6 +424,10 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): assert type(helpers.make_date()) == str + # string formatter + s = "asdf {unused} {used}" + assert helpers.safe_format(s, used="fdsa") == "asdf {unused} fdsa" + # punycode assert helpers.smart_encode_punycode("ドメイン.テスト") == "xn--eckwd4c7c.xn--zckzah" assert helpers.smart_decode_punycode("xn--eckwd4c7c.xn--zckzah") == "ドメイン.テスト" diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index c4e02ccb8..4d5654c86 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -153,9 +153,12 @@ async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock): await scan._cleanup() - scan1 = bbot_scanner("8.8.8.8") + scan1 = bbot_scanner("8.8.8.8", modules=["ipneighbor"]) scan2 = bbot_scanner("127.0.0.1") + await scan1._prep() + module = scan1.modules["ipneighbor"] + web_config = CORE.config.get("web", {}) user_agent = web_config.get("user_agent", "") headers = {"User-Agent": user_agent} @@ -252,7 +255,7 @@ async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock): uri=f"{base_path}/3", query_string={"page_size": "100", "offset": "200"} ).respond_with_data("page3") results = [] - agen = scan1.api_page_iter(template_url) + agen = module.api_page_iter(template_url) try: async for result in agen: if result and result.text.startswith("page"): @@ -262,7 +265,7 @@ async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock): finally: await agen.aclose() assert not results - agen = scan1.api_page_iter(template_url, json=False) + agen = module.api_page_iter(template_url, json=False) try: async for result in agen: if result and result.text.startswith("page"): diff --git a/bbot/test/test_step_2/module_tests/test_module_hunterio.py b/bbot/test/test_step_2/module_tests/test_module_hunterio.py index 903309117..263f304a3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_hunterio.py +++ b/bbot/test/test_step_2/module_tests/test_module_hunterio.py @@ -2,7 +2,7 @@ class TestHunterio(ModuleTestBase): - config_overrides = {"modules": {"hunterio": {"api_key": "asdf"}}} + config_overrides = {"modules": {"hunterio": {"api_key": ["asdf", "1234", "4321", "fdsa"]}}} async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( @@ -29,7 +29,7 @@ async def setup_before_prep(self, module_test): }, ) module_test.httpx_mock.add_response( - url="https://api.hunter.io/v2/domain-search?domain=blacklanternsecurity.com&api_key=asdf&limit=100&offset=0", + url="https://api.hunter.io/v2/domain-search?domain=blacklanternsecurity.com&api_key=fdsa&limit=100&offset=0", json={ "data": { "domain": "blacklanternsecurity.com", @@ -91,6 +91,70 @@ async def setup_before_prep(self, module_test): }, }, ) + module_test.httpx_mock.add_response( + url="https://api.hunter.io/v2/domain-search?domain=blacklanternsecurity.com&api_key=4321&limit=100&offset=100", + json={ + "data": { + "domain": "blacklanternsecurity.com", + "disposable": False, + "webmail": False, + "accept_all": False, + "pattern": "{first}", + "organization": "Black Lantern Security", + "description": None, + "twitter": None, + "facebook": None, + "linkedin": "https://linkedin.com/company/black-lantern-security", + "instagram": None, + "youtube": None, + "technologies": ["jekyll", "nginx"], + "country": "US", + "state": "CA", + "city": "Night City", + "postal_code": "12345", + "street": "123 Any St", + "emails": [ + { + "value": "fdsa@blacklanternsecurity.com", + "type": "generic", + "confidence": 77, + "sources": [ + { + "domain": "blacklanternsecurity.com", + "uri": "http://blacklanternsecurity.com", + "extracted_on": "2021-06-09", + "last_seen_on": "2023-03-21", + "still_on_page": True, + } + ], + "first_name": None, + "last_name": None, + "position": None, + "seniority": None, + "department": "support", + "linkedin": None, + "twitter": None, + "phone_number": None, + "verification": {"date": None, "status": None}, + } + ], + "linked_domains": [], + }, + "meta": { + "results": 1, + "limit": 100, + "offset": 0, + "params": { + "domain": "blacklanternsecurity.com", + "company": None, + "type": None, + "seniority": None, + "department": None, + }, + }, + }, + ) def check(self, module_test, events): - assert any(e.data == "asdf@blacklanternsecurity.com" for e in events), "Failed to detect email" + assert any(e.data == "asdf@blacklanternsecurity.com" for e in events), "Failed to detect email #1" + assert any(e.data == "fdsa@blacklanternsecurity.com" for e in events), "Failed to detect email #2" From 7b40b9cc3bf8e052d240407cf810504e1f625c08 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 12:12:30 -0400 Subject: [PATCH 149/254] adding more rubust test cleanup --- bbot/test/conftest.py | 56 +++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 7cccc950d..866188d59 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -1,4 +1,5 @@ import os +import gc import ssl import shutil import pytest @@ -38,17 +39,17 @@ def bbot_httpserver(): server = HTTPServer(host="127.0.0.1", port=8888, threaded=True) server.start() - yield server - - server.clear() - if server.is_running(): - server.stop() + try: + yield server + finally: + if server.is_running(): + server.stop() # Stop the server if still running + server.check_assertions() # Verify if all requests were asserted + server.clear() # Clear server state to ensure no lingering references - # this is to check if the client has made any request where no - # `assert_request` was called on it from the test + # Explicitly collect garbage after teardown to avoid retained references + gc.collect() - server.check_assertions() - server.clear() @pytest.fixture @@ -58,21 +59,20 @@ def bbot_httpserver_ssl(): keyfile = str(current_dir / "testsslkey.pem") certfile = str(current_dir / "testsslcert.pem") context.load_cert_chain(certfile, keyfile) + server = HTTPServer(host="127.0.0.1", port=9999, ssl_context=context, threaded=True) server.start() - yield server - - server.clear() - if server.is_running(): - server.stop() - - # this is to check if the client has made any request where no - # `assert_request` was called on it from the test - - server.check_assertions() - server.clear() + try: + yield server + finally: + if server.is_running(): + server.stop() # Stop the server if still running + server.check_assertions() # Verify if all requests were asserted + server.clear() # Clear server state to ensure no lingering references + # Explicitly collect garbage after teardown to avoid retained references + gc.collect() @pytest.fixture def non_mocked_hosts() -> list: @@ -84,13 +84,17 @@ def bbot_httpserver_allinterfaces(): server = HTTPServer(host="0.0.0.0", port=5556, threaded=True) server.start() - yield server + try: + yield server + finally: + if server.is_running(): + server.stop() # Stop the server if still running + server.check_assertions() # Verify if all requests were asserted + server.clear() # Clear server state to ensure no lingering references + + # Explicitly collect garbage after teardown to avoid retained references + gc.collect() - server.clear() - if server.is_running(): - server.stop() - server.check_assertions() - server.clear() class Interactsh_mock: From 2280a868b01889914fe78498d8b57a824e9f54b8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 12:15:05 -0400 Subject: [PATCH 150/254] black --- bbot/test/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 866188d59..901193925 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -51,7 +51,6 @@ def bbot_httpserver(): gc.collect() - @pytest.fixture def bbot_httpserver_ssl(): context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) @@ -74,6 +73,7 @@ def bbot_httpserver_ssl(): # Explicitly collect garbage after teardown to avoid retained references gc.collect() + @pytest.fixture def non_mocked_hosts() -> list: return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers @@ -96,7 +96,6 @@ def bbot_httpserver_allinterfaces(): gc.collect() - class Interactsh_mock: def __init__(self, name): self.name = name From 28770afb037a34ec25912403a15845de35260bde Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 12:54:53 -0400 Subject: [PATCH 151/254] Revert "adding more rubust test cleanup" This reverts commit 7b40b9cc3bf8e052d240407cf810504e1f625c08. --- bbot/test/conftest.py | 56 ++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 901193925..2d01bed56 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -1,5 +1,4 @@ import os -import gc import ssl import shutil import pytest @@ -39,17 +38,17 @@ def bbot_httpserver(): server = HTTPServer(host="127.0.0.1", port=8888, threaded=True) server.start() - try: - yield server - finally: - if server.is_running(): - server.stop() # Stop the server if still running - server.check_assertions() # Verify if all requests were asserted - server.clear() # Clear server state to ensure no lingering references + yield server + + server.clear() + if server.is_running(): + server.stop() - # Explicitly collect garbage after teardown to avoid retained references - gc.collect() + # this is to check if the client has made any request where no + # `assert_request` was called on it from the test + server.check_assertions() + server.clear() @pytest.fixture def bbot_httpserver_ssl(): @@ -58,20 +57,21 @@ def bbot_httpserver_ssl(): keyfile = str(current_dir / "testsslkey.pem") certfile = str(current_dir / "testsslcert.pem") context.load_cert_chain(certfile, keyfile) - server = HTTPServer(host="127.0.0.1", port=9999, ssl_context=context, threaded=True) server.start() - try: - yield server - finally: - if server.is_running(): - server.stop() # Stop the server if still running - server.check_assertions() # Verify if all requests were asserted - server.clear() # Clear server state to ensure no lingering references + yield server + + server.clear() + if server.is_running(): + server.stop() + + # this is to check if the client has made any request where no + # `assert_request` was called on it from the test + + server.check_assertions() + server.clear() - # Explicitly collect garbage after teardown to avoid retained references - gc.collect() @pytest.fixture @@ -84,17 +84,13 @@ def bbot_httpserver_allinterfaces(): server = HTTPServer(host="0.0.0.0", port=5556, threaded=True) server.start() - try: - yield server - finally: - if server.is_running(): - server.stop() # Stop the server if still running - server.check_assertions() # Verify if all requests were asserted - server.clear() # Clear server state to ensure no lingering references - - # Explicitly collect garbage after teardown to avoid retained references - gc.collect() + yield server + server.clear() + if server.is_running(): + server.stop() + server.check_assertions() + server.clear() class Interactsh_mock: def __init__(self, name): From 57acbe3f5eaead28d5580d9fa4a5eca278caaf91 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 12:58:25 -0400 Subject: [PATCH 152/254] explicitly setting asyncio test mode to module --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 77dac51fe..b43fa5967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,9 @@ mike = "^2.1.2" env = [ "BBOT_TESTING = True", ] +asyncio_mode = auto +asyncio_default_fixture_loop_scope = module # All test functions in the same .py file will share the same loop + [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] From b475562ec0d30984521b969cf5c7c2d2d625ed25 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 13:03:19 -0400 Subject: [PATCH 153/254] explicitly setting asyncio test value.. --- bbot/test/conftest.py | 3 ++- pyproject.toml | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 2d01bed56..7cccc950d 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -50,6 +50,7 @@ def bbot_httpserver(): server.check_assertions() server.clear() + @pytest.fixture def bbot_httpserver_ssl(): context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) @@ -73,7 +74,6 @@ def bbot_httpserver_ssl(): server.clear() - @pytest.fixture def non_mocked_hosts() -> list: return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers @@ -92,6 +92,7 @@ def bbot_httpserver_allinterfaces(): server.check_assertions() server.clear() + class Interactsh_mock: def __init__(self, name): self.name = name diff --git a/pyproject.toml b/pyproject.toml index b43fa5967..1f97b8318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,9 +86,8 @@ mike = "^2.1.2" env = [ "BBOT_TESTING = True", ] -asyncio_mode = auto -asyncio_default_fixture_loop_scope = module # All test functions in the same .py file will share the same loop - +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "module" [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] From 479ec7b89c87f8ff73e2b985c45c480e6561e3c1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 1 Oct 2024 13:31:14 -0400 Subject: [PATCH 154/254] fix trickest tests --- bbot/modules/base.py | 1 + bbot/modules/templates/github.py | 1 + bbot/modules/templates/shodan.py | 1 + bbot/modules/trickest.py | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 3a9793e20..18d14c600 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -311,6 +311,7 @@ async def require_api_key(self): self.hugesuccess(f"API is ready") return True except Exception as e: + self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" else: return None, "No API key set" diff --git a/bbot/modules/templates/github.py b/bbot/modules/templates/github.py index a5eb17ee6..eb470e9e8 100644 --- a/bbot/modules/templates/github.py +++ b/bbot/modules/templates/github.py @@ -36,6 +36,7 @@ async def setup(self): self.hugesuccess(f"API is ready") return True except Exception as e: + self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" return True diff --git a/bbot/modules/templates/shodan.py b/bbot/modules/templates/shodan.py index 42b3e5afe..5347ee718 100644 --- a/bbot/modules/templates/shodan.py +++ b/bbot/modules/templates/shodan.py @@ -28,6 +28,7 @@ async def setup(self): self.hugesuccess(f"API is ready") return True except Exception as e: + self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" return True diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py index b786d8048..33b4d672c 100644 --- a/bbot/modules/trickest.py +++ b/bbot/modules/trickest.py @@ -24,9 +24,9 @@ class Trickest(subdomain_enum_apikey): def prepare_api_request(self, url, kwargs): kwargs["headers"]["Authorization"] = f"Token {self.api_key}" + return url, kwargs async def ping(self): - url = f"{self.base_url}/dataset" response = await self.api_request(url) status_code = getattr(response, "status_code", 0) From dc25e10327ebe988e94f058727c1ac8102a4ee2d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 13:44:48 -0400 Subject: [PATCH 155/254] switching to function mode --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1f97b8318..b0bbf483b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,7 @@ env = [ "BBOT_TESTING = True", ] asyncio_mode = "auto" -asyncio_default_fixture_loop_scope = "module" +asyncio_default_fixture_loop_scope = "function" [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] From b36e1e8706b1c9fc47c8539cb1925c85779f2fc9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 1 Oct 2024 13:49:27 -0400 Subject: [PATCH 156/254] test abort threshold --- bbot/modules/base.py | 2 +- .../module_tests/test_module_c99.py | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 18d14c600..967338635 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -336,7 +336,7 @@ def cycle_api_key(self): @property def failed_request_abort_threshold(self): - return max(self._failed_request_abort_threshold, len(self._api_keys)) + return max(self._failed_request_abort_threshold, len(self._api_keys) * 2) async def ping(self): """Asynchronously checks the health of the configured API. diff --git a/bbot/test/test_step_2/module_tests/test_module_c99.py b/bbot/test/test_step_2/module_tests/test_module_c99.py index 9efc2a8c5..38944b133 100644 --- a/bbot/test/test_step_2/module_tests/test_module_c99.py +++ b/bbot/test/test_step_2/module_tests/test_module_c99.py @@ -1,7 +1,10 @@ +import httpx + from .base import ModuleTestBase class TestC99(ModuleTestBase): + module_name = "c99" config_overrides = {"modules": {"c99": {"api_key": "asdf"}}} async def setup_before_prep(self, module_test): @@ -23,3 +26,39 @@ async def setup_before_prep(self, module_test): def check(self, module_test, events): assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + + +class TestC99AbortThreshold(TestC99): + config_overrides = {"modules": {"c99": {"api_key": ["6789", "fdsa", "1234", "4321"]}}} + + async def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://api.c99.nl/randomnumber?key=fdsa&between=1,100&json", + json={"success": True, "output": 65}, + ) + + self.url_count = {} + + async def custom_callback(request): + url = str(request.url) + try: + self.url_count[url] += 1 + except KeyError: + self.url_count[url] = 1 + raise httpx.TimeoutException("timeout") + + module_test.httpx_mock.add_callback(custom_callback) + + def check(self, module_test, events): + assert module_test.module.failed_request_abort_threshold == 8 + assert module_test.module.errored == True + assert [e.data for e in events if e.type == "DNS_NAME"] == ["blacklanternsecurity.com"] + assert self.url_count == { + "https://api.c99.nl/randomnumber?key=6789&between=1,100&json": 1, + "https://api.c99.nl/randomnumber?key=4321&between=1,100&json": 1, + "https://api.c99.nl/randomnumber?key=1234&between=1,100&json": 1, + "https://api.c99.nl/subdomainfinder?key=fdsa&domain=blacklanternsecurity.com&json": 2, + "https://api.c99.nl/subdomainfinder?key=6789&domain=blacklanternsecurity.com&json": 2, + "https://api.c99.nl/subdomainfinder?key=4321&domain=blacklanternsecurity.com&json": 2, + "https://api.c99.nl/subdomainfinder?key=1234&domain=blacklanternsecurity.com&json": 2, + } From aa4a837c3e954e334d622edc2a5ae6cabca521af Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 1 Oct 2024 13:50:45 -0400 Subject: [PATCH 157/254] flaked --- bbot/modules/templates/github.py | 2 ++ bbot/modules/templates/shodan.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/bbot/modules/templates/github.py b/bbot/modules/templates/github.py index eb470e9e8..35c68a211 100644 --- a/bbot/modules/templates/github.py +++ b/bbot/modules/templates/github.py @@ -1,3 +1,5 @@ +import traceback + from bbot.modules.base import BaseModule diff --git a/bbot/modules/templates/shodan.py b/bbot/modules/templates/shodan.py index 5347ee718..3cc84022d 100644 --- a/bbot/modules/templates/shodan.py +++ b/bbot/modules/templates/shodan.py @@ -1,3 +1,5 @@ +import traceback + from bbot.modules.templates.subdomain_enum import subdomain_enum From e27ef6bcf132f185193a36a0459441f48526a558 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 14:21:49 -0400 Subject: [PATCH 158/254] temporary test --- bbot/scanner/preset/environ.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 4b7121e3c..4920c1620 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -118,19 +118,19 @@ def prepare(self): # ssl verification import urllib3 - urllib3.disable_warnings() - ssl_verify = self.preset.config.get("ssl_verify", False) - if not ssl_verify: - import requests - import functools - - requests.adapters.BaseAdapter.send = functools.partialmethod( - requests.adapters.BaseAdapter.send, verify=False - ) - requests.adapters.HTTPAdapter.send = functools.partialmethod( - requests.adapters.HTTPAdapter.send, verify=False - ) - requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) - requests.request = functools.partial(requests.request, verify=False) + # urllib3.disable_warnings() + # ssl_verify = self.preset.config.get("ssl_verify", False) + # if not ssl_verify: + # import requests + # import functools + + # requests.adapters.BaseAdapter.send = functools.partialmethod( + # requests.adapters.BaseAdapter.send, verify=False + # ) + # requests.adapters.HTTPAdapter.send = functools.partialmethod( + # requests.adapters.HTTPAdapter.send, verify=False + # ) + # requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) + # requests.request = functools.partial(requests.request, verify=False) return environ From adcd558653e9baf43d15770d6e5ce7f80c19ec99 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 14:26:03 -0400 Subject: [PATCH 159/254] temporary test 2 --- bbot/scanner/preset/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 4920c1620..1db06c958 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -116,7 +116,7 @@ def prepare(self): environ.pop("HTTPS_PROXY", None) # ssl verification - import urllib3 + #import urllib3 # urllib3.disable_warnings() # ssl_verify = self.preset.config.get("ssl_verify", False) From 6e77918300dc71b4329b51d6eafdc22521e0eaf8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 14:27:11 -0400 Subject: [PATCH 160/254] temporary test 3 --- bbot/scanner/preset/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 1db06c958..41012da06 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -116,7 +116,7 @@ def prepare(self): environ.pop("HTTPS_PROXY", None) # ssl verification - #import urllib3 + # import urllib3 # urllib3.disable_warnings() # ssl_verify = self.preset.config.get("ssl_verify", False) From ce1fcd1e12119acf1ccfe92a1a3dba3a31591dbc Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 1 Oct 2024 14:40:22 -0400 Subject: [PATCH 161/254] only patch requests once --- bbot/scanner/preset/environ.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 4b7121e3c..66253de32 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -6,6 +6,9 @@ from bbot.core.helpers.misc import cpu_architecture, os_platform, os_platform_friendly +REQUESTS_PATCHED = False + + def increase_limit(new_limit): try: import resource @@ -120,7 +123,10 @@ def prepare(self): urllib3.disable_warnings() ssl_verify = self.preset.config.get("ssl_verify", False) - if not ssl_verify: + + global REQUESTS_PATCHED + if not ssl_verify and not REQUESTS_PATCHED: + REQUESTS_PATCHED = True import requests import functools From c1532c281440c4a662e5d5a860f8b61acd17de4c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 15:07:42 -0400 Subject: [PATCH 162/254] Revert "temporary test 3" This reverts commit 6e77918300dc71b4329b51d6eafdc22521e0eaf8. --- bbot/scanner/preset/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 41012da06..1db06c958 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -116,7 +116,7 @@ def prepare(self): environ.pop("HTTPS_PROXY", None) # ssl verification - # import urllib3 + #import urllib3 # urllib3.disable_warnings() # ssl_verify = self.preset.config.get("ssl_verify", False) From 4023f58126c49dcad2f561ecd0ee79564b61004c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 15:07:50 -0400 Subject: [PATCH 163/254] Revert "temporary test 2" This reverts commit adcd558653e9baf43d15770d6e5ce7f80c19ec99. --- bbot/scanner/preset/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 1db06c958..4920c1620 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -116,7 +116,7 @@ def prepare(self): environ.pop("HTTPS_PROXY", None) # ssl verification - #import urllib3 + import urllib3 # urllib3.disable_warnings() # ssl_verify = self.preset.config.get("ssl_verify", False) From 3520b4e9b70d08313b2f769c940d8c9d50ec7647 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 15:07:59 -0400 Subject: [PATCH 164/254] Revert "temporary test" This reverts commit e27ef6bcf132f185193a36a0459441f48526a558. --- bbot/scanner/preset/environ.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index 4920c1620..4b7121e3c 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -118,19 +118,19 @@ def prepare(self): # ssl verification import urllib3 - # urllib3.disable_warnings() - # ssl_verify = self.preset.config.get("ssl_verify", False) - # if not ssl_verify: - # import requests - # import functools - - # requests.adapters.BaseAdapter.send = functools.partialmethod( - # requests.adapters.BaseAdapter.send, verify=False - # ) - # requests.adapters.HTTPAdapter.send = functools.partialmethod( - # requests.adapters.HTTPAdapter.send, verify=False - # ) - # requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) - # requests.request = functools.partial(requests.request, verify=False) + urllib3.disable_warnings() + ssl_verify = self.preset.config.get("ssl_verify", False) + if not ssl_verify: + import requests + import functools + + requests.adapters.BaseAdapter.send = functools.partialmethod( + requests.adapters.BaseAdapter.send, verify=False + ) + requests.adapters.HTTPAdapter.send = functools.partialmethod( + requests.adapters.HTTPAdapter.send, verify=False + ) + requests.Session.request = functools.partialmethod(requests.Session.request, verify=False) + requests.request = functools.partial(requests.request, verify=False) return environ From 713dd2141541f1dc6c62f2a44c551c9d6c04525c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 16:55:34 -0400 Subject: [PATCH 165/254] banning .'s from vhost wordlists --- bbot/modules/deadly/vhost.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py index 98991c53d..09a0a36e4 100644 --- a/bbot/modules/deadly/vhost.py +++ b/bbot/modules/deadly/vhost.py @@ -23,6 +23,7 @@ class vhost(ffuf): } deps_common = ["ffuf"] + banned_characters = set([" ","."]) in_scope_only = True @@ -103,7 +104,7 @@ async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=F def mutations_check(self, vhost): mutations_list = [] for mutation in self.helpers.word_cloud.mutations(vhost): - for i in ["", ".", "-"]: + for i in ["", "-"]: mutations_list.append(i.join(mutation)) mutations_list_file = self.helpers.tempfile(mutations_list, pipe=False) return mutations_list_file From 94afc7b8ce4809805d32747cde4513ec21f4b7ab Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 1 Oct 2024 16:55:53 -0400 Subject: [PATCH 166/254] black --- bbot/modules/deadly/vhost.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py index 09a0a36e4..66c1c516c 100644 --- a/bbot/modules/deadly/vhost.py +++ b/bbot/modules/deadly/vhost.py @@ -23,7 +23,7 @@ class vhost(ffuf): } deps_common = ["ffuf"] - banned_characters = set([" ","."]) + banned_characters = set([" ", "."]) in_scope_only = True From f2479ba8e692e10430d9c5a8d1e5e22d53c07d9e Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 2 Oct 2024 10:34:00 -0400 Subject: [PATCH 167/254] more tests --- bbot/modules/base.py | 34 +++--- .../module_tests/test_module_c99.py | 103 ++++++++++++++++-- .../module_tests/test_module_rapiddns.py | 94 +++++++++++++++- 3 files changed, 209 insertions(+), 22 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 967338635..df4bed023 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -63,7 +63,7 @@ class BaseModule: batch_wait (int): Seconds to wait before force-submitting a batch. Default is 10. - failed_request_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `api_request()` is used. Default is 5. + api_failure_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `api_request()` is used. Default is 5. _preserve_graph (bool): When set to True, accept events that may be duplicates but are necessary for construction of complete graph. Typically only enabled for output modules that need to maintain full chains of events, e.g. `neo4j` and `json`. Default is False. @@ -103,8 +103,11 @@ class BaseModule: _module_threads = 1 _batch_size = 1 batch_wait = 10 - # disable the module after this many failed requests in a row - _failed_request_abort_threshold = 5 + + # API retries, etc. + _api_retries = 2 + # disable the module after this many failed attempts in a row + _api_failure_abort_threshold = 3 # sleep for this many seconds after being rate limited _429_sleep_interval = 30 @@ -154,7 +157,7 @@ def __init__(self, scan): self._api_keys = [] # track number of failures (for .api_request()) - self._request_failures = 0 + self._api_request_failures = 0 self._tasks = [] self._event_received = asyncio.Condition() @@ -335,8 +338,12 @@ def cycle_api_key(self): self.debug(f"No extra API keys to cycle") @property - def failed_request_abort_threshold(self): - return max(self._failed_request_abort_threshold, len(self._api_keys) * 2) + def api_retries(self): + return max(self._api_retries + 1, len(self._api_keys)) + + @property + def api_failure_abort_threshold(self): + return (self.api_retries * self._api_failure_abort_threshold) + 1 async def ping(self): """Asynchronously checks the health of the configured API. @@ -1114,7 +1121,7 @@ async def api_request(self, *args, **kwargs): url = args[0] if args else kwargs.pop("url", "") # loop until we have a successful request - while 1: + for _ in range(self.api_retries): if not "headers" in kwargs: kwargs["headers"] = {} new_url, kwargs = self.prepare_api_request(url, kwargs) @@ -1124,12 +1131,14 @@ async def api_request(self, *args, **kwargs): success = False if r is None else r.is_success if success: - self._request_failures = 0 + self._api_request_failures = 0 else: status_code = getattr(r, "status_code", 0) - self._request_failures += 1 - if self._request_failures >= self.failed_request_abort_threshold: - self.set_error_state(f"Setting error state due to {self._request_failures:,} failed HTTP requests") + self._api_request_failures += 1 + if self._api_request_failures >= self.api_failure_abort_threshold: + self.set_error_state( + f"Setting error state due to {self._api_request_failures:,} failed HTTP requests" + ) else: # sleep for a bit if we're being rate limited if status_code == 429: @@ -1137,11 +1146,10 @@ async def api_request(self, *args, **kwargs): f"Sleeping for {self._429_sleep_interval:,} seconds due to rate limit (HTTP status: 429)" ) await asyncio.sleep(self._429_sleep_interval) - continue elif self._api_keys: # if request failed, cycle API keys and try again self.cycle_api_key() - continue + continue break return r diff --git a/bbot/test/test_step_2/module_tests/test_module_c99.py b/bbot/test/test_step_2/module_tests/test_module_c99.py index 38944b133..284b76e1e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_c99.py +++ b/bbot/test/test_step_2/module_tests/test_module_c99.py @@ -28,7 +28,7 @@ def check(self, module_test, events): assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" -class TestC99AbortThreshold(TestC99): +class TestC99AbortThreshold1(TestC99): config_overrides = {"modules": {"c99": {"api_key": ["6789", "fdsa", "1234", "4321"]}}} async def setup_before_prep(self, module_test): @@ -50,15 +50,102 @@ async def custom_callback(request): module_test.httpx_mock.add_callback(custom_callback) def check(self, module_test, events): - assert module_test.module.failed_request_abort_threshold == 8 - assert module_test.module.errored == True - assert [e.data for e in events if e.type == "DNS_NAME"] == ["blacklanternsecurity.com"] + assert module_test.module.api_failure_abort_threshold == 13 + assert module_test.module.errored == False + # assert module_test.module._api_request_failures == 4 + assert module_test.module.api_retries == 4 + assert set([e.data for e in events if e.type == "DNS_NAME"]) == {"blacklanternsecurity.com"} assert self.url_count == { "https://api.c99.nl/randomnumber?key=6789&between=1,100&json": 1, "https://api.c99.nl/randomnumber?key=4321&between=1,100&json": 1, "https://api.c99.nl/randomnumber?key=1234&between=1,100&json": 1, - "https://api.c99.nl/subdomainfinder?key=fdsa&domain=blacklanternsecurity.com&json": 2, - "https://api.c99.nl/subdomainfinder?key=6789&domain=blacklanternsecurity.com&json": 2, - "https://api.c99.nl/subdomainfinder?key=4321&domain=blacklanternsecurity.com&json": 2, - "https://api.c99.nl/subdomainfinder?key=1234&domain=blacklanternsecurity.com&json": 2, + "https://api.c99.nl/subdomainfinder?key=fdsa&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=6789&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=4321&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=1234&domain=blacklanternsecurity.com&json": 1, + } + + +class TestC99AbortThreshold2(TestC99AbortThreshold1): + targets = ["blacklanternsecurity.com", "evilcorp.com"] + + async def setup_before_prep(self, module_test): + await super().setup_before_prep(module_test) + await module_test.mock_dns( + { + "blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + "evilcorp.com": {"A": ["127.0.0.11"]}, + "evilcorp.net": {"A": ["127.0.0.22"]}, + "evilcorp.co.uk": {"A": ["127.0.0.33"]}, + } + ) + + def check(self, module_test, events): + assert module_test.module.api_failure_abort_threshold == 13 + assert module_test.module.errored == False + assert module_test.module._api_request_failures == 8 + assert module_test.module.api_retries == 4 + assert set([e.data for e in events if e.type == "DNS_NAME"]) == {"blacklanternsecurity.com", "evilcorp.com"} + assert self.url_count == { + "https://api.c99.nl/randomnumber?key=6789&between=1,100&json": 1, + "https://api.c99.nl/randomnumber?key=4321&between=1,100&json": 1, + "https://api.c99.nl/randomnumber?key=1234&between=1,100&json": 1, + "https://api.c99.nl/subdomainfinder?key=fdsa&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=6789&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=4321&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=1234&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=fdsa&domain=evilcorp.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=6789&domain=evilcorp.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=4321&domain=evilcorp.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=1234&domain=evilcorp.com&json": 1, + } + + +class TestC99AbortThreshold3(TestC99AbortThreshold2): + targets = ["blacklanternsecurity.com", "evilcorp.com", "evilcorp.net"] + + def check(self, module_test, events): + assert module_test.module.api_failure_abort_threshold == 13 + assert module_test.module.errored == False + assert module_test.module._api_request_failures == 12 + assert module_test.module.api_retries == 4 + assert set([e.data for e in events if e.type == "DNS_NAME"]) == { + "blacklanternsecurity.com", + "evilcorp.com", + "evilcorp.net", + } + assert self.url_count == { + "https://api.c99.nl/randomnumber?key=6789&between=1,100&json": 1, + "https://api.c99.nl/randomnumber?key=4321&between=1,100&json": 1, + "https://api.c99.nl/randomnumber?key=1234&between=1,100&json": 1, + "https://api.c99.nl/subdomainfinder?key=fdsa&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=6789&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=4321&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=1234&domain=blacklanternsecurity.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=fdsa&domain=evilcorp.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=6789&domain=evilcorp.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=4321&domain=evilcorp.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=1234&domain=evilcorp.com&json": 1, + "https://api.c99.nl/subdomainfinder?key=fdsa&domain=evilcorp.net&json": 1, + "https://api.c99.nl/subdomainfinder?key=6789&domain=evilcorp.net&json": 1, + "https://api.c99.nl/subdomainfinder?key=4321&domain=evilcorp.net&json": 1, + "https://api.c99.nl/subdomainfinder?key=1234&domain=evilcorp.net&json": 1, + } + + +class TestC99AbortThreshold4(TestC99AbortThreshold3): + targets = ["blacklanternsecurity.com", "evilcorp.com", "evilcorp.net", "evilcorp.co.uk"] + + def check(self, module_test, events): + assert module_test.module.api_failure_abort_threshold == 13 + assert module_test.module.errored == True + assert module_test.module._api_request_failures == 13 + assert module_test.module.api_retries == 4 + assert set([e.data for e in events if e.type == "DNS_NAME"]) == { + "blacklanternsecurity.com", + "evilcorp.com", + "evilcorp.net", + "evilcorp.co.uk", } + assert len(self.url_count) == 16 + assert all([v == 1 for v in self.url_count.values()]) diff --git a/bbot/test/test_step_2/module_tests/test_module_rapiddns.py b/bbot/test/test_step_2/module_tests/test_module_rapiddns.py index be49cb881..042a261c7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_rapiddns.py +++ b/bbot/test/test_step_2/module_tests/test_module_rapiddns.py @@ -1,3 +1,5 @@ +import httpx + from .base import ModuleTestBase @@ -6,7 +8,7 @@ class TestRapidDNS(ModuleTestBase): asdf.blacklanternsecurity.com asdf.blacklanternsecurity.com.""" - async def setup_after_prep(self, module_test): + async def setup_before_prep(self, module_test): module_test.module.abort_if = lambda e: False module_test.httpx_mock.add_response( url=f"https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result", text=self.web_body @@ -14,3 +16,93 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + + +class TestRapidDNSAbortThreshold1(TestRapidDNS): + module_name = "rapiddns" + + async def setup_before_prep(self, module_test): + self.url_count = {} + + async def custom_callback(request): + url = str(request.url) + try: + self.url_count[url] += 1 + except KeyError: + self.url_count[url] = 1 + raise httpx.TimeoutException("timeout") + + module_test.httpx_mock.add_callback(custom_callback) + + await module_test.mock_dns( + { + "blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + "evilcorp.com": {"A": ["127.0.0.11"]}, + "evilcorp.net": {"A": ["127.0.0.22"]}, + "evilcorp.co.uk": {"A": ["127.0.0.33"]}, + } + ) + + def check(self, module_test, events): + assert module_test.module.api_failure_abort_threshold == 10 + assert module_test.module.errored == False + assert module_test.module._api_request_failures == 3 + assert module_test.module.api_retries == 3 + assert set([e.data for e in events if e.type == "DNS_NAME"]) == {"blacklanternsecurity.com"} + assert self.url_count == { + "https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result": 3, + } + + +class TestRapidDNSAbortThreshold2(TestRapidDNSAbortThreshold1): + targets = ["blacklanternsecurity.com", "evilcorp.com"] + + def check(self, module_test, events): + assert module_test.module.api_failure_abort_threshold == 10 + assert module_test.module.errored == False + assert module_test.module._api_request_failures == 6 + assert module_test.module.api_retries == 3 + assert set([e.data for e in events if e.type == "DNS_NAME"]) == {"blacklanternsecurity.com", "evilcorp.com"} + assert self.url_count == { + "https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result": 3, + "https://rapiddns.io/subdomain/evilcorp.com?full=1#result": 3, + } + + +class TestRapidDNSAbortThreshold3(TestRapidDNSAbortThreshold1): + targets = ["blacklanternsecurity.com", "evilcorp.com", "evilcorp.net"] + + def check(self, module_test, events): + assert module_test.module.api_failure_abort_threshold == 10 + assert module_test.module.errored == False + assert module_test.module._api_request_failures == 9 + assert module_test.module.api_retries == 3 + assert set([e.data for e in events if e.type == "DNS_NAME"]) == { + "blacklanternsecurity.com", + "evilcorp.com", + "evilcorp.net", + } + assert self.url_count == { + "https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result": 3, + "https://rapiddns.io/subdomain/evilcorp.com?full=1#result": 3, + "https://rapiddns.io/subdomain/evilcorp.net?full=1#result": 3, + } + + +class TestRapidDNSAbortThreshold4(TestRapidDNSAbortThreshold1): + targets = ["blacklanternsecurity.com", "evilcorp.com", "evilcorp.net", "evilcorp.co.uk"] + + def check(self, module_test, events): + assert module_test.module.api_failure_abort_threshold == 10 + assert module_test.module.errored == True + assert module_test.module._api_request_failures == 10 + assert module_test.module.api_retries == 3 + assert set([e.data for e in events if e.type == "DNS_NAME"]) == { + "blacklanternsecurity.com", + "evilcorp.com", + "evilcorp.net", + "evilcorp.co.uk", + } + assert len(self.url_count) == 4 + assert list(self.url_count.values()).count(3) == 3 + assert list(self.url_count.values()).count(1) == 1 From 8a6ad3d2ab5f1c4df01fca525d4f2ed3483158d9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 2 Oct 2024 11:36:57 -0400 Subject: [PATCH 168/254] fix tests --- bbot/test/test_step_2/module_tests/test_module_rapiddns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_rapiddns.py b/bbot/test/test_step_2/module_tests/test_module_rapiddns.py index 042a261c7..2b3d3aaf0 100644 --- a/bbot/test/test_step_2/module_tests/test_module_rapiddns.py +++ b/bbot/test/test_step_2/module_tests/test_module_rapiddns.py @@ -8,7 +8,7 @@ class TestRapidDNS(ModuleTestBase): asdf.blacklanternsecurity.com asdf.blacklanternsecurity.com.""" - async def setup_before_prep(self, module_test): + async def setup_after_prep(self, module_test): module_test.module.abort_if = lambda e: False module_test.httpx_mock.add_response( url=f"https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result", text=self.web_body @@ -21,7 +21,7 @@ def check(self, module_test, events): class TestRapidDNSAbortThreshold1(TestRapidDNS): module_name = "rapiddns" - async def setup_before_prep(self, module_test): + async def setup_after_prep(self, module_test): self.url_count = {} async def custom_callback(request): From a2125c43cc0dd1e5cb77fa1dd817e3ed369ef5d4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 2 Oct 2024 12:32:37 -0400 Subject: [PATCH 169/254] fix zoomeye --- bbot/modules/zoomeye.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py index 2bf7789d5..ffba419dd 100644 --- a/bbot/modules/zoomeye.py +++ b/bbot/modules/zoomeye.py @@ -27,6 +27,7 @@ async def setup(self): def prepare_api_request(self, url, kwargs): kwargs["headers"]["API-KEY"] = self.api_key + return url, kwargs async def ping(self): url = f"{self.base_url}/resources-info" From 0d0a7f0e88c3e2f5e3c23a89b8fa94cb102b4a56 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 2 Oct 2024 13:55:49 -0400 Subject: [PATCH 170/254] fixing bug with confirmation detection --- bbot/modules/iis_shortnames.py | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index d3aabe430..d68ee37d6 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -41,34 +41,24 @@ async def detect(self, target): urls_and_kwargs = [] for method in ["GET", "POST", "OPTIONS", "DEBUG", "HEAD", "TRACE"]: kwargs = dict(method=method, allow_redirects=False, timeout=10) - urls_and_kwargs.append((control_url, kwargs, method)) - urls_and_kwargs.append((test_url, kwargs, method)) - - results = {} - async for url, kwargs, method, response in self.helpers.request_custom_batch(urls_and_kwargs): - try: - results[method][url] = response - except KeyError: - results[method] = {url: response} - for method, result in results.items(): confirmations = 0 - iterations = 4 # one failed detection is tolerated, as long as its not the first run + iterations = 5 # one failed detection is tolerated, as long as its not the first run while iterations > 0: - control = results[method].get(control_url, None) - test = results[method].get(test_url, None) - if control and test: - if control.status_code != test.status_code: + control_result = await self.helpers.request(control_url, **kwargs) + test_result = await self.helpers.request(test_url, **kwargs) + if control_result and test_result: + if control_result.status_code != test_result.status_code: confirmations += 1 - self.debug(f"New detection, number of confirmations: [{str(confirmations)}]") - if confirmations > 2: - technique = f"{str(control.status_code)}/{str(test.status_code)} HTTP Code" - detections.append((method, test.status_code, technique)) + self.debug(f"New detection on {target}, number of confirmations: [{str(confirmations)}]") + if confirmations > 3: + technique = f"{str(control_result.status_code)}/{str(test_result.status_code)} HTTP Code" + detections.append((method, test_result.status_code, technique)) break - elif ("Error Code0x80070002" in control.text) and ( - "Error Code0x00000000" in test.text + elif ("Error Code0x80070002" in control_result.text) and ( + "Error Code0x00000000" in test_result.text ): confirmations += 1 - if confirmations > 2: + if confirmations > 3: detections.append((method, 0, technique)) technique = "HTTP Body Error Message" break From cef76c2a3c368c15a1836591f0d8b96fc7d1e83a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 2 Oct 2024 13:59:26 -0400 Subject: [PATCH 171/254] black --- bbot/modules/iis_shortnames.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index d68ee37d6..63ae85d15 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -45,7 +45,7 @@ async def detect(self, target): iterations = 5 # one failed detection is tolerated, as long as its not the first run while iterations > 0: control_result = await self.helpers.request(control_url, **kwargs) - test_result = await self.helpers.request(test_url, **kwargs) + test_result = await self.helpers.request(test_url, **kwargs) if control_result and test_result: if control_result.status_code != test_result.status_code: confirmations += 1 From a469c55ac1b54973528f17d0546b215065e5313d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 2 Oct 2024 14:01:25 -0400 Subject: [PATCH 172/254] flake8 --- bbot/modules/iis_shortnames.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index 63ae85d15..d5bb03a80 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -37,8 +37,7 @@ async def detect(self, target): random_string = self.helpers.rand_string(8) control_url = f"{target}{random_string}*~1*/a.aspx" test_url = f"{target}*~1*/a.aspx" - - urls_and_kwargs = [] + for method in ["GET", "POST", "OPTIONS", "DEBUG", "HEAD", "TRACE"]: kwargs = dict(method=method, allow_redirects=False, timeout=10) confirmations = 0 From 4a5c0a8287b54222fd77a2880ef91a25edf234be Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 2 Oct 2024 14:02:57 -0400 Subject: [PATCH 173/254] black again --- bbot/modules/iis_shortnames.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index d5bb03a80..d3b40bc18 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -37,7 +37,7 @@ async def detect(self, target): random_string = self.helpers.rand_string(8) control_url = f"{target}{random_string}*~1*/a.aspx" test_url = f"{target}*~1*/a.aspx" - + for method in ["GET", "POST", "OPTIONS", "DEBUG", "HEAD", "TRACE"]: kwargs = dict(method=method, allow_redirects=False, timeout=10) confirmations = 0 From b713d85710a4995975c5d92c97bfad7523516d59 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 2 Oct 2024 14:55:42 -0400 Subject: [PATCH 174/254] improved yara full url regex --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index d2c65414e..b85881d8b 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -669,7 +669,7 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte class URLExtractor(ExcavateRule): yara_rules = { - "url_full": r'rule url_full { meta: tags = "spider-danger" description = "contains full URL" strings: $url_full = /https?:\/\/([\w\.-]+)([:\/\w\.-]*)/ condition: $url_full }', + "url_full": r'rule url_full { meta: tags = "spider-danger" description = "contains full URL" strings: $url_full = /https?:\/\/([\w\.-]+)(:\d{1,5})?([\/\w\.-]*)/ condition: $url_full }', "url_attr": r'rule url_attr { meta: tags = "spider-danger" description = "contains tag with src or href attribute" strings: $url_attr = /<[^>]+(href|src)=["\'][^"\']*["\'][^>]*>/ condition: $url_attr }', } full_url_regex = re.compile(r"(https?)://((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)") From 52a4cab97fdb212eb1c308ae41a19edb9d471259 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 2 Oct 2024 15:40:38 -0400 Subject: [PATCH 175/254] filedownload, http output module tweaks --- bbot/modules/filedownload.py | 14 ++++++++------ bbot/modules/output/http.py | 34 +++++++++++++++++++++------------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py index 7649be4d3..872a447a1 100644 --- a/bbot/modules/filedownload.py +++ b/bbot/modules/filedownload.py @@ -25,12 +25,12 @@ class filedownload(BaseModule): "bak", # Backup File "bash", # Bash Script or Configuration "bashrc", # Bash Script or Configuration - "conf", # Configuration File "cfg", # Configuration File + "conf", # Configuration File "crt", # Certificate File "csv", # Comma Separated Values File "db", # SQLite Database File - "sqlite", # SQLite Database File + "dll", # Windows Dynamic Link Library "doc", # Microsoft Word Document (Old Format) "docx", # Microsoft Word Document "exe", # Windows PE executable @@ -39,7 +39,6 @@ class filedownload(BaseModule): "ini", # Initialization File "jar", # Java Archive "key", # Private Key File - "pub", # Public Key File "log", # Log File "markdown", # Markdown File "md", # Markdown File @@ -55,23 +54,26 @@ class filedownload(BaseModule): "ppt", # Microsoft PowerPoint Presentation (Old Format) "pptx", # Microsoft PowerPoint Presentation "ps1", # PowerShell Script + "pub", # Public Key File "raw", # Raw Image File Format "rdp", # Remote Desktop Protocol File "sh", # Shell Script "sql", # SQL Database Dump + "sqlite", # SQLite Database File "swp", # Swap File (temporary file, often Vim) "sxw", # OpenOffice.org Writer document - "tar", # Tar Archive "tar.gz", # Gzip-Compressed Tar Archive - "zip", # Zip Archive + "tar", # Tar Archive "txt", # Plain Text Document "vbs", # Visual Basic Script + "war", # Java Web Archive "wpd", # WordPerfect Document "xls", # Microsoft Excel Spreadsheet (Old Format) "xlsx", # Microsoft Excel Spreadsheet "xml", # eXtensible Markup Language File - "yml", # YAML Ain't Markup Language "yaml", # YAML Ain't Markup Language + "yml", # YAML Ain't Markup Language + "zip", # Zip Archive ], "max_filesize": "10MB", "base_64_encoded_file": "false", diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index e4bc79562..1a5c092d2 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -52,16 +52,24 @@ async def setup(self): async def handle_event(self, event): while 1: - try: - await self.helpers.request( - url=self.url, - method=self.method, - auth=self.auth, - headers=self.headers, - json=event.json(siem_friendly=self.siem_friendly), - raise_error=True, - ) - break - except WebError as e: - self.warning(f"Error sending {event}: {e}, retrying...") - await self.helpers.sleep(1) + response = await self.helpers.request( + url=self.url, + method=self.method, + auth=self.auth, + headers=self.headers, + json=event.json(siem_friendly=self.siem_friendly), + ) + is_success = False if response is None else response.is_success + if not is_success: + self.warning(f"Error sending {event} (HTTP status code: {status_code}), retrying...") + body = getattr(response, "text", "") + self.debug(body) + + status_code = getattr(response, "status_code", 0) + if status_code == 429: + sleep_interval = 10 + else: + sleep_interval = 1 + await self.helpers.sleep(sleep_interval) + continue + break From cf19f037cb1a1f6760f4bf631bfe62b55b47cfec Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 2 Oct 2024 15:45:05 -0400 Subject: [PATCH 176/254] flaked --- bbot/modules/output/http.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 1a5c092d2..9d9241da0 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -1,4 +1,3 @@ -from bbot.errors import WebError from bbot.modules.output.base import BaseOutputModule @@ -61,11 +60,10 @@ async def handle_event(self, event): ) is_success = False if response is None else response.is_success if not is_success: + status_code = getattr(response, "status_code", 0) self.warning(f"Error sending {event} (HTTP status code: {status_code}), retrying...") body = getattr(response, "text", "") self.debug(body) - - status_code = getattr(response, "status_code", 0) if status_code == 429: sleep_interval = 10 else: From 66c4cfa78a8dcb8e6b70c2e74febb7867a4d4aef Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 3 Oct 2024 10:55:54 -0400 Subject: [PATCH 177/254] fix autopublish --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 02cb65923..799c0101e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -127,7 +127,7 @@ jobs: git switch gh-pages git push publish_code: - needs: update_docs + needs: test runs-on: ubuntu-latest if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable') continue-on-error: true From 21fd6fea6cb7d3c7e35e4ceed1db2c46b43f866a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 12:10:20 -0400 Subject: [PATCH 178/254] merging author names --- bbot/modules/deadly/ffuf.py | 2 +- bbot/modules/iis_shortnames.py | 2 +- bbot/modules/paramminer_headers.py | 2 +- bbot/modules/wayback.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py index 4643c9826..6144d0b13 100644 --- a/bbot/modules/deadly/ffuf.py +++ b/bbot/modules/deadly/ffuf.py @@ -10,7 +10,7 @@ class ffuf(BaseModule): watched_events = ["URL"] produced_events = ["URL_UNVERIFIED"] flags = ["aggressive", "active"] - meta = {"description": "A fast web fuzzer written in Go", "created_date": "2022-04-10", "author": "@pmueller"} + meta = {"description": "A fast web fuzzer written in Go", "created_date": "2022-04-10", "author": "@liquidsec"} options = { "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt", diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index d3b40bc18..6a173a929 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -20,7 +20,7 @@ class iis_shortnames(BaseModule): meta = { "description": "Check for IIS shortname vulnerability", "created_date": "2022-04-15", - "author": "@pmueller", + "author": "@liquidsec", } options = {"detect_only": True, "max_node_count": 50} options_desc = { diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index ca2894ce3..56090f6a2 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -15,7 +15,7 @@ class paramminer_headers(BaseModule): meta = { "description": "Use smart brute-force to check for common HTTP header parameters", "created_date": "2022-04-15", - "author": "@pmueller", + "author": "@liquidsec", } options = { "wordlist": "", # default is defined within setup function diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py index 647ea342f..fbb30da7a 100644 --- a/bbot/modules/wayback.py +++ b/bbot/modules/wayback.py @@ -10,7 +10,7 @@ class wayback(subdomain_enum): meta = { "description": "Query archive.org's API for subdomains", "created_date": "2022-04-01", - "author": "@pmueller", + "author": "@liquidsec", } options = {"urls": False, "garbage_threshold": 10} options_desc = { From b8740328b62ccb4cfcbfe0e5d7b30d7e85fe4a8f Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 12:14:08 -0400 Subject: [PATCH 179/254] names --- bbot/core/helpers/names_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 95130de90..120ceff60 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -165,6 +165,7 @@ "overzealous", "paranoid", "pasty", + "peckish", "pedantic", "pernicious", "perturbed", @@ -634,6 +635,7 @@ "teresa", "terry", "theoden", + "theon", "theresa", "thomas", "tiffany", From a608f3ea0af592577b57767d22607738e1b5f549 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 3 Oct 2024 12:35:55 -0400 Subject: [PATCH 180/254] names --- bbot/core/helpers/names_generator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 95130de90..34c6166db 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -10,9 +10,11 @@ "affectionate", "aggravated", "aggrieved", + "agoraphobic", "almighty", "anal", "atrocious", + "autistic", "awkward", "baby", "begrudged", @@ -113,6 +115,7 @@ "imaginary", "immense", "immoral", + "impulsive", "incomprehensible", "inebriated", "inexplicable", @@ -277,6 +280,7 @@ "aaron", "abigail", "adam", + "adeem", "alan", "albert", "alex", @@ -414,6 +418,7 @@ "evelyn", "faramir", "florence", + "fox", "frances", "francis", "frank", @@ -508,6 +513,7 @@ "kevin", "kimberly", "kyle", + "kylie", "lantern", "larry", "laura", @@ -531,6 +537,7 @@ "lupin", "madison", "magnus", + "marcus", "margaret", "maria", "marie", @@ -629,6 +636,7 @@ "stephen", "steven", "susan", + "syrina", "tammy", "taylor", "teresa", From 09252f33ae6343499450234ba4103da9cee80c0c Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 4 Oct 2024 12:01:52 -0400 Subject: [PATCH 181/254] fix duplicate uuid for scan events --- bbot/core/helpers/names_generator.py | 2 ++ bbot/scanner/scanner.py | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 95130de90..eec538bb3 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -212,6 +212,7 @@ "sneaky", "soft", "sophisticated", + "spicy", "spiteful", "squishy", "steamy", @@ -450,6 +451,7 @@ "hermione", "homer", "howard", + "hunter", "irene", "isaac", "isabella", diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 19ea7106a..a9df6fbc3 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -116,6 +116,7 @@ def __init__( **kwargs (list[str], optional): Additional keyword arguments (passed through to `Preset`). """ self._root_event = None + self._finish_event = None self.start_time = None self.end_time = None self.duration = None @@ -377,8 +378,8 @@ async def async_start(self): new_activity = await self.finish() if not new_activity: self._success = True - await self._mark_finished() - yield self.root_event + scan_finish_event = await self._mark_finished() + yield scan_finish_event break await asyncio.sleep(0.1) @@ -434,8 +435,7 @@ async def _mark_finished(self): status_message = f"Scan {self.name} completed in {self.duration_human} with status {status}" - scan_finish_event = self.make_root_event(status_message) - scan_finish_event.data["status"] = status + scan_finish_event = self.finish_event(status_message, status) # queue final scan event with output modules output_modules = [m for m in self.modules.values() if m._type == "output" and m.name != "python"] @@ -451,6 +451,7 @@ async def _mark_finished(self): self.status = status log_fn(status_message) + return scan_finish_event def _start_modules(self): self.verbose(f"Starting module worker loops") @@ -990,6 +991,14 @@ def root_event(self): self._root_event.data["status"] = self.status return self._root_event + def finish_event(self, context=None, status=None): + if self._finish_event is None: + if context is None or status is None: + raise ValueError("Must specify context and status") + self._finish_event = self.make_root_event(context) + self._finish_event.status = status + return self._finish_event + def make_root_event(self, context): root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True, context=context) root_event._id = self.id From 5b17891f35c35bed3ecf1e36315c167d4e290889 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 4 Oct 2024 12:53:23 -0400 Subject: [PATCH 182/254] added tests --- bbot/test/test_step_1/test_scan.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index bbff8a60b..d0223acab 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -118,3 +118,26 @@ async def test_speed_counter(): await asyncio.sleep(0.2) # only 5 should show assert 4 <= counter.speed <= 5 + + +@pytest.mark.asyncio +async def test_python_output_matches_json(bbot_scanner): + import json + + scan = bbot_scanner( + "blacklanternsecurity.com", + config={"speculate": True, "dns": {"minimal": False}, "scope": {"report_distance": 10}}, + ) + await scan.helpers.dns._mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.1"]}}) + events = [e.json() async for e in scan.async_start()] + output_json = scan.home / "output.json" + json_events = [] + for line in open(output_json): + json_events.append(json.loads(line)) + + assert len(events) == 5 + assert len([e for e in events if e["type"] == "SCAN"]) == 2 + assert len([e for e in events if e["type"] == "DNS_NAME"]) == 1 + assert len([e for e in events if e["type"] == "ORG_STUB"]) == 1 + assert len([e for e in events if e["type"] == "IP_ADDRESS"]) == 1 + assert events == json_events From 80b466aaa9ff70ca8784861fd21ebd6c803f4f6e Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 4 Oct 2024 13:09:12 -0400 Subject: [PATCH 183/254] ensure status --- bbot/scanner/scanner.py | 2 +- bbot/test/test_step_1/test_scan.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index a9df6fbc3..85b9a0073 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -996,7 +996,7 @@ def finish_event(self, context=None, status=None): if context is None or status is None: raise ValueError("Must specify context and status") self._finish_event = self.make_root_event(context) - self._finish_event.status = status + self._finish_event.data["status"] = status return self._finish_event def make_root_event(self, context): diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index d0223acab..3f80807af 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -136,7 +136,9 @@ async def test_python_output_matches_json(bbot_scanner): json_events.append(json.loads(line)) assert len(events) == 5 - assert len([e for e in events if e["type"] == "SCAN"]) == 2 + scan_events = [e for e in events if e["type"] == "SCAN"] + assert len(scan_events) == 2 + assert all([isinstance(e["data"]["status"], str) for e in scan_events]) assert len([e for e in events if e["type"] == "DNS_NAME"]) == 1 assert len([e for e in events if e["type"] == "ORG_STUB"]) == 1 assert len([e for e in events if e["type"] == "IP_ADDRESS"]) == 1 From b4ec45a487fc97fd43eea176cc7fe8de168e340e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 4 Oct 2024 15:42:24 -0400 Subject: [PATCH 184/254] fixing bug with parent_url when querystring is present --- bbot/core/helpers/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index f08444cd4..30dda1c67 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -365,7 +365,7 @@ def parent_url(u): if path.parent == path: return None else: - return urlunparse(parsed._replace(path=str(path.parent))) + return urlunparse(parsed._replace(path=str(path.parent), query='')) def url_parents(u): From 23acdbfd88112836a65c450e8f846867b31f6284 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 4 Oct 2024 15:47:19 -0400 Subject: [PATCH 185/254] black --- bbot/core/helpers/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 30dda1c67..9a42732b0 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -365,7 +365,7 @@ def parent_url(u): if path.parent == path: return None else: - return urlunparse(parsed._replace(path=str(path.parent), query='')) + return urlunparse(parsed._replace(path=str(path.parent), query="")) def url_parents(u): From 495a45201aa7f99a6201dab1595d3102d8a4b8be Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 4 Oct 2024 17:17:26 -0400 Subject: [PATCH 186/254] helper to test parent_url helper --- bbot/test/test_step_1/test_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 8bb62917a..91067324d 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -49,6 +49,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): assert helpers.url_depth("http://evilcorp.com/") == 0 assert helpers.url_depth("http://evilcorp.com") == 0 + assert helpers.parent_url("http://evilcorp.com/subdir1/subdir2?foo=bar") == "http://evilcorp.com/subdir1" + ### MISC ### assert helpers.is_domain("evilcorp.co.uk") assert not helpers.is_domain("www.evilcorp.co.uk") From d0ab2347d346b09e31073022e96642c5347e1d16 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 6 Oct 2024 00:03:48 -0400 Subject: [PATCH 187/254] add subdomainradar.io module --- bbot/modules/base.py | 2 +- bbot/modules/subdomainradar.py | 127 +++++++++++ .../test_module_subdomainradar.py | 207 ++++++++++++++++++ 3 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 bbot/modules/subdomainradar.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_subdomainradar.py diff --git a/bbot/modules/base.py b/bbot/modules/base.py index df4bed023..9e709c46d 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -312,7 +312,7 @@ async def require_api_key(self): try: await self.ping() self.hugesuccess(f"API is ready") - return True + return True, "" except Exception as e: self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py new file mode 100644 index 000000000..6e8eee873 --- /dev/null +++ b/bbot/modules/subdomainradar.py @@ -0,0 +1,127 @@ +import time +import asyncio + +from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey + + +class SubdomainRadar(subdomain_enum_apikey): + watched_events = ["DNS_NAME"] + produced_events = ["DNS_NAME"] + flags = ["subdomain-enum", "passive", "safe"] + meta = { + "description": "Query the Subdomain API for subdomains", + "created_date": "2022-07-08", + "author": "@TheTechromancer", + "auth_required": True, + } + options = {"api_key": "", "group": "fast", "timeout": 120} + options_desc = {"api_key": "SubDomainRadar.io API key", "group": "", "timeout": "Timeout in seconds"} + + base_url = "https://api.subdomainradar.io" + group_choices = ("fast", "medium", "deep") + + async def setup(self): + self.group = self.config.get("group", "fast").strip().lower() + if self.group not in self.group_choices: + return False, f'Invalid group: "{self.group}", please choose from {",".join(self.group_choices)}' + success, reason = await self.require_api_key() + if not success: + return success, reason + # convert groups to enumerators + enumerators = {} + response = await self.api_request(f"{self.base_url}/enumerators/groups") + status_code = getattr(response, "status_code", 0) + if status_code != 200: + return False, f"Failed to get enumerators: (HTTP status code: {status_code})" + else: + try: + j = response.json() + except Exception: + return False, f"Failed to get enumerators: failed to parse response as JSON" + for group in j: + group_name = group.get("name", "").strip().lower() + if group_name: + group_enumerators = [] + for enumerator in group.get("enumerators", []): + enumerator_name = enumerator.get("display_name", "") + if enumerator_name: + group_enumerators.append(enumerator_name) + if group_enumerators: + enumerators[group_name] = group_enumerators + + self.enumerators = enumerators.get(self.group, []) + if not self.enumerators: + return False, f'No enumerators found for group: "{self.group}" ({self.enumerators})' + + self.enum_tasks = {} + self.poll_task = asyncio.create_task(self.task_poll_loop()) + + return True + + def prepare_api_request(self, url, kwargs): + if self.api_key: + kwargs["headers"] = {"Authorization": f"Bearer {self.api_key}"} + return url, kwargs + + async def ping(self): + url = f"{self.base_url}/profile" + response = await self.api_request(url) + assert getattr(response, "status_code", 0) == 200 + + async def handle_event(self, event): + query = self.make_query(event) + # start enumeration task + url = f"{self.base_url}/enumerate" + response = await self.api_request(url, json={"domains": [query], "enumerators": self.enumerators}) + try: + j = response.json() + except Exception: + self.warning(f"Failed to parse response as JSON: {getattr(response, 'text', '')}") + return + task_id = j.get("tasks", {}).get(query, "") + if not task_id: + self.warning(f"Failed to initiate enumeration task for {query}") + return + self.enum_tasks[query] = (task_id, time.time(), event) + self.debug(f"Started enumeration task for {query}; task id: {task_id}") + + async def task_poll_loop(self): + # async with self._task_counter.count(f"{self.name}.task_poll_loop()"): + while 1: + for query, (task_id, start_time, event) in list(self.enum_tasks.items()): + url = f"{self.base_url}/tasks/{task_id}" + response = await self.api_request(url) + if getattr(response, "status_code", 0) == 200: + finished = await self.parse_response(response, query, event) + if finished: + self.enum_tasks.pop(query) + continue + # if scan is finishing, consider timeout + if self.scan.status == "FINISHING": + if start_time + self.timeout < time.time(): + self.enum_tasks.pop(query) + self.info(f"Enumeration task for {query} timed out") + + if self.scan.status == "FINISHING" and not self.enum_tasks: + break + await self.helpers.sleep(5) + + async def parse_response(self, response, query, event): + j = response.json() + status = j.get("status", "") + if status.lower() == "completed": + for subdomain in j.get("subdomains", []): + hostname = subdomain.get("subdomain", "") + if hostname and hostname.endswith(f".{query}") and not hostname == event.data: + await self.emit_event( + hostname, + "DNS_NAME", + event, + abort_if=self.abort_if, + context=f'{{module}} searched SubDomainRadar.io API for "{query}" and found {{event.type}}: {{event.data}}', + ) + return True + return False + + async def finish(self): + await self.poll_task diff --git a/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py b/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py new file mode 100644 index 000000000..07319ee00 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py @@ -0,0 +1,207 @@ +from .base import ModuleTestBase + + +class TestSubDomainRadar(ModuleTestBase): + config_overrides = {"modules": {"subdomainradar": {"api_key": "asdf"}}} + + async def setup_before_prep(self, module_test): + await module_test.mock_dns( + { + "blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + "www.blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + "asdf.blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + } + ) + module_test.httpx_mock.add_response( + url="https://api.subdomainradar.io/profile", + match_headers={"Authorization": "Bearer asdf"}, + ) + module_test.httpx_mock.add_response( + url="https://api.subdomainradar.io/enumerate", + json={ + "tasks": {"blacklanternsecurity.com": "86de4531-0a67-41fe-b5e4-8ce8207d6245"}, + "message": "Tasks initiated", + }, + match_headers={"Authorization": "Bearer asdf"}, + ) + module_test.httpx_mock.add_response( + url="https://api.subdomainradar.io/tasks/86de4531-0a67-41fe-b5e4-8ce8207d6245", + match_headers={"Authorization": "Bearer asdf"}, + json={ + "task_id": "86de4531-0a67-41fe-b5e4-8ce8207d6245", + "status": "completed", + "domain": "blacklanternsecurity.com", + "subdomains": [ + { + "subdomain": "www.blacklanternsecurity.com", + "ip": None, + "reverse_dns": [], + "country": None, + "timestamp": None, + }, + { + "subdomain": "asdf.blacklanternsecurity.com", + "ip": None, + "reverse_dns": [], + "country": None, + "timestamp": None, + }, + ], + "total_subdomains": 2, + "rank": None, + "whois": { + "domain_name": ["BLACKLANTERNSECURITY.COM", "blacklanternsecurity.com"], + "registrar": "MarkMonitor, Inc.", + "creation_date": ["1992-11-04T05:00:00", "1992-11-04T05:00:00+00:00"], + "expiration_date": ["2026-11-03T05:00:00", "2026-11-03T00:00:00+00:00"], + "last_updated": ["2024-10-02T10:15:20", "2024-10-02T10:15:20+00:00"], + "status": [ + "clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited", + "clientTransferProhibited https://icann.org/epp#clientTransferProhibited", + "clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited", + "serverDeleteProhibited https://icann.org/epp#serverDeleteProhibited", + "serverTransferProhibited https://icann.org/epp#serverTransferProhibited", + "serverUpdateProhibited https://icann.org/epp#serverUpdateProhibited", + "clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)", + "clientTransferProhibited (https://www.icann.org/epp#clientTransferProhibited)", + "clientDeleteProhibited (https://www.icann.org/epp#clientDeleteProhibited)", + "serverUpdateProhibited (https://www.icann.org/epp#serverUpdateProhibited)", + "serverTransferProhibited (https://www.icann.org/epp#serverTransferProhibited)", + "serverDeleteProhibited (https://www.icann.org/epp#serverDeleteProhibited)", + ], + "nameservers": [ + "A1-12.AKAM.NET", + "A10-67.AKAM.NET", + "A12-64.AKAM.NET", + "A28-65.AKAM.NET", + "A7-66.AKAM.NET", + "A9-67.AKAM.NET", + "EDNS69.ULTRADNS.BIZ", + "EDNS69.ULTRADNS.COM", + "EDNS69.ULTRADNS.NET", + "EDNS69.ULTRADNS.ORG", + "edns69.ultradns.biz", + "a12-64.akam.net", + "edns69.ultradns.net", + "edns69.ultradns.org", + "a10-67.akam.net", + "a28-65.akam.net", + "a9-67.akam.net", + "a1-12.akam.net", + "a7-66.akam.net", + "edns69.ultradns.com", + ], + "emails": [ + "abusecomplaints@markmonitor.com", + "admin@dnstinations.com", + "whoisrequest@markmonitor.com", + ], + "dnssec": "unsigned", + "org": "DNStination Inc.", + "address": "3450 Sacramento Street, Suite 405", + "city": "San Francisco", + "state": "CA", + "zipcode": None, + "country": "US", + }, + "enumerators": ["Aquarius Enumerator", "Beta Enumerator", "Chi Enumerator", "Eta Enumerator"], + "timestamp": "2024-10-06T02:48:10.075636", + "error": None, + "is_notification": False, + "notification_domain_id": None, + "demo": False, + "user_id": 49, + "time_to_finish": 41, + }, + ) + module_test.httpx_mock.add_response( + url="https://api.subdomainradar.io/enumerators/groups", + match_headers={"Authorization": "Bearer asdf"}, + json=[ + { + "id": "1", + "name": "Fast", + "description": "Enumerators optimized for high-speed scanning and rapid data collection", + "enumerators": [ + {"display_name": "Beta Enumerator"}, + {"display_name": "Chi Enumerator"}, + {"display_name": "Aquarius Enumerator"}, + {"display_name": "Eta Enumerator"}, + ], + }, + { + "id": "2", + "name": "Medium", + "description": "Enumerators balanced for moderate speed with a focus on thoroughness", + "enumerators": [ + {"display_name": "Kappa Enumerator"}, + {"display_name": "Lambda Enumerator"}, + {"display_name": "Mu Enumerator"}, + {"display_name": "Pi Enumerator"}, + {"display_name": "Tau Enumerator"}, + {"display_name": "Beta Enumerator"}, + {"display_name": "Chi Enumerator"}, + {"display_name": "Psi Enumerator"}, + {"display_name": "Aquarius Enumerator"}, + {"display_name": "Zeta Enumerator"}, + {"display_name": "Eta Enumerator"}, + ], + }, + { + "id": "3", + "name": "Deep", + "description": "Enumerators designed for exhaustive searches and in-depth data analysis", + "enumerators": [ + {"display_name": "Alpha Enumerator"}, + {"display_name": "Kappa Enumerator"}, + {"display_name": "Lambda Enumerator"}, + {"display_name": "Mu Enumerator"}, + {"display_name": "Nu Enumerator"}, + {"display_name": "Xi Enumerator"}, + {"display_name": "Pi Enumerator"}, + {"display_name": "Rho Enumerator"}, + {"display_name": "Sigma Enumerator"}, + {"display_name": "Tau Enumerator"}, + {"display_name": "Beta Enumerator"}, + {"display_name": "Chi Enumerator"}, + {"display_name": "Omega Enumerator"}, + {"display_name": "Psi Enumerator"}, + {"display_name": "Phi Enumerator"}, + {"display_name": "Axon Enumerator"}, + {"display_name": "Aquarius Enumerator"}, + {"display_name": "Pegasus Enumerator"}, + {"display_name": "Petra Enumerator"}, + {"display_name": "Oasis Enumerator"}, + {"display_name": "Mike Enumerator"}, + {"display_name": "Cat Enumerator"}, + {"display_name": "Brutus Enumerator"}, + {"display_name": "Dee Enumerator"}, + {"display_name": "Jul Enumerator"}, + {"display_name": "Eve Enumerator"}, + {"display_name": "Frank Enumerator"}, + {"display_name": "Gus Enumerator"}, + {"display_name": "Hank Enumerator"}, + {"display_name": "Delta Enumerator"}, + {"display_name": "Ivy Enumerator"}, + {"display_name": "Jack Enumerator"}, + {"display_name": "Karl Enumerator"}, + {"display_name": "Liam Enumerator"}, + {"display_name": "Nora Enumerator"}, + {"display_name": "Mars Enumerator"}, + {"display_name": "Neptune Enumerator"}, + {"display_name": "Orion Enumerator"}, + {"display_name": "Oedipus Enumerator"}, + {"display_name": "Pandora Enumerator"}, + {"display_name": "Epsilon Enumerator"}, + {"display_name": "Zeta Enumerator"}, + {"display_name": "Eta Enumerator"}, + {"display_name": "Theta Enumerator"}, + {"display_name": "Iota Enumerator"}, + ], + }, + ], + ) + + def check(self, module_test, events): + assert any(e.data == "www.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #1" + assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain #2" From ce00470f6583f79826401602d76f28dc110e91b0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 6 Oct 2024 00:07:23 -0400 Subject: [PATCH 188/254] fix trufflehog macos url --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index db3497ac9..306c4bd00 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -31,7 +31,7 @@ class trufflehog(BaseModule): { "name": "Download trufflehog", "unarchive": { - "src": "https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz", + "src": "https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH}.tar.gz", "include": "trufflehog", "dest": "#{BBOT_TOOLS}", "remote_src": True, From c0a6bbbac8cd9b5857d675272c4046f7d9da9a84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 04:31:19 +0000 Subject: [PATCH 189/254] Bump pycryptodome from 3.20.0 to 3.21.0 Bumps [pycryptodome](https://github.com/Legrandin/pycryptodome) from 3.20.0 to 3.21.0. - [Release notes](https://github.com/Legrandin/pycryptodome/releases) - [Changelog](https://github.com/Legrandin/pycryptodome/blob/master/Changelog.rst) - [Commits](https://github.com/Legrandin/pycryptodome/compare/v3.20.0...v3.21.0) --- updated-dependencies: - dependency-name: pycryptodome dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 70 ++++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/poetry.lock b/poetry.lock index 75b27e134..3b1d3ef89 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1700,43 +1700,43 @@ files = [ [[package]] name = "pycryptodome" -version = "3.20.0" +version = "3.21.0" description = "Cryptographic library for Python" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"}, - {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"}, - {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"}, - {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"}, - {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"}, - {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"}, - {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"}, - {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"}, - {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"}, - {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"}, - {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"}, - {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"}, +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pycryptodome-3.21.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:dad9bf36eda068e89059d1f07408e397856be9511d7113ea4b586642a429a4fd"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:a1752eca64c60852f38bb29e2c86fca30d7672c024128ef5d70cc15868fa10f4"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:3ba4cc304eac4d4d458f508d4955a88ba25026890e8abff9b60404f76a62c55e"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cb087b8612c8a1a14cf37dd754685be9a8d9869bed2ffaaceb04850a8aeef7e"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:26412b21df30b2861424a6c6d5b1d8ca8107612a4cfa4d0183e71c5d200fb34a"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-win32.whl", hash = "sha256:cc2269ab4bce40b027b49663d61d816903a4bd90ad88cb99ed561aadb3888dd3"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-win_amd64.whl", hash = "sha256:0fa0a05a6a697ccbf2a12cec3d6d2650b50881899b845fac6e87416f8cb7e87d"}, + {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6cce52e196a5f1d6797ff7946cdff2038d3b5f0aba4a43cb6bf46b575fd1b5bb"}, + {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:a915597ffccabe902e7090e199a7bf7a381c5506a747d5e9d27ba55197a2c568"}, + {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4e74c522d630766b03a836c15bff77cb657c5fdf098abf8b1ada2aebc7d0819"}, + {file = "pycryptodome-3.21.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:a3804675283f4764a02db05f5191eb8fec2bb6ca34d466167fc78a5f05bbe6b3"}, + {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:2480ec2c72438430da9f601ebc12c518c093c13111a5c1644c82cdfc2e50b1e4"}, + {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:de18954104667f565e2fbb4783b56667f30fb49c4d79b346f52a29cb198d5b6b"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de4b7263a33947ff440412339cb72b28a5a4c769b5c1ca19e33dd6cd1dcec6e"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0714206d467fc911042d01ea3a1847c847bc10884cf674c82e12915cfe1649f8"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d85c1b613121ed3dbaa5a97369b3b757909531a959d229406a75b912dd51dd1"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8898a66425a57bcf15e25fc19c12490b87bd939800f39a03ea2de2aea5e3611a"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:932c905b71a56474bff8a9c014030bc3c882cee696b448af920399f730a650c2"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:18caa8cfbc676eaaf28613637a89980ad2fd96e00c564135bf90bc3f0b34dd93"}, + {file = "pycryptodome-3.21.0-cp36-abi3-win32.whl", hash = "sha256:280b67d20e33bb63171d55b1067f61fbd932e0b1ad976b3a184303a3dad22764"}, + {file = "pycryptodome-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b7aa25fc0baa5b1d95b7633af4f5f1838467f1815442b22487426f94e0d66c53"}, + {file = "pycryptodome-3.21.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:2cb635b67011bc147c257e61ce864879ffe6d03342dc74b6045059dfbdedafca"}, + {file = "pycryptodome-3.21.0-pp27-pypy_73-win32.whl", hash = "sha256:4c26a2f0dc15f81ea3afa3b0c87b87e501f235d332b7f27e2225ecb80c0b1cdd"}, + {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d5ebe0763c982f069d3877832254f64974139f4f9655058452603ff559c482e8"}, + {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee86cbde706be13f2dec5a42b52b1c1d1cbb90c8e405c68d0755134735c8dc6"}, + {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fd54003ec3ce4e0f16c484a10bc5d8b9bd77fa662a12b85779a2d2d85d67ee0"}, + {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5dfafca172933506773482b0e18f0cd766fd3920bd03ec85a283df90d8a17bc6"}, + {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:590ef0898a4b0a15485b05210b4a1c9de8806d3ad3d47f74ab1dc07c67a6827f"}, + {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f35e442630bc4bc2e1878482d6f59ea22e280d7121d7adeaedba58c23ab6386b"}, + {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff99f952db3db2fbe98a0b355175f93ec334ba3d01bbde25ad3a5a33abc02b58"}, + {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8acd7d34af70ee63f9a849f957558e49a98f8f1634f86a59d2be62bb8e93f71c"}, + {file = "pycryptodome-3.21.0.tar.gz", hash = "sha256:f7787e0d469bdae763b876174cf2e6c0f7be79808af26b1da96f1a64bcf47297"}, ] [[package]] From 25c1d7546493449cf52e28bed62f9488a97efe85 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 04:31:46 +0000 Subject: [PATCH 190/254] Bump pre-commit from 3.8.0 to 4.0.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.8.0 to 4.0.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.8.0...v4.0.0) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 75b27e134..8925a49fd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1621,13 +1621,13 @@ plugin = ["poetry (>=1.2.0,<2.0.0)"] [[package]] name = "pre-commit" -version = "3.8.0" +version = "4.0.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" files = [ - {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, - {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, + {file = "pre_commit-4.0.0-py2.py3-none-any.whl", hash = "sha256:0ca2341cf94ac1865350970951e54b1a50521e57b7b500403307aed4315a1234"}, + {file = "pre_commit-4.0.0.tar.gz", hash = "sha256:5d9807162cc5537940f94f266cbe2d716a75cfad0d78a317a92cac16287cfed6"}, ] [package.dependencies] @@ -3071,4 +3071,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "fb2f904f634456d6e72915cdf3040249aa2dc498106b1879732e0276df7ea082" +content-hash = "c96e967a9cebef670770ca025ed60ea5124f64e2bf2833b4c5ab215fe582a2ae" diff --git a/pyproject.toml b/pyproject.toml index b0bbf483b..9a1f1f9f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ poetry-dynamic-versioning = ">=0.21.4,<1.5.0" urllib3 = "^2.0.2" werkzeug = ">=2.3.4,<4.0.0" pytest-env = ">=0.8.2,<1.2.0" -pre-commit = "^3.4.0" +pre-commit = ">=3.4,<5.0" black = "^24.1.1" pytest-cov = "^5.0.0" pytest-rerunfailures = "^14.0" From 00ffd00ea74ac0e3a35b3f43bfe796795dbdd479 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 04:32:15 +0000 Subject: [PATCH 191/254] Bump dnspython from 2.6.1 to 2.7.0 Bumps [dnspython](https://github.com/rthalley/dnspython) from 2.6.1 to 2.7.0. - [Release notes](https://github.com/rthalley/dnspython/releases) - [Changelog](https://github.com/rthalley/dnspython/blob/main/doc/whatsnew.rst) - [Commits](https://github.com/rthalley/dnspython/compare/v2.6.1...v2.7.0) --- updated-dependencies: - dependency-name: dnspython dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 75b27e134..2e494f8f6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -595,21 +595,21 @@ files = [ [[package]] name = "dnspython" -version = "2.6.1" +version = "2.7.0" description = "DNS toolkit" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "dnspython-2.6.1-py3-none-any.whl", hash = "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50"}, - {file = "dnspython-2.6.1.tar.gz", hash = "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc"}, + {file = "dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86"}, + {file = "dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1"}, ] [package.extras] -dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "sphinx (>=7.2.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] -dnssec = ["cryptography (>=41)"] +dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "hypercorn (>=0.16.0)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "quart-trio (>=0.11.0)", "sphinx (>=7.2.0)", "sphinx-rtd-theme (>=2.0.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] +dnssec = ["cryptography (>=43)"] doh = ["h2 (>=4.1.0)", "httpcore (>=1.0.0)", "httpx (>=0.26.0)"] -doq = ["aioquic (>=0.9.25)"] -idna = ["idna (>=3.6)"] +doq = ["aioquic (>=1.0.0)"] +idna = ["idna (>=3.7)"] trio = ["trio (>=0.23)"] wmi = ["wmi (>=1.5.1)"] From e42e0e4a9542872554abfbaee3c73cbac491cbb2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 7 Oct 2024 13:44:36 -0400 Subject: [PATCH 192/254] zesty --- bbot/core/helpers/names_generator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 7ccae89e0..a0a569e53 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -93,7 +93,6 @@ "frolicking", "furry", "fuzzy", - "gay", "gentle", "giddy", "glowering", @@ -189,7 +188,6 @@ "psychic", "puffy", "pure", - "queer", "questionable", "rabid", "raging", @@ -276,6 +274,7 @@ "wispy", "witty", "woolly", + "zesty", ] names = [ From 95c0a9caa81ae10aa6b013cb8db83536995567d9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 7 Oct 2024 14:08:29 -0400 Subject: [PATCH 193/254] update description --- bbot/modules/subdomainradar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index 6e8eee873..0dafc446c 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -15,7 +15,7 @@ class SubdomainRadar(subdomain_enum_apikey): "auth_required": True, } options = {"api_key": "", "group": "fast", "timeout": 120} - options_desc = {"api_key": "SubDomainRadar.io API key", "group": "", "timeout": "Timeout in seconds"} + options_desc = {"api_key": "SubDomainRadar.io API key", "group": "The enumeration group to use. Choose from fast, medium, deep", "timeout": "Timeout in seconds"} base_url = "https://api.subdomainradar.io" group_choices = ("fast", "medium", "deep") From 5329ed1f6bba52defd8950a73a879efafc9ab966 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 7 Oct 2024 14:56:55 -0400 Subject: [PATCH 194/254] blacked --- bbot/modules/subdomainradar.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index 0dafc446c..e1a3f026f 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -15,7 +15,11 @@ class SubdomainRadar(subdomain_enum_apikey): "auth_required": True, } options = {"api_key": "", "group": "fast", "timeout": 120} - options_desc = {"api_key": "SubDomainRadar.io API key", "group": "The enumeration group to use. Choose from fast, medium, deep", "timeout": "Timeout in seconds"} + options_desc = { + "api_key": "SubDomainRadar.io API key", + "group": "The enumeration group to use. Choose from fast, medium, deep", + "timeout": "Timeout in seconds", + } base_url = "https://api.subdomainradar.io" group_choices = ("fast", "medium", "deep") From bfd4547b8c79e249d20358f892c9fab1770b86b4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 7 Oct 2024 17:35:28 -0400 Subject: [PATCH 195/254] gracefully handle timeouts/finish --- bbot/modules/base.py | 7 +++- bbot/modules/hunterio.py | 4 +- bbot/modules/ip2location.py | 4 +- bbot/modules/ipstack.py | 4 +- bbot/modules/leakix.py | 4 +- bbot/modules/postman_download.py | 3 +- bbot/modules/securitytrails.py | 4 +- bbot/modules/subdomainradar.py | 39 ++++++++++++++++--- bbot/modules/templates/github.py | 3 +- bbot/modules/templates/shodan.py | 5 +-- .../test_module_subdomainradar.py | 1 + 11 files changed, 56 insertions(+), 22 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 9e709c46d..7711d3181 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -331,7 +331,7 @@ def api_key(self, api_keys): self._api_keys = list(api_keys) def cycle_api_key(self): - if self._api_keys: + if len(self._api_keys) > 1: self.verbose(f"Cycling API key") self._api_keys.insert(0, self._api_keys.pop()) else: @@ -355,7 +355,8 @@ async def ping(self): async def ping(self): r = await self.api_request(f"{self.base_url}/ping") resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content + if getattr(r, "status_code", 0) != 200: + raise ValueError(resp_content) Returns: None @@ -1134,6 +1135,8 @@ async def api_request(self, *args, **kwargs): self._api_request_failures = 0 else: status_code = getattr(r, "status_code", 0) + response_text = getattr(r, "text", "") + self.trace(f"API response to {url} failed with status code {status_code}: {response_text}") self._api_request_failures += 1 if self._api_request_failures >= self.api_failure_abort_threshold: self.set_error_state( diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index f5a275b41..0ece92800 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -20,8 +20,8 @@ class hunterio(subdomain_enum_apikey): async def ping(self): url = f"{self.base_url}/account?api_key={{api_key}}" r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content + if getattr(r, "status_code", 0) != 200: + raise ValueError(getattr(r, "text", "API does not appear to be operational")) async def handle_event(self, event): query = self.make_query(event) diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index 4118d3471..aee3e4456 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -33,8 +33,8 @@ async def setup(self): async def ping(self): url = self.build_url("8.8.8.8") r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content + if getattr(r, "status_code", 0) != 200: + raise ValueError(getattr(r, "text", "API does not appear to be operational")) def build_url(self, data): url = f"{self.base_url}/?key={{api_key}}&ip={data}&format=json&source=bbot" diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index f3caf77f0..f332a864c 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -30,8 +30,8 @@ async def setup(self): async def ping(self): url = f"{self.base_url}/check?access_key={{api_key}}" r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content + if getattr(r, "status_code", 0) != 200: + raise ValueError(getattr(r, "text", "API does not appear to be operational")) async def handle_event(self, event): try: diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index bb63abd80..6d47f66a8 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -33,8 +33,8 @@ def prepare_api_request(self, url, kwargs): async def ping(self): url = f"{self.base_url}/host/1.2.3.4.5" r = await self.helpers.request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) != 401, resp_content + if getattr(r, "status_code", 0) != 200: + raise ValueError(getattr(r, "text", "API does not appear to be operational")) async def request_url(self, query): url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}" diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 6222e41c0..4a3e8182f 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -33,7 +33,8 @@ def prepare_api_request(self, url, kwargs): async def ping(self): url = f"{self.api_url}/me" response = await self.api_request(url) - assert getattr(response, "status_code", 0) == 200, response.text + if getattr(response, "status_code", 0) != 200: + raise ValueError(getattr(response, "text", "API does not appear to be operational")) async def filter_event(self, event): if event.type == "CODE_REPOSITORY": diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index 8a24ed1b6..6c1f19a10 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -23,8 +23,8 @@ async def setup(self): async def ping(self): url = f"{self.base_url}/ping?apikey={{api_key}}" r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content + if getattr(r, "status_code", 0) != 200: + raise ValueError(getattr(r, "text", "API does not appear to be operational")) async def request_url(self, query): url = f"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}" diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index e1a3f026f..e8d2d8025 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -26,6 +26,7 @@ class SubdomainRadar(subdomain_enum_apikey): async def setup(self): self.group = self.config.get("group", "fast").strip().lower() + self.timeout = self.config.get("timeout", 120) if self.group not in self.group_choices: return False, f'Invalid group: "{self.group}", please choose from {",".join(self.group_choices)}' success, reason = await self.require_api_key() @@ -69,14 +70,17 @@ def prepare_api_request(self, url, kwargs): async def ping(self): url = f"{self.base_url}/profile" - response = await self.api_request(url) - assert getattr(response, "status_code", 0) == 200 + r = await self.api_request(url) + if getattr(r, "status_code", 0) != 200: + raise ValueError(getattr(r, "text", "API does not appear to be operational")) async def handle_event(self, event): query = self.make_query(event) # start enumeration task url = f"{self.base_url}/enumerate" - response = await self.api_request(url, json={"domains": [query], "enumerators": self.enumerators}) + response = await self.api_request( + url, method="POST", json={"domains": [query], "enumerators": self.enumerators} + ) try: j = response.json() except Exception: @@ -84,7 +88,7 @@ async def handle_event(self, event): return task_id = j.get("tasks", {}).get(query, "") if not task_id: - self.warning(f"Failed to initiate enumeration task for {query}") + self.warning(f"Failed to start enumeration for {query}") return self.enum_tasks[query] = (task_id, time.time(), event) self.debug(f"Started enumeration task for {query}; task id: {task_id}") @@ -128,4 +132,29 @@ async def parse_response(self, response, query, event): return False async def finish(self): - await self.poll_task + start_time = time.time() + while self.enum_tasks and not self.poll_task.done(): + elapsed_time = time.time() - start_time + if elapsed_time >= self.timeout: + self.warning(f"Timed out waiting for the following tasks to finish: {self.enum_tasks}") + for query, (task_id, _, _) in list(self.enum_tasks.items()): + url = f"{self.base_url}/tasks/{task_id}" + self.warning(f" - {query} ({url})") + break + + self.verbose(f"Waiting for enumeration task poll loop to finish ({elapsed_time}/{self.timeout} seconds)") + + try: + # Wait for the task to complete or for 10 seconds, whichever comes first + await asyncio.wait_for(asyncio.shield(self.poll_task), timeout=10) + except asyncio.TimeoutError: + # This just means our 10-second check has elapsed, not that the task failed + pass + + # Cancel the poll_task if it's still running + if not self.poll_task.done(): + self.poll_task.cancel() + try: + await self.poll_task + except asyncio.CancelledError: + pass diff --git a/bbot/modules/templates/github.py b/bbot/modules/templates/github.py index 35c68a211..49353c9ff 100644 --- a/bbot/modules/templates/github.py +++ b/bbot/modules/templates/github.py @@ -45,4 +45,5 @@ async def setup(self): async def ping(self): url = f"{self.base_url}/zen" response = await self.helpers.request(url, headers=self.headers) - assert getattr(response, "status_code", 0) == 200, response.text + if getattr(response, "status_code", 0) != 200: + raise ValueError(getattr(response, "text", "API does not appear to be operational")) diff --git a/bbot/modules/templates/shodan.py b/bbot/modules/templates/shodan.py index 3cc84022d..defd300d5 100644 --- a/bbot/modules/templates/shodan.py +++ b/bbot/modules/templates/shodan.py @@ -32,10 +32,9 @@ async def setup(self): except Exception as e: self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" - return True async def ping(self): url = f"{self.base_url}/api-info?key={self.api_key}" r = await self.api_request(url) - resp_content = getattr(r, "text", "") - assert getattr(r, "status_code", 0) == 200, resp_content + if getattr(r, "status_code", 0) != 200: + raise ValueError(getattr(r, "text", "API does not appear to be operational")) diff --git a/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py b/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py index 07319ee00..c2bb827f3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py +++ b/bbot/test/test_step_2/module_tests/test_module_subdomainradar.py @@ -18,6 +18,7 @@ async def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.subdomainradar.io/enumerate", + method="POST", json={ "tasks": {"blacklanternsecurity.com": "86de4531-0a67-41fe-b5e4-8ce8207d6245"}, "message": "Tasks initiated", From 80d649647a6e682ca169d667d8d7073c01f62c21 Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 8 Oct 2024 00:24:47 +0000 Subject: [PATCH 196/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 306c4bd00..7761279e4 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -14,7 +14,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.82.6", + "version": "3.82.7", "config": "", "only_verified": True, "concurrency": 8, From 5e6c433e85bb7dfe77df23bece9fb592354bfac3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 8 Oct 2024 10:38:29 -0400 Subject: [PATCH 197/254] general cleanup for ping() function --- bbot/modules/base.py | 39 +++++++++++++++++++++---------- bbot/modules/bevigil.py | 3 --- bbot/modules/builtwith.py | 4 ---- bbot/modules/c99.py | 6 +---- bbot/modules/chaos.py | 6 +---- bbot/modules/hunterio.py | 7 +----- bbot/modules/ip2location.py | 4 +--- bbot/modules/ipstack.py | 7 +----- bbot/modules/leakix.py | 7 +----- bbot/modules/postman_download.py | 6 ----- bbot/modules/securitytrails.py | 7 +----- bbot/modules/subdomainradar.py | 7 +----- bbot/modules/templates/github.py | 7 +----- bbot/modules/templates/postman.py | 1 + bbot/modules/templates/shodan.py | 7 +----- bbot/modules/trickest.py | 10 +------- bbot/modules/virustotal.py | 4 ---- 17 files changed, 39 insertions(+), 93 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 7711d3181..60a4ffd7a 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -345,26 +345,41 @@ def api_retries(self): def api_failure_abort_threshold(self): return (self.api_retries * self._api_failure_abort_threshold) + 1 - async def ping(self): + async def ping(self, url=None): """Asynchronously checks the health of the configured API. - This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. This method should include an assert statement to validate the API's health, typically by making a test request to a known endpoint. + This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. It makes a test request to a known endpoint to validate the API's health. - Example Usage: - In your implementation, if the API has a "/ping" endpoint: - async def ping(self): - r = await self.api_request(f"{self.base_url}/ping") - resp_content = getattr(r, "text", "") - if getattr(r, "status_code", 0) != 200: - raise ValueError(resp_content) + The method uses the `ping_url` attribute if defined, or falls back to a provided URL. If neither is available, no request is made. + + Args: + url (str, optional): A specific URL to use for the ping request. If not provided, the method will use the `ping_url` attribute. Returns: None Raises: - AssertionError: If the API does not respond as expected. - """ - return + ValueError: If the API response is not successful (status code != 200). + + Example Usage: + To use this method, simply define the `ping_url` attribute in your module: + + class MyModule(BaseModule): + ping_url = "https://api.example.com/ping" + + Alternatively, you can override this method for more complex health checks: + + async def ping(self): + r = await self.api_request(f"{self.base_url}/complex-health-check") + if r.status_code != 200 or r.json().get('status') != 'healthy': + raise ValueError(f"API unhealthy: {r.text}") + """ + if url is None: + url = getattr(self, "ping_url", "") + if url: + r = await self.api_request(url) + if getattr(r, "status_code", 0) != 200: + raise ValueError(getattr(r, "text", "API does not appear to be operational")) @property def batch_size(self): diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index 50e891811..f3889e7fd 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -29,9 +29,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"]["X-Access-Token"] = self.api_key return url, kwargs - async def ping(self): - pass - async def handle_event(self, event): query = self.make_query(event) subdomains = await self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains) diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 76f8a4e39..19e880034 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -27,10 +27,6 @@ class builtwith(subdomain_enum_apikey): options_desc = {"api_key": "Builtwith API key", "redirects": "Also look up inbound and outbound redirects"} base_url = "https://api.builtwith.com" - async def ping(self): - # builtwith does not have a ping feature, so we skip it to save API credits - return - async def handle_event(self, event): query = self.make_query(event) # domains diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 062b5523e..ea7536c53 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -15,11 +15,7 @@ class c99(subdomain_enum_apikey): options_desc = {"api_key": "c99.nl API key"} base_url = "https://api.c99.nl" - - async def ping(self): - url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json" - response = await self.api_request(url) - assert response.json()["success"] == True + ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json" async def request_url(self, query): url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json" diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py index ecc960690..cba4e7ea4 100644 --- a/bbot/modules/chaos.py +++ b/bbot/modules/chaos.py @@ -15,11 +15,7 @@ class chaos(subdomain_enum_apikey): options_desc = {"api_key": "Chaos API key"} base_url = "https://dns.projectdiscovery.io/dns" - - async def ping(self): - url = f"{self.base_url}/example.com" - response = await self.api_request(url) - assert response.json()["domain"] == "example.com" + ping_url = f"{base_url}/example.com" def prepare_api_request(self, url, kwargs): kwargs["headers"]["Authorization"] = self.api_key diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index 0ece92800..8977ddfbe 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -15,14 +15,9 @@ class hunterio(subdomain_enum_apikey): options_desc = {"api_key": "Hunter.IO API key"} base_url = "https://api.hunter.io/v2" + ping_url = f"{base_url}/account?api_key={{api_key}}" limit = 100 - async def ping(self): - url = f"{self.base_url}/account?api_key={{api_key}}" - r = await self.api_request(url) - if getattr(r, "status_code", 0) != 200: - raise ValueError(getattr(r, "text", "API does not appear to be operational")) - async def handle_event(self, event): query = self.make_query(event) for entry in await self.query(query): diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index aee3e4456..2a4b387f4 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -32,9 +32,7 @@ async def setup(self): async def ping(self): url = self.build_url("8.8.8.8") - r = await self.api_request(url) - if getattr(r, "status_code", 0) != 200: - raise ValueError(getattr(r, "text", "API does not appear to be operational")) + await super().ping(url) def build_url(self, data): url = f"{self.base_url}/?key={{api_key}}&ip={data}&format=json&source=bbot" diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index f332a864c..02cfe0f3d 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -23,16 +23,11 @@ class Ipstack(BaseModule): suppress_dupes = False base_url = "http://api.ipstack.com" + ping_url = f"{base_url}/check?access_key={{api_key}}" async def setup(self): return await self.require_api_key() - async def ping(self): - url = f"{self.base_url}/check?access_key={{api_key}}" - r = await self.api_request(url) - if getattr(r, "status_code", 0) != 200: - raise ValueError(getattr(r, "text", "API does not appear to be operational")) - async def handle_event(self, event): try: url = f"{self.base_url}/{event.data}?access_key={{api_key}}" diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index 6d47f66a8..22be7513d 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -15,6 +15,7 @@ class leakix(subdomain_enum_apikey): } base_url = "https://leakix.net" + ping_url = f"{base_url}/host/1.2.3.4.5" async def setup(self): ret = await super(subdomain_enum_apikey, self).setup() @@ -30,12 +31,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"]["api-key"] = self.api_key return url, kwargs - async def ping(self): - url = f"{self.base_url}/host/1.2.3.4.5" - r = await self.helpers.request(url) - if getattr(r, "status_code", 0) != 200: - raise ValueError(getattr(r, "text", "API does not appear to be operational")) - async def request_url(self, query): url = f"{self.base_url}/api/subdomains/{self.helpers.quote(query)}" response = await self.api_request(url) diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 4a3e8182f..104f4261b 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -30,12 +30,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"]["X-Api-Key"] = self.api_key return url, kwargs - async def ping(self): - url = f"{self.api_url}/me" - response = await self.api_request(url) - if getattr(response, "status_code", 0) != 200: - raise ValueError(getattr(response, "text", "API does not appear to be operational")) - async def filter_event(self, event): if event.type == "CODE_REPOSITORY": if "postman" not in event.tags: diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index 6c1f19a10..c74450307 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -15,17 +15,12 @@ class securitytrails(subdomain_enum_apikey): options_desc = {"api_key": "SecurityTrails API key"} base_url = "https://api.securitytrails.com/v1" + ping_url = f"{base_url}/ping?apikey={{api_key}}" async def setup(self): self.limit = 100 return await super().setup() - async def ping(self): - url = f"{self.base_url}/ping?apikey={{api_key}}" - r = await self.api_request(url) - if getattr(r, "status_code", 0) != 200: - raise ValueError(getattr(r, "text", "API does not appear to be operational")) - async def request_url(self, query): url = f"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}" response = await self.api_request(url) diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index e8d2d8025..0b945eee7 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -22,6 +22,7 @@ class SubdomainRadar(subdomain_enum_apikey): } base_url = "https://api.subdomainradar.io" + ping_url = f"{base_url}/profile" group_choices = ("fast", "medium", "deep") async def setup(self): @@ -68,12 +69,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"] = {"Authorization": f"Bearer {self.api_key}"} return url, kwargs - async def ping(self): - url = f"{self.base_url}/profile" - r = await self.api_request(url) - if getattr(r, "status_code", 0) != 200: - raise ValueError(getattr(r, "text", "API does not appear to be operational")) - async def handle_event(self, event): query = self.make_query(event) # start enumeration task diff --git a/bbot/modules/templates/github.py b/bbot/modules/templates/github.py index 49353c9ff..b5c4403e3 100644 --- a/bbot/modules/templates/github.py +++ b/bbot/modules/templates/github.py @@ -11,6 +11,7 @@ class github(BaseModule): _qsize = 1 base_url = "https://api.github.com" + ping_url = f"{base_url}/zen" def prepare_api_request(self, url, kwargs): kwargs["headers"]["Authorization"] = f"token {self.api_key}" @@ -41,9 +42,3 @@ async def setup(self): self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" return True - - async def ping(self): - url = f"{self.base_url}/zen" - response = await self.helpers.request(url, headers=self.headers) - if getattr(response, "status_code", 0) != 200: - raise ValueError(getattr(response, "text", "API does not appear to be operational")) diff --git a/bbot/modules/templates/postman.py b/bbot/modules/templates/postman.py index 38cc3d04b..2048abc32 100644 --- a/bbot/modules/templates/postman.py +++ b/bbot/modules/templates/postman.py @@ -10,6 +10,7 @@ class postman(BaseModule): base_url = "https://www.postman.com/_api" api_url = "https://api.getpostman.com" html_url = "https://www.postman.com" + ping_url = f"{api_url}/me" headers = { "Content-Type": "application/json", diff --git a/bbot/modules/templates/shodan.py b/bbot/modules/templates/shodan.py index defd300d5..536046439 100644 --- a/bbot/modules/templates/shodan.py +++ b/bbot/modules/templates/shodan.py @@ -8,6 +8,7 @@ class shodan(subdomain_enum): options_desc = {"api_key": "Shodan API key"} base_url = "https://api.shodan.io" + ping_url = f"{base_url}/api-info?key={{api_key}}" async def setup(self): await super().setup() @@ -32,9 +33,3 @@ async def setup(self): except Exception as e: self.trace(traceback.format_exc()) return None, f"Error with API ({str(e).strip()})" - - async def ping(self): - url = f"{self.base_url}/api-info?key={self.api_key}" - r = await self.api_request(url) - if getattr(r, "status_code", 0) != 200: - raise ValueError(getattr(r, "text", "API does not appear to be operational")) diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py index 33b4d672c..c17aa6160 100644 --- a/bbot/modules/trickest.py +++ b/bbot/modules/trickest.py @@ -19,6 +19,7 @@ class Trickest(subdomain_enum_apikey): } base_url = "https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be" + ping_url = f"{base_url}/dataset" dataset_id = "a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc" page_size = 50 @@ -26,15 +27,6 @@ def prepare_api_request(self, url, kwargs): kwargs["headers"]["Authorization"] = f"Token {self.api_key}" return url, kwargs - async def ping(self): - url = f"{self.base_url}/dataset" - response = await self.api_request(url) - status_code = getattr(response, "status_code", 0) - if status_code != 200: - response_text = getattr(response, "text", "no response from server") - return False, response_text - return True - async def handle_event(self, event): query = self.make_query(event) async for result_batch in self.query(query): diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index 87e82766d..98f469e39 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -16,10 +16,6 @@ class virustotal(subdomain_enum_apikey): base_url = "https://www.virustotal.com/api/v3" - async def ping(self): - # virustotal does not have a ping function - return - def prepare_api_request(self, url, kwargs): kwargs["headers"]["x-apikey"] = self.api_key return url, kwargs From a22c1c24cf22de787ce7581dae08df161af45433 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 8 Oct 2024 11:11:01 -0400 Subject: [PATCH 198/254] small cleanups --- bbot/modules/subdomainradar.py | 5 ++++- bbot/scanner/scanner.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index 0b945eee7..ffc7b8fe4 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -25,6 +25,9 @@ class SubdomainRadar(subdomain_enum_apikey): ping_url = f"{base_url}/profile" group_choices = ("fast", "medium", "deep") + # set this really high so the poll loop finishes as soon as possible + _qsize = 9999999 + async def setup(self): self.group = self.config.get("group", "fast").strip().lower() self.timeout = self.config.get("timeout", 120) @@ -137,7 +140,7 @@ async def finish(self): self.warning(f" - {query} ({url})") break - self.verbose(f"Waiting for enumeration task poll loop to finish ({elapsed_time}/{self.timeout} seconds)") + self.verbose(f"Waiting for enumeration task poll loop to finish ({int(elapsed_time)}/{self.timeout} seconds)") try: # Wait for the task to complete or for 10 seconds, whichever comes first diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 85b9a0073..35cbaf220 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -444,7 +444,6 @@ async def _mark_finished(self): # wait until output modules are flushed while 1: modules_finished = all([m.finished for m in output_modules]) - self.verbose(modules_finished) if modules_finished: break await asyncio.sleep(0.05) From 2a4e88f526d7eda8b293151a3ed5d7330b04e4ac Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 8 Oct 2024 11:12:42 -0400 Subject: [PATCH 199/254] comments --- bbot/modules/subdomainradar.py | 4 +++- bbot/modules/templates/subdomain_enum.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index ffc7b8fe4..16d2a564f 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -140,7 +140,9 @@ async def finish(self): self.warning(f" - {query} ({url})") break - self.verbose(f"Waiting for enumeration task poll loop to finish ({int(elapsed_time)}/{self.timeout} seconds)") + self.verbose( + f"Waiting for enumeration task poll loop to finish ({int(elapsed_time)}/{self.timeout} seconds)" + ) try: # Wait for the task to complete or for 10 seconds, whichever comes first diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index e161636b2..95a040b1c 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -20,8 +20,8 @@ class subdomain_enum(BaseModule): # whether to reject wildcard DNS_NAMEs reject_wildcards = "strict" - # set qsize to 10. this helps combat rate limiting by ensuring that a query doesn't execute - # until the queue is ready to receive its results + # set qsize to 10. this helps combat rate limiting by ensuring the next query doesn't execute + # until the result from the previous queue have been consumed by the scan # we don't use 1 because it causes delays due to the asyncio.sleep; 10 gives us reasonable buffer room _qsize = 10 From 071132058037e51d2c65ceaa64893995e10c9de7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 8 Oct 2024 16:19:24 -0400 Subject: [PATCH 200/254] fix c99 ping --- bbot/modules/base.py | 3 ++- bbot/modules/c99.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 60a4ffd7a..946506094 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -379,7 +379,8 @@ async def ping(self): if url: r = await self.api_request(url) if getattr(r, "status_code", 0) != 200: - raise ValueError(getattr(r, "text", "API does not appear to be operational")) + response_text = getattr(r, "text", "no response from server") + raise ValueError(response_text) @property def batch_size(self): diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index ea7536c53..7e703966b 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -17,6 +17,11 @@ class c99(subdomain_enum_apikey): base_url = "https://api.c99.nl" ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json" + async def ping(self): + url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json" + response = await self.api_request(url) + assert response.json()["success"] == True, getattr(response, "text", "no response from server") + async def request_url(self, query): url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json" return await self.api_request(url) From 469435f7e861877c3c491938b7b7436cb6369eeb Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Tue, 8 Oct 2024 21:29:55 +0100 Subject: [PATCH 201/254] Adding playstore module WIP --- bbot/core/event/base.py | 12 +++ bbot/modules/google_playstore.py | 90 +++++++++++++++++++ .../test_module_google_playstore.py | 83 +++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 bbot/modules/google_playstore.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_google_playstore.py diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index d5a9552e2..74973dde2 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1537,6 +1537,18 @@ class RAW_DNS_RECORD(DictHostEvent, DnsEvent): _always_emit_tags = ["target"] +class MOBILE_APP(DictEvent): + _always_emit = True + + # class _data_validator(BaseModel): + # app_id: str + # url: str + # _validate_url = field_validator("url")(validators.validate_url) + + def _pretty_string(self): + return self.data["url"] + + def make_event( data, event_type=None, diff --git a/bbot/modules/google_playstore.py b/bbot/modules/google_playstore.py new file mode 100644 index 000000000..4feb6dea9 --- /dev/null +++ b/bbot/modules/google_playstore.py @@ -0,0 +1,90 @@ +from bbot.modules.base import BaseModule + + +class google_playstore(BaseModule): + watched_events = ["ORG_STUB", "CODE_REPOSITORY"] + produced_events = ["MOBILE_APP"] + flags = ["passive", "safe", "code-enum"] + meta = { + "description": "Search for android applications on play.google.com", + "created_date": "2024-10-08", + "author": "@domwhewell-sage", + } + + base_url = "https://play.google.com" + + async def handle_event(self, event): + if event.type == "CODE_REPOSITORY": + await self.handle_url(event) + elif event.type == "ORG_STUB": + await self.handle_org_stub(event) + + async def handle_url(self, event): + repo_url = event.data.get("url") + app_id = repo_url.split("id=")[1].split("&")[0] + await self.emit_event( + {"id": app_id, "url": repo_url}, + "MOBILE_APP", + tags="android", + parent=event, + context=f'{{module}} extracted the mobile app name "{app_id}" from: {repo_url}', + ) + + async def handle_org_stub(self, event): + org_name = event.data + self.verbose(f"Searching for any android applications for {org_name}") + for apk_name in await self.query(org_name): + valid_apk = await self.validate_apk(apk_name) + if valid_apk: + self.verbose(f"Got {apk_name} from playstore") + await self.emit_event( + {"id": apk_name, "url": f"{self.base_url}/store/apps/details?id={apk_name}"}, + "MOBILE_APP", + tags="android", + parent=event, + context=f'{{module}} searched play.google.com for apps belonging to "{org_name}" and found "{apk_name}" to be in scope', + ) + else: + self.debug(f"Got {apk_name} from playstore app details does not contain any in-scope URLs or Emails") + + async def query(self, query): + app_links = [] + url = f"{self.base_url}/store/search?q={self.helpers.quote(query)}&c=apps" + r = await self.helpers.request(url) + if r is None: + return app_links + status_code = getattr(r, "status_code", 0) + try: + html = self.helpers.beautifulsoup(r.content, "html.parser") + except Exception as e: + self.warning(f"Failed to parse html response from {r.url} (HTTP status: {status_code}): {e}") + return app_links + links = html.find_all("a", href=True) + app_links = [a["href"].split("id=")[1].split("&")[0] for a in links if "/store/apps/details?id=" in a["href"]] + return app_links + + async def validate_apk(self, apk_name): + in_scope = False + url = f"{self.base_url}/store/apps/details?id={apk_name}" + r = await self.helpers.request(url) + if r is None: + return in_scope + status_code = getattr(r, "status_code", 0) + try: + html = self.helpers.beautifulsoup(r.content, "html.parser") + except Exception as e: + self.warning(f"Failed to parse html response from {r.url} (HTTP status: {status_code}): {e}") + return in_scope + # The developer meta tag usually contains the developer's URL + developer_meta = html.find("meta", attrs={"name": "appstore:developer_url"}) + developer_url = developer_meta["content"] if developer_meta else None + if self.scan.in_scope(developer_url): + in_scope = True + # If the developers URL is left blank then a support email is usually provided + links = html.find_all("a", href=True) + emails = [a["href"].split("mailto:")[1] for a in links if "mailto:" in a["href"]] + for email in emails: + if self.scan.in_scope(email): + in_scope = True + break + return in_scope diff --git a/bbot/test/test_step_2/module_tests/test_module_google_playstore.py b/bbot/test/test_step_2/module_tests/test_module_google_playstore.py new file mode 100644 index 000000000..f73a79bf3 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_google_playstore.py @@ -0,0 +1,83 @@ +from .base import ModuleTestBase + + +class TestGoogle_Playstore(ModuleTestBase): + modules_overrides = ["google_playstore", "speculate"] + + async def setup_after_prep(self, module_test): + await module_test.mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.99"]}}) + module_test.httpx_mock.add_response( + url="https://play.google.com/store/search?q=blacklanternsecurity&c=apps", + text=""" + + + "blacklanternsecurity" - Android Apps on Google Play + + + + + + """, + ) + module_test.httpx_mock.add_response( + url="https://play.google.com/store/apps/details?id=com.bbot.test", + text=""" + + + BBOT + + + + + + + """, + ) + module_test.httpx_mock.add_response( + url="https://play.google.com/store/apps/details?id=com.bbot.other", + text=""" + + + BBOT + + + + + + + + """, + ) + + def check(self, module_test, events): + assert len(events) == 6 + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com" and e.scope_distance == 0 + ] + ), "Failed to emit target DNS_NAME" + assert 1 == len( + [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 0] + ), "Failed to find ORG_STUB" + assert 1 == len( + [ + e + for e in events + if e.type == "MOBILE_APP" + and "android" in e.tags + and e.data["id"] == "com.bbot.test" + and e.data["url"] == "https://play.google.com/store/apps/details?id=com.bbot.test" + ] + ), "Failed to find bbot android app" + assert 1 == len( + [ + e + for e in events + if e.type == "MOBILE_APP" + and "android" in e.tags + and e.data["id"] == "com.bbot.other" + and e.data["url"] == "https://play.google.com/store/apps/details?id=com.bbot.other" + ] + ), "Failed to find other bbot android app" From 3e5c13b4349e83ec1733a7eb180146cd01a21092 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Tue, 8 Oct 2024 21:46:25 +0100 Subject: [PATCH 202/254] added in filter event --- bbot/modules/google_playstore.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bbot/modules/google_playstore.py b/bbot/modules/google_playstore.py index 4feb6dea9..861febb2f 100644 --- a/bbot/modules/google_playstore.py +++ b/bbot/modules/google_playstore.py @@ -13,6 +13,12 @@ class google_playstore(BaseModule): base_url = "https://play.google.com" + async def filter_event(self, event): + if event.type == "CODE_REPOSITORY": + if "android" not in event.tags: + return False, "event is not an android repository" + return True + async def handle_event(self, event): if event.type == "CODE_REPOSITORY": await self.handle_url(event) From 469fbc65fa94c1926ea97a846ad179e0898b741f Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Wed, 9 Oct 2024 13:38:18 +0100 Subject: [PATCH 203/254] trufflehog fix --- bbot/modules/trufflehog.py | 11 ++--------- bbot/modules/unstructured.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 7761279e4..ee3f7416f 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -1,5 +1,4 @@ import json -from pathlib import Path from bbot.modules.base import BaseModule @@ -65,7 +64,6 @@ async def setup(self): if not self.github_token: self.deleted_forks = False return None, "A github api_key must be provided to the github modules for deleted forks to be scanned" - self.processed = set() return True async def filter_event(self, event): @@ -78,12 +76,8 @@ async def filter_event(self, event): else: return False, "Deleted forks is not enabled" else: - path = event.data["path"] - for processed in self.processed: - processed_path = Path(processed) - new_path = Path(path) - if new_path.is_relative_to(processed_path): - return False, "Parent folder has already been processed" + if "parsed-folder" in event.tags: + return False, "Not accepting parsed-folder events" return True async def handle_event(self, event): @@ -94,7 +88,6 @@ async def handle_event(self, event): module = "github-experimental" else: path = event.data["path"] - self.processed.add(path) if "git" in event.tags: module = "git" elif "docker" in event.tags: diff --git a/bbot/modules/unstructured.py b/bbot/modules/unstructured.py index 9c5e58996..c1440a1f3 100644 --- a/bbot/modules/unstructured.py +++ b/bbot/modules/unstructured.py @@ -94,7 +94,7 @@ async def handle_event(self, event): if not any(ignored_folder in str(file_path) for ignored_folder in self.ignored_folders): if any(file_path.name.endswith(f".{ext}") for ext in self.extensions): file_event = self.make_event( - {"path": str(file_path)}, "FILESYSTEM", tags=["parsed_folder", "file"], parent=event + {"path": str(file_path)}, "FILESYSTEM", tags=["parsed-folder", "file"], parent=event ) await self.emit_event(file_event) elif "file" in event.tags: From d6200bee1cfbc0f7527138a5443a2bfa45e76561 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 9 Oct 2024 11:21:06 -0400 Subject: [PATCH 204/254] fix github_org api key bug --- bbot/modules/templates/github.py | 8 +++++--- .../module_tests/test_module_github_org.py | 19 +++++++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/bbot/modules/templates/github.py b/bbot/modules/templates/github.py index b5c4403e3..6769d00ab 100644 --- a/bbot/modules/templates/github.py +++ b/bbot/modules/templates/github.py @@ -21,15 +21,17 @@ async def setup(self): await super().setup() self.headers = {} api_keys = set() - for module_name in ("github", "github_codesearch", "github_org", "git_clone"): - module_config = self.scan.config.get("modules", {}).get(module_name, {}) + modules_config = self.scan.config.get("modules", {}) + git_modules = [m for m in modules_config if str(m).startswith("git")] + for module_name in git_modules: + module_config = modules_config.get(module_name, {}) api_key = module_config.get("api_key", "") if isinstance(api_key, str): api_key = [api_key] for key in api_key: key = key.strip() if key: - api_keys.update(key) + api_keys.add(key) if not api_keys: if self.auth_required: return None, "No API key set" diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py index 039c6125b..d8003fd2a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_github_org.py +++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py @@ -10,9 +10,12 @@ async def setup_before_prep(self, module_test): {"blacklanternsecurity.com": {"A": ["127.0.0.99"]}, "github.com": {"A": ["127.0.0.99"]}} ) - module_test.httpx_mock.add_response(url="https://api.github.com/zen") + module_test.httpx_mock.add_response( + url="https://api.github.com/zen", match_headers={"Authorization": "token asdf"} + ) module_test.httpx_mock.add_response( url="https://api.github.com/orgs/blacklanternsecurity", + match_headers={"Authorization": "token asdf"}, json={ "login": "blacklanternsecurity", "id": 25311592, @@ -48,6 +51,7 @@ async def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.github.com/orgs/blacklanternsecurity/repos?per_page=100&page=1", + match_headers={"Authorization": "token asdf"}, json=[ { "id": 459780477, @@ -154,6 +158,7 @@ async def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.github.com/orgs/blacklanternsecurity/members?per_page=100&page=1", + match_headers={"Authorization": "token asdf"}, json=[ { "login": "TheTechromancer", @@ -179,6 +184,7 @@ async def setup_before_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://api.github.com/users/TheTechromancer/repos?per_page=100&page=1", + match_headers={"Authorization": "token asdf"}, json=[ { "id": 688270318, @@ -332,7 +338,7 @@ def check(self, module_test, events): class TestGithub_Org_No_Members(TestGithub_Org): - config_overrides = {"modules": {"github_org": {"include_members": False}}} + config_overrides = {"modules": {"github_org": {"include_members": False}, "github": {"api_key": "asdf"}}} def check(self, module_test, events): assert len(events) == 6 @@ -360,7 +366,7 @@ def check(self, module_test, events): class TestGithub_Org_MemberRepos(TestGithub_Org): - config_overrides = {"modules": {"github_org": {"include_member_repos": True}}} + config_overrides = {"modules": {"github_org": {"include_member_repos": True}, "github": {"api_key": "asdf"}}} def check(self, module_test, events): assert len(events) == 8 @@ -378,7 +384,12 @@ def check(self, module_test, events): class TestGithub_Org_Custom_Target(TestGithub_Org): targets = ["ORG:blacklanternsecurity"] - config_overrides = {"scope": {"report_distance": 10}, "omit_event_types": [], "speculate": True} + config_overrides = { + "scope": {"report_distance": 10}, + "omit_event_types": [], + "speculate": True, + "modules": {"github": {"api_key": "asdf"}}, + } def check(self, module_test, events): assert len(events) == 8 From 94f1c1696a152675bf4a590f532cd8e9596ee6d7 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Thu, 10 Oct 2024 10:45:10 +0100 Subject: [PATCH 205/254] Move folder crawling to speculate --- bbot/modules/internal/speculate.py | 25 ++++++++++++++++-- bbot/modules/unstructured.py | 41 ++++++++++-------------------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 622c0bfe0..53d22aede 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -1,5 +1,6 @@ import random import ipaddress +from pathlib import Path from bbot.core.helpers import validators from bbot.modules.internal.base import BaseInternalModule @@ -23,8 +24,9 @@ class speculate(BaseInternalModule): "SOCIAL", "AZURE_TENANT", "USERNAME", + "FILESYSTEM", ] - produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB"] + produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB", "FILESYSTEM"] flags = ["passive"] meta = { "description": "Derive certain event types from others by common sense", @@ -32,10 +34,11 @@ class speculate(BaseInternalModule): "author": "@liquidsec", } - options = {"max_hosts": 65536, "ports": "80,443"} + options = {"max_hosts": 65536, "ports": "80,443", "ignore_folders": [".git"]} options_desc = { "max_hosts": "Max number of IP_RANGE hosts to convert into IP_ADDRESS events", "ports": "The set of ports to speculate on", + "ignore_folders": "Subfolders to ignore when crawling downloaded folders", } scope_distance_modifier = 1 _priority = 4 @@ -72,6 +75,13 @@ async def setup(self): self.hugewarning(f'Enabling the "portscan" module is highly recommended') self.range_to_ip = False + self.ignored_folders = self.config.get("ignore_folders", []) + + return True + + async def filter_event(self, event): + if event.type == "FILESYSTEM" and "folder" not in event.tags: + return False, "Event is not a folder" return True async def handle_event(self, event): @@ -196,3 +206,14 @@ async def handle_event(self, event): email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=["affiliate"]) if email_event: await self.emit_event(email_event, context="detected {event.type}: {event.data}") + + # FILESYSTEM (folder) --> FILESYSTEM (files) + if event.type == "FILESYSTEM": + folder_path = Path(event.data["path"]) + for file_path in folder_path.rglob("*"): + # If the file is not in an ignored folder and if it has an allowed extension raise it as a FILESYSTEM event + if not any(ignored_folder in str(file_path) for ignored_folder in self.ignored_folders): + file_event = self.make_event( + {"path": str(file_path)}, "FILESYSTEM", tags=["parsed_folder", "file"], parent=event + ) + await self.emit_event(file_event) diff --git a/bbot/modules/unstructured.py b/bbot/modules/unstructured.py index 9c5e58996..cf58a59b6 100644 --- a/bbot/modules/unstructured.py +++ b/bbot/modules/unstructured.py @@ -1,12 +1,11 @@ import os -from pathlib import Path from bbot.modules.base import BaseModule class unstructured(BaseModule): watched_events = ["FILESYSTEM"] - produced_events = ["FILESYSTEM", "RAW_TEXT"] + produced_events = ["RAW_TEXT"] flags = ["passive", "safe"] meta = { "description": "Module to extract data from files", @@ -59,11 +58,9 @@ class unstructured(BaseModule): "yml", # YAML Ain't Markup Language "yaml", # YAML Ain't Markup Language ], - "ignore_folders": [".git"], } options_desc = { "extensions": "File extensions to parse", - "ignore_folders": "Subfolders to ignore when crawling downloaded folders", } deps_apt = ["libmagic-dev", "poppler-utils", "tesseract-ocr", "libreoffice", "pandoc"] @@ -73,41 +70,29 @@ class unstructured(BaseModule): async def setup(self): self.extensions = list(set([e.lower().strip(".") for e in self.config.get("extensions", [])])) - self.ignored_folders = self.config.get("ignore_folders", []) # Do not send user statistics to the unstructured library os.environ["SCARF_NO_ANALYTICS"] = "true" return True async def filter_event(self, event): - if "file" not in event.tags and "folder" not in event.tags: - return False, "Event is not a file or folder" if "file" in event.tags: if not any(event.data["path"].endswith(f".{ext}") for ext in self.extensions): return False, "File extension not in the allowed list" + else: + return False, "Event is not a file" return True async def handle_event(self, event): - if "folder" in event.tags: - folder_path = Path(event.data["path"]) - for file_path in folder_path.rglob("*"): - # If the file is not in an ignored folder and if it has an allowed extension raise it as a FILESYSTEM event - if not any(ignored_folder in str(file_path) for ignored_folder in self.ignored_folders): - if any(file_path.name.endswith(f".{ext}") for ext in self.extensions): - file_event = self.make_event( - {"path": str(file_path)}, "FILESYSTEM", tags=["parsed_folder", "file"], parent=event - ) - await self.emit_event(file_event) - elif "file" in event.tags: - file_path = event.data["path"] - content = await self.scan.helpers.run_in_executor_mp(extract_text, file_path) - if content: - raw_text_event = self.make_event( - content, - "RAW_TEXT", - context=f"Extracted text from {file_path}", - parent=event, - ) - await self.emit_event(raw_text_event) + file_path = event.data["path"] + content = await self.scan.helpers.run_in_executor_mp(extract_text, file_path) + if content: + raw_text_event = self.make_event( + content, + "RAW_TEXT", + context=f"Extracted text from {file_path}", + parent=event, + ) + await self.emit_event(raw_text_event) async def finish(self): del os.environ["SCARF_NO_ANALYTICS"] From b4a343bab102279d4d22bee877a2923d7d83ff4e Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Thu, 10 Oct 2024 12:07:54 +0100 Subject: [PATCH 206/254] Remove unstructured from trufflehog test as not required & set gitclone test to get the folder event --- bbot/test/test_step_2/module_tests/test_module_git_clone.py | 2 +- bbot/test/test_step_2/module_tests/test_module_trufflehog.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_git_clone.py b/bbot/test/test_step_2/module_tests/test_module_git_clone.py index 15bc54fb3..f0b91a2a1 100644 --- a/bbot/test/test_step_2/module_tests/test_module_git_clone.py +++ b/bbot/test/test_step_2/module_tests/test_module_git_clone.py @@ -215,7 +215,7 @@ class TestGit_CloneWithBlob(TestGit_Clone): config_overrides = {"folder_blobs": True} def check(self, module_test, events): - filesystem_events = [e for e in events if e.type == "FILESYSTEM"] + filesystem_events = [e for e in events if e.type == "FILESYSTEM" and "folder" in e.tags] assert len(filesystem_events) == 1 assert all(["blob" in e.data for e in filesystem_events]) filesystem_event = filesystem_events[0] diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py index a838ad6ab..46798dd94 100644 --- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py +++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py @@ -14,7 +14,6 @@ class TestTrufflehog(ModuleTestBase): "github_org", "speculate", "git_clone", - "unstructured", "github_workflows", "dockerhub", "docker_pull", @@ -1135,7 +1134,7 @@ def check(self, module_test, events): and "Raw result: [https://admin:admin@the-internet.herokuapp.com]" in e.data["description"] and "RawV2 result: [https://admin:admin@the-internet.herokuapp.com/basic_auth]" in e.data["description"] ] - # Trufflehog should find 4 verifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder + # Trufflehog should find 4 verifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman. assert 4 == len(vuln_events), "Failed to find secret in events" github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].parent folder = Path(github_repo_event.data["path"]) @@ -1197,7 +1196,7 @@ def check(self, module_test, events): and "Potential Secret Found." in e.data["description"] and "Raw result: [https://admin:admin@internal.host.com]" in e.data["description"] ] - # Trufflehog should find 4 unverifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder + # Trufflehog should find 4 unverifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman. assert 4 == len(finding_events), "Failed to find secret in events" github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].parent folder = Path(github_repo_event.data["path"]) From fba30f6ce1767114339f61d8c5e128d38c558dbf Mon Sep 17 00:00:00 2001 From: GitHub Date: Fri, 11 Oct 2024 00:24:51 +0000 Subject: [PATCH 207/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index ee3f7416f..159f0a057 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -13,7 +13,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.82.7", + "version": "3.82.8", "config": "", "only_verified": True, "concurrency": 8, From 6f25268c82cf6b3dcd1ceaf4567b0c67a2c6f406 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Fri, 11 Oct 2024 17:48:18 +0100 Subject: [PATCH 208/254] Remove the `_data_validator` from `MOBILE_APP` event --- bbot/core/event/base.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 74973dde2..39e344de7 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1540,11 +1540,6 @@ class RAW_DNS_RECORD(DictHostEvent, DnsEvent): class MOBILE_APP(DictEvent): _always_emit = True - # class _data_validator(BaseModel): - # app_id: str - # url: str - # _validate_url = field_validator("url")(validators.validate_url) - def _pretty_string(self): return self.data["url"] From be9ddade3d0e612a53bf4b129a4017abcde3cc31 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 13 Oct 2024 22:49:56 -0400 Subject: [PATCH 209/254] Convert excavate to intercept module --- bbot/modules/base.py | 3 +- bbot/modules/httpx.py | 3 - bbot/modules/internal/cloudcheck.py | 4 +- bbot/modules/internal/dnsresolve.py | 4 +- bbot/modules/internal/excavate.py | 56 +++++++++++++++++-- bbot/scanner/manager.py | 6 +- .../module_tests/test_module_httpx.py | 4 +- 7 files changed, 63 insertions(+), 17 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 946506094..89660edb2 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1559,7 +1559,7 @@ def critical(self, *args, trace=True, **kwargs): self.trace() -class InterceptModule(BaseModule): +class BaseInterceptModule(BaseModule): """ An Intercept Module is a special type of high-priority module that gets early access to events. @@ -1571,7 +1571,6 @@ class InterceptModule(BaseModule): """ accept_dupes = True - suppress_dupes = False _intercept = True async def _worker(self): diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index deda243a0..2cd2c0504 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -172,9 +172,6 @@ async def handle_batch(self, *events): httpx_ip = j.get("host", "") if httpx_ip: tags.append(f"ip-{httpx_ip}") - # detect login pages - if self.helpers.web.is_login_page(j.get("body", "")): - tags.append("login-page") # grab title title = self.helpers.tagify(j.get("title", ""), maxlen=30) if title: diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py index 9b7b6e147..392c8e0c5 100644 --- a/bbot/modules/internal/cloudcheck.py +++ b/bbot/modules/internal/cloudcheck.py @@ -1,7 +1,7 @@ -from bbot.modules.base import InterceptModule +from bbot.modules.base import BaseInterceptModule -class CloudCheck(InterceptModule): +class CloudCheck(BaseInterceptModule): watched_events = ["*"] meta = {"description": "Tag events by cloud provider, identify cloud resources like storage buckets"} scope_distance_modifier = 1 diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 5dc4acc83..53b317d9a 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -3,11 +3,11 @@ from bbot.errors import ValidationError from bbot.core.helpers.dns.engine import all_rdtypes -from bbot.modules.base import InterceptModule, BaseModule from bbot.core.helpers.dns.helpers import extract_targets +from bbot.modules.base import BaseInterceptModule, BaseModule -class DNSResolve(InterceptModule): +class DNSResolve(BaseInterceptModule): watched_events = ["*"] _priority = 1 scope_distance_modifier = None diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index b85881d8b..d07f5feb1 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -6,6 +6,7 @@ from pathlib import Path from bbot.errors import ExcavateError import bbot.core.helpers.regexes as bbot_regexes +from bbot.modules.base import BaseInterceptModule from bbot.modules.internal.base import BaseInternalModule from urllib.parse import urlparse, urljoin, parse_qs, urlunparse @@ -279,7 +280,7 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte await self.report(event_data, event, yara_rule_settings, discovery_context) -class excavate(BaseInternalModule): +class excavate(BaseInternalModule, BaseInterceptModule): """ Example (simple) Excavate Rules: @@ -310,6 +311,7 @@ class excavateTestRule(ExcavateRule): "custom_yara_rules": "Include custom Yara rules", } scope_distance_modifier = None + accept_dupes = False _module_threads = 8 @@ -669,8 +671,32 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte class URLExtractor(ExcavateRule): yara_rules = { - "url_full": r'rule url_full { meta: tags = "spider-danger" description = "contains full URL" strings: $url_full = /https?:\/\/([\w\.-]+)(:\d{1,5})?([\/\w\.-]*)/ condition: $url_full }', - "url_attr": r'rule url_attr { meta: tags = "spider-danger" description = "contains tag with src or href attribute" strings: $url_attr = /<[^>]+(href|src)=["\'][^"\']*["\'][^>]*>/ condition: $url_attr }', + "url_full": ( + r""" + rule url_full { + meta: + tags = "spider-danger" + description = "contains full URL" + strings: + $url_full = /https?:\/\/([\w\.-]+)(:\d{1,5})?([\/\w\.-]*)/ + condition: + $url_full + } + """ + ), + "url_attr": ( + r""" + rule url_attr { + meta: + tags = "spider-danger" + description = "contains tag with src or href attribute" + strings: + $url_attr = /<[^>]+(href|src)=["\'][^"\']*["\'][^>]*>/ + condition: + $url_attr + } + """ + ), } full_url_regex = re.compile(r"(https?)://((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)") full_url_regex_strict = re.compile(r"^(https?):\/\/([\w.-]+)(?::\d{1,5})?(\/[\w\/\.-]*)?(\?[^\s]+)?$") @@ -749,6 +775,26 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte for domain_str in yara_results[identifier]: await self.report(domain_str, event, yara_rule_settings, discovery_context, event_type="DNS_NAME") + class LoginPageExtractor(ExcavateRule): + yara_rules = { + "login_page": r""" + rule login_page { + meta: + description = "Detects login pages with username and password fields" + strings: + $username_field = /]+name=["']?(user|login|email)/ nocase + $password_field = /]+name=["']?passw?/ nocase + condition: + $username_field and $password_field + } + """ + } + + async def process(self, yara_results, event, yara_rule_settings, discovery_context): + self.excavate.critical(f"Login page detected: {event.data['url']}") + if yara_results: + event.add_tag("login-page") + def add_yara_rule(self, rule_name, rule_content, rule_instance): rule_instance.name = rule_name self.yara_rules_dict[rule_name] = rule_content @@ -829,7 +875,9 @@ async def setup(self): yara_rules_combined = "\n".join(self.yara_rules_dict.values()) try: self.info(f"Compiling {len(self.yara_rules_dict):,} YARA rules") - self.yara_rules = yara.compile(source=yara_rules_combined) + for rule_name, rule_content in self.yara_rules_dict.items(): + self.info(f"Compiling YARA rule [{rule_name}]") + self.yara_rules = yara.compile(source=yara_rules_combined) except yara.SyntaxError as e: self.debug(yara_rules_combined) return False, f"Yara Rules failed to compile with error: [{e}]" diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 70658e69d..720331625 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -1,10 +1,10 @@ import asyncio from contextlib import suppress -from bbot.modules.base import InterceptModule +from bbot.modules.base import BaseInterceptModule -class ScanIngress(InterceptModule): +class ScanIngress(BaseInterceptModule): """ This is always the first intercept module in the chain, responsible for basic scope checks @@ -169,7 +169,7 @@ def is_incoming_duplicate(self, event, add=False): return False -class ScanEgress(InterceptModule): +class ScanEgress(BaseInterceptModule): """ This is always the last intercept module in the chain, responsible for executing and acting on the `abort_if` and `on_success_callback` functions. diff --git a/bbot/test/test_step_2/module_tests/test_module_httpx.py b/bbot/test/test_step_2/module_tests/test_module_httpx.py index ef9744516..c05b6842d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_httpx.py +++ b/bbot/test/test_step_2/module_tests/test_module_httpx.py @@ -1,8 +1,10 @@ from .base import ModuleTestBase -class TestHTTPX(ModuleTestBase): +class TestHTTPXBase(ModuleTestBase): targets = ["http://127.0.0.1:8888/url", "127.0.0.1:8888"] + module_name = "httpx" + modules_overrides = ["httpx", "excavate"] config_overrides = {"modules": {"httpx": {"store_responses": True}}} # HTML for a page with a login form From 8642bc3edadd2b6c96c25735b8f2973ae3879ba2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 13 Oct 2024 22:50:32 -0400 Subject: [PATCH 210/254] remove old login page helper --- bbot/core/helpers/web/web.py | 47 ------------------------------------ 1 file changed, 47 deletions(-) diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py index a49748008..28c8b5f37 100644 --- a/bbot/core/helpers/web/web.py +++ b/bbot/core/helpers/web/web.py @@ -464,53 +464,6 @@ def beautifulsoup( log.debug(f"Error parsing beautifulsoup: {e}") return False - user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]] - pass_keywords = [re.compile(r, re.I) for r in ["pass"]] - - def is_login_page(self, html): - """ - TODO: convert this into an excavate YARA rule - - Determines if the provided HTML content contains a login page. - - This function parses the HTML to search for forms with input fields typically used for - authentication. If it identifies password fields or a combination of username and password - fields, it returns True. - - Args: - html (str): The HTML content to analyze. - - Returns: - bool: True if the HTML contains a login page, otherwise False. - - Examples: - >>> is_login_page('
') - True - - >>> is_login_page('
') - False - """ - try: - soup = BeautifulSoup(html, "html.parser") - except Exception as e: - log.debug(f"Error parsing html: {e}") - return False - - forms = soup.find_all("form") - - # first, check for obvious password fields - for form in forms: - if form.find_all("input", {"type": "password"}): - return True - - # next, check for forms that have both a user-like and password-like field - for form in forms: - user_fields = sum(bool(form.find_all("input", {"name": r})) for r in self.user_keywords) - pass_fields = sum(bool(form.find_all("input", {"name": r})) for r in self.pass_keywords) - if user_fields and pass_fields: - return True - return False - def response_to_json(self, response): """ Convert web response to JSON object, similar to the output of `httpx -irr -json` From ac983ec788785e6ab2180fbc44f52467e579d795 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 13 Oct 2024 22:55:58 -0400 Subject: [PATCH 211/254] flaked --- bbot/core/helpers/web/web.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py index 28c8b5f37..b05b2d798 100644 --- a/bbot/core/helpers/web/web.py +++ b/bbot/core/helpers/web/web.py @@ -1,4 +1,3 @@ -import re import logging import warnings from pathlib import Path From 3fc0ef762dab9e20ce1a6c6aa1bdfbce0488115f Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 14 Oct 2024 00:17:56 -0400 Subject: [PATCH 212/254] fix excavate tests --- bbot/modules/internal/excavate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index d07f5feb1..ad34b6efb 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -875,9 +875,7 @@ async def setup(self): yara_rules_combined = "\n".join(self.yara_rules_dict.values()) try: self.info(f"Compiling {len(self.yara_rules_dict):,} YARA rules") - for rule_name, rule_content in self.yara_rules_dict.items(): - self.info(f"Compiling YARA rule [{rule_name}]") - self.yara_rules = yara.compile(source=yara_rules_combined) + self.yara_rules = yara.compile(source=yara_rules_combined) except yara.SyntaxError as e: self.debug(yara_rules_combined) return False, f"Yara Rules failed to compile with error: [{e}]" From 1b039f5d7310b2f7313077d27e7a6260a080b02d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 05:01:50 +0000 Subject: [PATCH 213/254] Bump mkdocstrings-python from 1.11.1 to 1.12.0 Bumps [mkdocstrings-python](https://github.com/mkdocstrings/python) from 1.11.1 to 1.12.0. - [Release notes](https://github.com/mkdocstrings/python/releases) - [Changelog](https://github.com/mkdocstrings/python/blob/main/CHANGELOG.md) - [Commits](https://github.com/mkdocstrings/python/compare/1.11.1...1.12.0) --- updated-dependencies: - dependency-name: mkdocstrings-python dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7f3e8eeb1..727c85ab7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1340,13 +1340,13 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] [[package]] name = "mkdocstrings-python" -version = "1.11.1" +version = "1.12.0" description = "A Python handler for mkdocstrings." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "mkdocstrings_python-1.11.1-py3-none-any.whl", hash = "sha256:a21a1c05acef129a618517bb5aae3e33114f569b11588b1e7af3e9d4061a71af"}, - {file = "mkdocstrings_python-1.11.1.tar.gz", hash = "sha256:8824b115c5359304ab0b5378a91f6202324a849e1da907a3485b59208b797322"}, + {file = "mkdocstrings_python-1.12.0-py3-none-any.whl", hash = "sha256:1f48c9ea6d1d6cd1fefc7389f003841e9c65e50016ba40342f340ca901bc60b9"}, + {file = "mkdocstrings_python-1.12.0.tar.gz", hash = "sha256:2121671354fff208fff1278ce9c961aee2b736a1688f70064c4fa76a00241b34"}, ] [package.dependencies] From 8d9984b4b7ab13aa89e6b55754c9041ad5131f80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 05:02:11 +0000 Subject: [PATCH 214/254] Bump mkdocstrings from 0.26.1 to 0.26.2 Bumps [mkdocstrings](https://github.com/mkdocstrings/mkdocstrings) from 0.26.1 to 0.26.2. - [Release notes](https://github.com/mkdocstrings/mkdocstrings/releases) - [Changelog](https://github.com/mkdocstrings/mkdocstrings/blob/main/CHANGELOG.md) - [Commits](https://github.com/mkdocstrings/mkdocstrings/compare/0.26.1...0.26.2) --- updated-dependencies: - dependency-name: mkdocstrings dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7f3e8eeb1..b3a1df2a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1312,13 +1312,13 @@ files = [ [[package]] name = "mkdocstrings" -version = "0.26.1" +version = "0.26.2" description = "Automatic documentation from sources, for MkDocs." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "mkdocstrings-0.26.1-py3-none-any.whl", hash = "sha256:29738bfb72b4608e8e55cc50fb8a54f325dc7ebd2014e4e3881a49892d5983cf"}, - {file = "mkdocstrings-0.26.1.tar.gz", hash = "sha256:bb8b8854d6713d5348ad05b069a09f3b79edbc6a0f33a34c6821141adb03fe33"}, + {file = "mkdocstrings-0.26.2-py3-none-any.whl", hash = "sha256:1248f3228464f3b8d1a15bd91249ce1701fe3104ac517a5f167a0e01ca850ba5"}, + {file = "mkdocstrings-0.26.2.tar.gz", hash = "sha256:34a8b50f1e6cfd29546c6c09fbe02154adfb0b361bb758834bf56aa284ba876e"}, ] [package.dependencies] From 0390043d74837357dcdfdc039ff39977261804c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 05:02:41 +0000 Subject: [PATCH 215/254] Bump black from 24.8.0 to 24.10.0 Bumps [black](https://github.com/psf/black) from 24.8.0 to 24.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.8.0...24.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7f3e8eeb1..15f96983d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -131,33 +131,33 @@ lxml = ["lxml"] [[package]] name = "black" -version = "24.8.0" +version = "24.10.0" description = "The uncompromising code formatter." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "black-24.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6"}, - {file = "black-24.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb"}, - {file = "black-24.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42"}, - {file = "black-24.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a"}, - {file = "black-24.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1"}, - {file = "black-24.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af"}, - {file = "black-24.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4"}, - {file = "black-24.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af"}, - {file = "black-24.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368"}, - {file = "black-24.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed"}, - {file = "black-24.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018"}, - {file = "black-24.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2"}, - {file = "black-24.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd"}, - {file = "black-24.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2"}, - {file = "black-24.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e"}, - {file = "black-24.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920"}, - {file = "black-24.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c"}, - {file = "black-24.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e"}, - {file = "black-24.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47"}, - {file = "black-24.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb"}, - {file = "black-24.8.0-py3-none-any.whl", hash = "sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed"}, - {file = "black-24.8.0.tar.gz", hash = "sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f"}, + {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, + {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, + {file = "black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f"}, + {file = "black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e"}, + {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"}, + {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"}, + {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"}, + {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"}, + {file = "black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3"}, + {file = "black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65"}, + {file = "black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f"}, + {file = "black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8"}, + {file = "black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981"}, + {file = "black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b"}, + {file = "black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2"}, + {file = "black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b"}, + {file = "black-24.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:17374989640fbca88b6a448129cd1745c5eb8d9547b464f281b251dd00155ccd"}, + {file = "black-24.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:63f626344343083322233f175aaf372d326de8436f5928c042639a4afbbf1d3f"}, + {file = "black-24.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfa1d0cb6200857f1923b602f978386a3a2758a65b52e0950299ea014be6800"}, + {file = "black-24.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cd9c95431d94adc56600710f8813ee27eea544dd118d45896bb734e9d7a0dc7"}, + {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"}, + {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"}, ] [package.dependencies] @@ -171,7 +171,7 @@ typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +d = ["aiohttp (>=3.10)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] From 0754577c9673e3c21225b022afbd9cfe5350b398 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 05:03:05 +0000 Subject: [PATCH 216/254] Bump pre-commit from 4.0.0 to 4.0.1 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v4.0.0...v4.0.1) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7f3e8eeb1..3f9bf37cc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1621,13 +1621,13 @@ plugin = ["poetry (>=1.2.0,<2.0.0)"] [[package]] name = "pre-commit" -version = "4.0.0" +version = "4.0.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" files = [ - {file = "pre_commit-4.0.0-py2.py3-none-any.whl", hash = "sha256:0ca2341cf94ac1865350970951e54b1a50521e57b7b500403307aed4315a1234"}, - {file = "pre_commit-4.0.0.tar.gz", hash = "sha256:5d9807162cc5537940f94f266cbe2d716a75cfad0d78a317a92cac16287cfed6"}, + {file = "pre_commit-4.0.1-py2.py3-none-any.whl", hash = "sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878"}, + {file = "pre_commit-4.0.1.tar.gz", hash = "sha256:80905ac375958c0444c65e9cebebd948b3cdb518f335a091a670a89d652139d2"}, ] [package.dependencies] From ff6d47ee6bb4ed4bd6029a9dfcaacc496d595a61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 05:03:23 +0000 Subject: [PATCH 217/254] Bump cloudcheck from 5.0.1.571 to 5.0.1.595 Bumps [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck) from 5.0.1.571 to 5.0.1.595. - [Commits](https://github.com/blacklanternsecurity/cloudcheck/commits) --- updated-dependencies: - dependency-name: cloudcheck dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7f3e8eeb1..09b827daf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -402,13 +402,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cloudcheck" -version = "5.0.1.571" +version = "5.0.1.595" description = "Check whether an IP address belongs to a cloud provider" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "cloudcheck-5.0.1.571-py3-none-any.whl", hash = "sha256:5a3a06f74fa7ab6828d27182f42b199e89966b615618f87c87c05f3f347fa7dd"}, - {file = "cloudcheck-5.0.1.571.tar.gz", hash = "sha256:342f0b84bd7f8dddbfcd08277f8bf23fd0c868232a97bc20be7c7af196330b02"}, + {file = "cloudcheck-5.0.1.595-py3-none-any.whl", hash = "sha256:68acec63b09400fa0409ae7f3ffa817cbc891bf8a2ac63f9610a3b049a4bf57d"}, + {file = "cloudcheck-5.0.1.595.tar.gz", hash = "sha256:38456074332ed2ba928e7073e3928a5223a6005a64124b4b342d8b9599ca10e0"}, ] [package.dependencies] From 2ba4d684781e5563b2b19ba57f09d20c7d24cb2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 05:03:43 +0000 Subject: [PATCH 218/254] Bump mkdocs-material from 9.5.39 to 9.5.40 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.39 to 9.5.40. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.39...9.5.40) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7f3e8eeb1..a0b5a5bef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1272,13 +1272,13 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.39" +version = "9.5.40" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.39-py3-none-any.whl", hash = "sha256:0f2f68c8db89523cb4a59705cd01b4acd62b2f71218ccb67e1e004e560410d2b"}, - {file = "mkdocs_material-9.5.39.tar.gz", hash = "sha256:25faa06142afa38549d2b781d475a86fb61de93189f532b88e69bf11e5e5c3be"}, + {file = "mkdocs_material-9.5.40-py3-none-any.whl", hash = "sha256:8e7a16ada34e79a7b6459ff2602584222f522c738b6a023d1bea853d5049da6f"}, + {file = "mkdocs_material-9.5.40.tar.gz", hash = "sha256:b69d70e667ec51fc41f65e006a3184dd00d95b2439d982cb1586e4c018943156"}, ] [package.dependencies] From 6211cd38ed301376730075f152e813e2265f734c Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 14 Oct 2024 17:25:05 +0100 Subject: [PATCH 219/254] Change the validate function to use the async helpers --- bbot/modules/google_playstore.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/bbot/modules/google_playstore.py b/bbot/modules/google_playstore.py index 861febb2f..5fa1027d0 100644 --- a/bbot/modules/google_playstore.py +++ b/bbot/modules/google_playstore.py @@ -70,27 +70,24 @@ async def query(self, query): return app_links async def validate_apk(self, apk_name): + """ + Check the app details page the "App support" section will include URLs or Emails to the app developer + """ in_scope = False url = f"{self.base_url}/store/apps/details?id={apk_name}" r = await self.helpers.request(url) if r is None: return in_scope status_code = getattr(r, "status_code", 0) - try: - html = self.helpers.beautifulsoup(r.content, "html.parser") - except Exception as e: - self.warning(f"Failed to parse html response from {r.url} (HTTP status: {status_code}): {e}") - return in_scope - # The developer meta tag usually contains the developer's URL - developer_meta = html.find("meta", attrs={"name": "appstore:developer_url"}) - developer_url = developer_meta["content"] if developer_meta else None - if self.scan.in_scope(developer_url): - in_scope = True - # If the developers URL is left blank then a support email is usually provided - links = html.find_all("a", href=True) - emails = [a["href"].split("mailto:")[1] for a in links if "mailto:" in a["href"]] - for email in emails: - if self.scan.in_scope(email): + if status_code == 200: + html = r.text + in_scope_hosts = await self.scan.extract_in_scope_hostnames(html) + if in_scope_hosts: in_scope = True - break + for email in await self.helpers.re.extract_emails(html): + if self.scan.in_scope(email): + in_scope = True + break + else: + self.warning(f"Failed to fetch {url} (HTTP status: {status_code})") return in_scope From 8d0add404ef34c3b1d6b3a6744917585972e4f0a Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Mon, 14 Oct 2024 19:02:21 +0100 Subject: [PATCH 220/254] added new module --- bbot/modules/apkpure.py | 53 +++++++++++++++ .../module_tests/test_module_apkpure.py | 68 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 bbot/modules/apkpure.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_apkpure.py diff --git a/bbot/modules/apkpure.py b/bbot/modules/apkpure.py new file mode 100644 index 000000000..210dfef0e --- /dev/null +++ b/bbot/modules/apkpure.py @@ -0,0 +1,53 @@ +from pathlib import Path +from bbot.modules.base import BaseModule + + +class apkpure(BaseModule): + watched_events = ["MOBILE_APP"] + produced_events = ["FILESYSTEM"] + flags = ["passive", "safe", "code-enum"] + meta = { + "description": "Download android applications from apkpure.com", + "created_date": "2024-10-11", + "author": "@domwhewell-sage", + } + options = {"output_folder": ""} + options_desc = {"output_folder": "Folder to download apk's to"} + + async def setup(self): + output_folder = self.config.get("output_folder") + if output_folder: + self.output_dir = Path(output_folder) / "apk_files" + else: + self.output_dir = self.scan.home / "apk_files" + self.helpers.mkdir(self.output_dir) + return await super().setup() + + async def filter_event(self, event): + if event.type == "MOBILE_APP": + if "android" not in event.tags: + return False, "event is not an android app" + return True + + async def handle_event(self, event): + app_id = event.data.get("id", "") + path = await self.download_apk(app_id) + if path: + await self.emit_event( + {"path": str(path)}, + "FILESYSTEM", + tags=["apk", "file"], + parent=event, + context=f'{{module}} downloaded the apk "{app_id}" to: {path}', + ) + + async def download_apk(self, app_id): + path = None + url = f"https://d.apkpure.com/b/XAPK/{app_id}?version=latest" + self.helpers.mkdir(self.output_dir / app_id) + file_destination = self.output_dir / app_id / f"{app_id}.xapk" + result = await self.helpers.download(url, warn=False, filename=file_destination) + if result: + self.info(f'Downloaded "{app_id}" from "{url}", saved to {file_destination}') + path = file_destination + return path diff --git a/bbot/test/test_step_2/module_tests/test_module_apkpure.py b/bbot/test/test_step_2/module_tests/test_module_apkpure.py new file mode 100644 index 000000000..59a32a66f --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_apkpure.py @@ -0,0 +1,68 @@ +from pathlib import Path +from .base import ModuleTestBase + + +class TestAPKPure(ModuleTestBase): + modules_overrides = ["apkpure", "google_playstore", "speculate"] + + async def setup_after_prep(self, module_test): + await module_test.mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.99"]}}) + module_test.httpx_mock.add_response( + url="https://play.google.com/store/search?q=blacklanternsecurity&c=apps", + text=""" + + + "blacklanternsecurity" - Android Apps on Google Play + + + + + """, + ) + module_test.httpx_mock.add_response( + url="https://play.google.com/store/apps/details?id=com.bbot.test", + text=""" + + + BBOT + + + + + + + """, + ) + module_test.httpx_mock.add_response( + url="https://d.apkpure.com/b/XAPK/com.bbot.test?version=latest", + text="apk", + ) + + def check(self, module_test, events): + assert len(events) == 6 + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com" and e.scope_distance == 0 + ] + ), "Failed to emit target DNS_NAME" + assert 1 == len( + [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 0] + ), "Failed to find ORG_STUB" + assert 1 == len( + [ + e + for e in events + if e.type == "MOBILE_APP" + and "android" in e.tags + and e.data["id"] == "com.bbot.test" + and e.data["url"] == "https://play.google.com/store/apps/details?id=com.bbot.test" + ] + ), "Failed to find bbot android app" + filesystem_event = [ + e for e in events if e.type == "FILESYSTEM" and "com.bbot.test.xapk" in e.data["path"] and "apk" in e.tags + ] + assert 1 == len(filesystem_event), "Failed to download apk" + file = Path(filesystem_event[0].data["path"]) + assert file.is_file(), "Destination xapk doesn't exist" From c59ab925c5bb87daa10837ae5ceaf9de666ba981 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 15 Oct 2024 00:28:34 -0400 Subject: [PATCH 221/254] fix excavate tests --- bbot/core/event/base.py | 5 ----- bbot/modules/internetdb.py | 4 +++- bbot/scanner/manager.py | 8 -------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index d5a9552e2..fddd0c424 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -440,11 +440,6 @@ def always_emit(self): no_host_information = not bool(self.host) return self._always_emit or always_emit_tags or no_host_information - @property - def quick_emit(self): - no_host_information = not bool(self.host) - return self._quick_emit or no_host_information - @property def id(self): """ diff --git a/bbot/modules/internetdb.py b/bbot/modules/internetdb.py index 55b613f16..52c5040b2 100644 --- a/bbot/modules/internetdb.py +++ b/bbot/modules/internetdb.py @@ -48,6 +48,9 @@ class internetdb(BaseModule): "show_open_ports": "Display OPEN_TCP_PORT events in output, even if they didn't lead to an interesting discovery" } + # we get lots of 404s, that's normal + _api_failure_abort_threshold = 9999999999 + _qsize = 500 base_url = "https://internetdb.shodan.io" @@ -113,7 +116,6 @@ async def _parse_response(self, data: dict, event, ip): "OPEN_TCP_PORT", parent=event, internal=(not self.show_open_ports), - quick=True, context=f'{{module}} queried Shodan\'s InternetDB API for "{query_host}" and found {{event.type}}: {{event.data}}', ) vulns = data.get("vulns", []) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 720331625..3cb1f5fdf 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -115,14 +115,6 @@ async def handle_event(self, event, **kwargs): # nerf event's priority if it's not in scope event.module_priority += event.scope_distance - async def forward_event(self, event, kwargs): - # if a module qualifies for "quick-emit", we skip all the intermediate modules like dns and cloud - # and forward it straight to the egress module - if event.quick_emit: - await self.scan.egress_module.queue_event(event, kwargs) - else: - await super().forward_event(event, kwargs) - @property def non_intercept_modules(self): if self._non_intercept_modules is None: From d63ca27bb7dd69773302a31f72900c153b6d40fa Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 15 Oct 2024 00:38:02 -0400 Subject: [PATCH 222/254] replace beautifulsoup with regex helper --- bbot/modules/google_playstore.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bbot/modules/google_playstore.py b/bbot/modules/google_playstore.py index 5fa1027d0..171bcb9b1 100644 --- a/bbot/modules/google_playstore.py +++ b/bbot/modules/google_playstore.py @@ -13,6 +13,10 @@ class google_playstore(BaseModule): base_url = "https://play.google.com" + async def setup(self): + self.app_link_regex = self.helpers.re.compile(r"/store/apps/details\?id=([a-zA-Z0-9._-]+)") + return True + async def filter_event(self, event): if event.type == "CODE_REPOSITORY": if "android" not in event.tags: @@ -61,12 +65,12 @@ async def query(self, query): return app_links status_code = getattr(r, "status_code", 0) try: - html = self.helpers.beautifulsoup(r.content, "html.parser") + html_content = r.content.decode("utf-8") + # Use regex to find all app links + app_links = await self.helpers.re.findall(self.app_link_regex, html_content) except Exception as e: self.warning(f"Failed to parse html response from {r.url} (HTTP status: {status_code}): {e}") return app_links - links = html.find_all("a", href=True) - app_links = [a["href"].split("id=")[1].split("&")[0] for a in links if "/store/apps/details?id=" in a["href"]] return app_links async def validate_apk(self, apk_name): @@ -84,10 +88,6 @@ async def validate_apk(self, apk_name): in_scope_hosts = await self.scan.extract_in_scope_hostnames(html) if in_scope_hosts: in_scope = True - for email in await self.helpers.re.extract_emails(html): - if self.scan.in_scope(email): - in_scope = True - break else: self.warning(f"Failed to fetch {url} (HTTP status: {status_code})") return in_scope From f614557f60d182a4bc803009a87c0fcaac9589cf Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 15 Oct 2024 08:41:27 -0400 Subject: [PATCH 223/254] fixing in-scope hostname extraction --- bbot/scanner/scanner.py | 2 +- bbot/test/test_step_1/test_regexes.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 85b9a0073..7b2fec21b 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -1064,7 +1064,7 @@ def dns_regexes_yara(self): Returns a list of DNS hostname regexes formatted specifically for compatibility with YARA rules. """ if self._dns_regexes_yara is None: - self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)+") + self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)*") return self._dns_regexes_yara @property diff --git a/bbot/test/test_step_1/test_regexes.py b/bbot/test/test_step_1/test_regexes.py index 26afa42f6..77ccc987e 100644 --- a/bbot/test/test_step_1/test_regexes.py +++ b/bbot/test/test_step_1/test_regexes.py @@ -376,18 +376,25 @@ async def test_regex_helper(): # test yara hostname extractor helper scan = Scanner("evilcorp.com", "www.evilcorp.net", "evilcorp.co.uk") host_blob = """ + https://evilcorp.com/ https://asdf.evilcorp.com/ https://asdf.www.evilcorp.net/ https://asdf.www.evilcorp.co.uk/ https://asdf.www.evilcorp.com/ https://asdf.www.evilcorp.com/ + https://test.api.www.evilcorp.net/ """ extracted = await scan.extract_in_scope_hostnames(host_blob) assert extracted == { - "asdf.www.evilcorp.net", + "evilcorp.co.uk", + "evilcorp.com", + "www.evilcorp.com", "asdf.evilcorp.com", "asdf.www.evilcorp.com", - "www.evilcorp.com", + "www.evilcorp.net", + "api.www.evilcorp.net", + "asdf.www.evilcorp.net", + "test.api.www.evilcorp.net", "asdf.www.evilcorp.co.uk", "www.evilcorp.co.uk", } From 5ab09d10df3e22f6698ef1397611b3b34d6d3b90 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 15 Oct 2024 09:37:42 -0400 Subject: [PATCH 224/254] fix stats tests --- bbot/test/test_step_1/test_modules_basic.py | 23 +++++++++++---------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 9fbaed085..e2f55f8dc 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -380,13 +380,14 @@ async def handle_event(self, event): scan.modules["dummy"] = dummy(scan) events = [e async for e in scan.async_start()] - assert len(events) == 9 + assert len(events) == 10 + for e in events: + log.critical(e) assert 2 == len([e for e in events if e.type == "SCAN"]) - assert 3 == len([e for e in events if e.type == "DNS_NAME"]) + assert 4 == len([e for e in events if e.type == "DNS_NAME"]) # one from target and one from speculate assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com"]) - # the reason we don't have a DNS_NAME for www.evilcorp.com is because FINDING.quick_emit = True - assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com"]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.evilcorp.com"]) assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "evilcorp"]) assert 1 == len([e for e in events if e.type == "FINDING"]) @@ -394,7 +395,7 @@ async def handle_event(self, event): assert scan.stats.events_emitted_by_type == { "SCAN": 1, - "DNS_NAME": 3, + "DNS_NAME": 4, "URL": 1, "ORG_STUB": 1, "URL_UNVERIFIED": 1, @@ -414,7 +415,7 @@ async def handle_event(self, event): assert dummy_stats.produced == {"FINDING": 1, "URL": 1} assert dummy_stats.produced_total == 2 assert dummy_stats.consumed == { - "DNS_NAME": 2, + "DNS_NAME": 3, "FINDING": 1, "OPEN_TCP_PORT": 1, "ORG_STUB": 1, @@ -422,26 +423,26 @@ async def handle_event(self, event): "URL": 1, "URL_UNVERIFIED": 1, } - assert dummy_stats.consumed_total == 8 + assert dummy_stats.consumed_total == 9 python_stats = scan.stats.module_stats["python"] assert python_stats.produced == {} assert python_stats.produced_total == 0 assert python_stats.consumed == { - "DNS_NAME": 3, + "DNS_NAME": 4, "FINDING": 1, "ORG_STUB": 1, "SCAN": 1, "URL": 1, "URL_UNVERIFIED": 1, } - assert python_stats.consumed_total == 8 + assert python_stats.consumed_total == 9 speculate_stats = scan.stats.module_stats["speculate"] assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1} assert speculate_stats.produced_total == 3 - assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 2, "URL_UNVERIFIED": 1, "IP_ADDRESS": 2} - assert speculate_stats.consumed_total == 6 + assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 3, "URL_UNVERIFIED": 1, "IP_ADDRESS": 3} + assert speculate_stats.consumed_total == 8 @pytest.mark.asyncio From cb21aebfbd3077ca230113a548fbcc0d8446696e Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Tue, 15 Oct 2024 18:10:33 +0100 Subject: [PATCH 225/254] Added a mock apk file to the test --- bbot/test/bbot_fixtures.py | 7 +++++++ bbot/test/owasp_mastg.apk | Bin 0 -> 66651 bytes .../module_tests/test_module_apkpure.py | 5 +++-- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 bbot/test/owasp_mastg.apk diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index abad144d1..0b2a0ec57 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -42,6 +42,13 @@ def tempwordlist(content): return filename +def tempapkfile(): + current_dir = Path(__file__).parent + with open(current_dir / "owasp_mastg.apk", "rb") as f: + apk_file = f.read() + return apk_file + + @pytest.fixture def clean_default_config(monkeypatch): clean_config = OmegaConf.merge( diff --git a/bbot/test/owasp_mastg.apk b/bbot/test/owasp_mastg.apk new file mode 100644 index 0000000000000000000000000000000000000000..9a4f638f1c4a5296fb4eace04b328aecce659b79 GIT binary patch literal 66651 zcmb5VV~{98w=LMVZQHhO+qT`OZQHhO+wRjgPTRJozx!^ynLGF~8L=y)vZ690vewSM zbFC}|XZn+gZ;*~$^+ilGDX6Eh(!~#m7FkGi^{iZY<70h-Bxc|5u(>%n6htdCG z-vzx&ev*A7paO4^n+>b~<5qm~$2H3Hkk3Zy3r8*X2xG@UVv0iCXB{il9{|wx?w1%N zt~F6}eFmP*D+)92CSe~9mNDxHwG7}qw3q*yq?SL4lr=+iESb~nRADz#V>HLV^FN@o zm5X;68N(CW+Apv4=6ZoVexOYFhuneHfk+%}$!q|xL&;chO=s%;1`C}@n7aoMm%;6D z#qX!;HW5O5|h&Y6&{y<{#}z!|Yh*gaR+v;RruzTg?f9BJwCE)#k}*u$%55 z(-UHM%F?;q72b>{AKoYIN2I)p%Cb~>ma6K&sii{{q82U==U%`bvdj}YLUzfGX^{33 zVq=(nM81Y=Oy`tS1W3jQla31eggIcp^_6w|@$66Pu(Xwb9#MJryXT?3g*Lo?hkb)>UKG}n(JD9FfqfRnD%lts zr;gAqbAy;+%;i)ksF!NV&}dtwk8i!1a?>r&&#t+>OD(q6ORaA=_4b@vS!&ewMosqC zS(`fbY@-n;7c+Cs7jiR`z8!j-`&rVDSZa0qrzu}fz1?Bi^sAI7Q|8BM*OtEcGl?G` z@Cv^mMfru}{{kt%Gts2PFKEd@0RRwxTV+L61ZgGZ#OQ@Zl~m}Ilm(eBj8x4ImaHBu zK4z!JXPfgQREK}fKO*68{H}jm40FLu5MxS4NJK))fIoIYyrUqpbMdd03nC|70hy7B zk&&eivH`lsqm^Tc+2yBN2-^Y6u^lU*u^E!_;|mi410sa!rY|zKwUe|^kTR~u&-sIs zjFL{D<6mRo7>q+VgO8t}nccm>Ik&gn)J{McpeupbRoQJ1IFwDL=p*4tv7V z!q7xd$H2h)Ge~8x4r7e720K|=dmDCIeMnl0duFQNx*w@mmJ6M_m3Uvl&l%vCuZ3mmdW_P1$lJM2 z>2n_B^X4df{F>Yy-zYOVBLS-sC!IIVdQ7eO&S<)6rN8&f zNXNv)K&QqU#~ud?Qb>&xjTSvz*x<~<%F4}BV+A26`S~35tvYri)`~O)3(?(*#@hTW zLZ|PF zuqtSw5qLP)lQw?qlV3=ax$#{kIOwt%c+7jK1#8r6c8|)jK|(AWL9c&jjq188<8|Kg z)viYK2|w8^#qNVs^I55V>#%*tU7a6&Ggs5rhT64*)cowp|B6_RfEvk@PepHe_Vn#$ z(ylRIZ9azI^($G~TJgJV(tg_=2@x}QyKDDy9M)6W#dR@OOdXY(u>20wmz&XCZfJCO z53f^;>HC2U3(E1nx$k{AuGV@>meqS44n(KY-!|fZp7pjge#u%Sav$Jgu0xIbA$9zm zm3?cosxU-0#%CaO1#~VDWRgu{Bw8ghcUf|oSr{JA_pq|J^WHRl4k^p^nDK3Dy~r>* zKN&fjh?yb8@|B|>`l3Z|)IsC?=2RkkEd3i$O|r>#q%p@Dmp1B~%=0{b?LtKFr7!<> zbbj4+^RWV2kH_VBYHPAhu2+TJ#L;?$qdz{>vqeQ%w%lba8X~+7mZllXOx>5QobB_v z-i_Sxvd)z}Q)=tsx7c+vDzf;KaT zzp>Hzcj-a+e=Ix7Vlj(TafJ*x{dZjvKS5*L7$S%$GRFWz!e{>o#)av#CZ}Ri>UluE z-*aTlnz|m=JS;QW-hAJ=m&7E&sxn2Au@jJv)~#5wWWs{Pkp`%sC@Z{pF?^7l;twf4 z`Eb^(6O7tHNsCCRs3-j~BlX#;5UuH0wjc`^fL$~-4tjQ@vWkX0xp&odw`>?3);IO; zOrH%oZY$bINHhH#^>+|Mst}oRV8FG*i(UW| zUBd$@Yz(!E|Gw3U@FSH}S)jbU?<_P_C;(I4mWNS%eru`&A80|6Jh_09GfugIIN>4R z8K?@`piL(R?lP;j(z}-3#pIYgUp?Y?b@rs)G=g36j#&C<0evrq|I>e;S%s{;JKL-f zlXXZ#AF}6zi;97^#7Tp&T+agKTUtLG=qy4en(CeWZgfyNRT#g8q7hf(nAD)H7b{3u zGD4?{hb?9&pWS?^bmR=VOFiIwjCF3CdFnBCxQ^56NLg29WM_-_M5~ra5})R+F5UUs zv9!ax#3bkTfO#PCbvJQRS9Ze43r+lX2-ySdkz(M@d#QY((W(!#ykk!&J_TiMk9x9Z zb_iFh>FFZ3LHX3!(0t?J35P|W-Hra^}>{w&TvsXHW9oSpO&7L5eTbTJu@MuzM zw$gu-2#2`%iauqH)a{KfF}Y`)f>N^K|FAgNx?(5{!@H3N5bnf1lwveG84`1{3Cy-U zb=2|>{XaozPPa$E_X|qaUr_!hLjLbqC@Ux@DJH6{LMJO0lO!EiD1hL9L-q}R1J)IS zIAQ|NgOa^}^!m(XnGaf8FcK8Kd{ zPP}DhSj%R%9{q_*mrYXr&daU=k;$k>`xxfrYekQR~skwtC8|l zYT@^56dbckYCDcd`&;yqZTu~UkO87ZvF@218N4WI^8j%-YoYJLI>1e;>0-GCy=%U-QqH2_~d$ty=DaEsB3X? z&8^OhSoI#TQpF@rQWTD2Tn!*-N2MgD^pUD3w-a^1uqDk(zi|go{`tF`S$)V0I$O7nZcx5$xQg?&LP1bV~kgyV07Rs{Gk z2{%>*9yL+q#v)ewk@vdx(D$&ySCL%b4d@)W+rer4 zqxLTL$oEhkV7+0OWJO+r=!0&8@B>Z;<_7EsF7|A12`$3ZoeiL5g1iQR4d4z10Z{_L z)%u6_7;iaV(LA8M;Xe^RF}eZtfartN2e9{WZ}DESUm?b4L?Gwkm<(WK0v`H7Kox-U z0MQ51`VaQ9_Q3bb_SE-OZ@Jo`c0m0Ae*l7C%RwEm8}cOsY7DT~;N1Lk1VD2LU_1Ni z>azkhHi(Tb*y_Xng>%nJH6#@Ys-rsf?fZ(q&uOT(%jH>i5#PHpoAr{~T$MSZ&VS~>;XRL-6|{mpgeNbNAFb?poJR^PaZjC+ zGnlvDc;F@um{Gv41))aHbeKB=O!MXukV(|Ap32KABS(GY@O$g&!T!Nh79#Hom%cM+a>O zBy+bH&fIC-f0YuLg?47XHU~bc))WIiI4RsCowU}HMq_6kG^M%H0)7_$;VjhGjhjN( z+~MTkf9s}OiXB2p>WE)HF!IU7wt*bswpq+YJm<7owN2@*yyrr_)aR+9T-ipM@%dwid0xqa%8fhY9JYrTp?r`hH_|%!47pn{L(|2m}c!g@Klv^ zy7~1{@IR$OSX$!i0+u==45mW`OCDi!8rkjX(@l3*ne{p>QYLd%C(}DyPPwpb|8}#^wV|>0Xe^bkWMbxmcD)g-`!ry{uE^T38GCpV8eppWQ(Wb>7JFqbb9HOjd;Pe5Aegrz zm8aV3z<>aKgWvrj_mRFoM|*rSy1msCHKL>c^DpoKE-#%kT^o-Lx|84A=E>&gj`9Gx z7%t1_D=ah=9?N;OpRnj=UST($bBo(V2ji00!=j%-f=ei2GXv6H!NSET)96*HMrnc; zw*c*QL&Qyn+lPtU=d{@_vo)#g(4wJyBiBmH`N&%@k6I&##8h%tPKo6rQ_XFM2~8ri zc-|z{=g5iVNMcqt5GTY*;WnkHHTyc9bd{}RQ5<$xul1`1@q9~1zh3IQy#{^jv$mm8 zm!W0>!nQ*-?VN1L)qCo$g(j)$8{%nu9t9>-q`*axy;@gUuGIh=niD(LeN8Cm$95dz zsn$nmibJ_L7bvvXX=TiHxAKiN9QCRRA#kTZz6hyOlH%Z4zFP4DYVfZ zy_s>lDHKjke!9we8XOy$esd*~Evn9Hta{CC2dXr8KlW@gX2t%yE^PwX$GX5yHBnc`;8UcedXzK9y#WiM@WNm z+?F+}G8)U#3tE5OTdH-l;R_9vThyCjY&V#sQsWdCDH;;=^t4xS_2;xjSH)0HGkqV< zJF?V`6M@-P(r|&VH4PBw%W@WM%bYSU&p&=TU*hI_e|kP3Dt*_YUG|nv$%_ra}!p^~foii^j?48FZ|+ov<07-mW|LP`c;j4| zn_bQDC_#U|yGyX_DuO9h;yyTpfm22si?C@J(h)IYYyZSY;kb*=3{7{RcU87TPDo~w zdZJaBxiN`P(o6~6(=5!v}1JMfM} zxH4e2)-6(?A=^|*W1&o9!0G;*hcdP|T+S{YpaM;*KBe6J&A8qAU#to|>l7Is!;dmB zH5g?arVOLKaf{X|b8DPlB>PRR!dg%U-$Z+WsL#{`JdK7m-txQMo+T9hMM$G)f@zhB zzDCitI36axFGr6BH~3>=4&Q>ihaZ+Ko?8ZAqTcn?qZt_3VjeLkVNEH^Cje*F{2-hj z*LW_Rn>fFfK_mCO4zbt$8nREFVq|Y}d?7x`tpmwKT!k0Cp308Z2D#7tFqKeT#fRdZ z$i4p~ix)YmIiH`RCne>CdeYaV^GxHT3ba=zMzo$)N@3dO+Jh*)Rj5{=c`SGEw+giF zJCCz&hoVGP$HlR!O-fR$jn*qMA6F%l&4lALfH3EJKfLw#|O0;7EoWa*S03C6dfNI zf36?i9?ZGnC=2ir5sN;OnG0zfVdprPYrwDl!S@I~fS&uEZ^3B#896}n2GAYAW(U+A zAh~iH6NM~mO)DrMOysVfqfa_ijHMPK`Ki#~uau5E^X*o9(S%~5WfhA@z?GpYT(-M( zUf{rpGAl&2Lcdm?af6>x6o2q>iz%{!LrqV+T3N|3s-YD1OlUX)}I zZWJ^KC8y2nN;C$;tGcNW|9ExH|M|_jj*En3-n_@$E~6(cN6o=CNslu=IAVXdaJ%@1 zU*t3e;^pYWYwWkUpCD;_pvE@%-XM7bm<(do@j5n&9S}`;MGfvPpKo$iRbW!|e zi(gF)m|^8FSbtW{U}`5`>mK-8+&4X+zH`N|P+4Al4~|c+ZUjwU2r61=mAAFNaO7TA zvHrXaUBPx8?|lB>&{C}uxbpOilJQ@}p#3*mY)$Q4>Hl;2-#BrV?~)o8K;dPNlZ_0~ z6`U7Z8?}m(p*CvLc*7*blv$HCn}`li@lDJ#w6rGmjmRB_#75^C+Mmt#x;7Rn_COYJ z2vGN@hqq$qARwC?N6ko3FD7$0L0BS1gI-MO?6S6ZF1wL^3FdHX*|eNKX=RGT?xNJO z)gqDHNzE~w_@m3&)~7a!mSX3W^(La0E_U=$bKb04#2IN-h8-%_2Hzi5%yDw*9_}4E zj{b%iE|3~D`*bHO;``$Nek5{}aO*qCT=>E<>hLe)H&asieh!=~6YkpM%zZ3rr5Vfc z^KKeI$7@dNl|Oyq&Bes~GTu`Ee%t*o-2m!$O8!?#6z~*|LjnN!j}ZZY|Cd|YS~}Po zI?!5}I9SrUvCvx@>)RN*+8JAzI?*}UnX|3~1N{16go2znEELv%dV-ad5K;QACH`X& zzbYAk>Q$3C005Gjq==x3$HrB*S3Rz3%BWw}+`*35byl|>((mS0I4}SM2!g3!COO|< ziWqa@|ra6 zqxWRn`umix%-#Kl?_^6L<)^p`4$?>NX6NgwW%YffO${6lKP+x&Z2w@+UE(p=*C!)X zzMGj{>BsE+XO)}sW^2#hymH`B!pBwHLg}nHt0%@wqc?ec{=U`UWAmc)*|~M`Y$ck; zO69LJ7{L3i#tVfc?ZHFSy{Sgb3-vT$2qx~U~JLHLN655T!Vu-{2e zV))4WZpH~#W-ll~j14q5!adw8L`2+Kv4HN4F)%q)Fnp{SK)AN@6}9A+UXAvA0mtlk zAsKMxKxJ7|{6*MzQ9P%#!NA@4<%oqN(z9JiLCtBvs-X+LR%Wo(NS9o8f1vJ9fhD4U zI=poDt&I0P3{i*}K!v7oCv4|2K~r2nz!8HzCew??Y35Y`NyYm)bbsQDmmmF!;W+?T zEBdpY=BlQ^}$cn@vRa(g!SwfAKSx?$Szcj?HIT|m=3<17hIqCGyh)Vgx-*Pix_ZyPWTc_ z_~L@A)9abjTv~@c*k*<^&Ge*uLB|95Ix+(!F7t;_NBrU#VhtaMu>cotmpMT*TwrK* zBIq4p;Hi9TxnailRn~i#|La}IMdJZ`Wz17d|A2#d_Zp#NTH#}w z&{#b9LmBV|PY@a&K+ZFwn@lN0-dC_rb847x!0j$n+-=}DMSHJIIW6s%ng-@&KVG>c z$yg%kS$|`>fB@8S#GAk!&|Xy$Ume>7D?x`e4lPgIs3r{F^l04S{&up|^jqIzQstwednCd|-22o~osJV+ej#_dQ{${f#9jsrVyH~++~ABclqb(+vEW#d{^J;_ z(9Xld2@VQ6T_G31VcCxTot#(>sAIe+8XmliCX6$Fq>aw}9%#*aAWaXRY99FQHsl-* z0})9}DJCo%#s+_~0dph#&NoAs)u_{6T!uTe&kVc&Ef0Q=tH?oX6iM|1F04A%uNWwM%>!7fHc z#JK!l)i_2oQ1X$%-=dx(Qy>>PC}Nbjt`pF{Qoks)&pLpg^o>APi_%skd{Dr`VtJ$& zAoX$pM5Aa+iddV4a2IMuc<4bPCehINSE3AkI~B#6wO{f>K>Sd%fD!?*kMXty9&DC5 zc&Rj!q$DA9u%=*Qml|7uMHT@?1w|E}g3Q$_`rrTyzL_VzmpQ(%(k4vyz{06)6+Nd6 z!LSOH1dI8|EF(g@&{RmKu`(JpY#@+aabltJ9A>s+?N0z5OM2fsLH>$>vnzdH>+{;* zt%q=~&FK3=Yj!84Gk&|SD$t3x?IUQ4zcs7(M;6%;kO)%T7U4GT_98)(8cuJh{ZuuzUhyD1;^NfQ`SSqEuJ zvYum!vPd$9wGnFP)C}bE2ZdHHf$If7s4mPs-`+;(m`)O+B{a#c>NMB3c=nZ>WK~^1 zV-mToXxYhMGMXB*S@7S9inj9M!mpn#ak21(bp}05=#c10U2Qz$WsBa_1f3GK*V$Lf ztFTawiQza=1vx)j72EgA8&ln8bsY!{Qv~woy*rGyN8;_y8x!VVf^mEqY9I z*td(R8l&n)Q86o%JbiJ6bE7`85S3wGv!1Zd{4L?tH(TDG#VuE2qUUmyeJX+u&3vDJ#}i-cA0jc_uU z9kDIsSW<8S_A7@V`ll`fB+AnKYWI+6TDV1MMg7Qmx>Sxn$lWQp(Az4*qZ7$eF@_P|&bfG8%mF zJ^mz!o>>96GEe;v@{1x_`t-5gk6-$bcWKfp9Txif)_NzOx2eZUh|+#yt~uo16@@cG zgfIz43}5)7SB|xk3~S}ElLJ_<_e3r>e-RmNmkM?}QW$7edGRtPj=Osin{6?G*-J4; zV?{>SXSXQaT(<&7t0r{7L6<(S@@wUD$vgHP6?TjoHX;e?Sc6WmVo!Gu%yIv`oqlUv`?mC_aZ<7O9 ztf)^)y_t{R=yf9J7f~g*{$PeHN#a#GeM*--`T{Suy9-23^3dOfHLi+)5(#tud=-7K zzv+2?^vvZX4uoY54@nMO`GIP>RGWEs0Do5RBwn&_`^_M7u!eB4#r~Wq`Z!DDa-N-& zx15Sd$?wXKy=zO*>RoB-mH;h7C0-TVZl{>YePN8iwk7%ejELFJlza{z6vuJrOg@|U zFu9W4D5EabBw@k%^j$}4*BnPn5F9X!qeJ6e3qtIBMDzgRHJkBRDss$jVr@e>8Q%oV(k z;-|mYQIF?(LRJ*XLj*VCW|hO)mDyhXA^Ul_7a)pV2$bQ@K<3}moWs_35Dy)S#mQuo zCp|Gr4aYu|3*>&py53_`ic(MFd;^l2`y@HtjSz`g7!@DSerH@DuQh$2rBBrMLzdg+ zDelUf^?St|A-^_x&%GFs^xB%N$$yi~CO%%gW7j9^agp^|uIg%I_W5C-(ETKhocn~o z9#cMY|-IgydY7FKJf%5X>S;~ z;0&+)+4ph!`ikyIE=KYFu17o)u2zE6SoXtO+}KVmoZQBP>c>~*#k@ zLT#hl%w>~8LK2V&%^J!@?kE0qHysZAIX&niQLMs zq)3`i=z7p{SXO?3Q+~ep?s-|iaou}!DdPu=VK{l4r?S}*bR7#9r{&6ggw+*yzxh>M z<3~HZhSwigmM!Kh&?)2c0~miCXg971l%yv9abeU=n}x0=Vs%GsS1rYV$GaYuD9=Pp zF0(O8YEF$>j!;g1M>b!r%r7BYa_aX{5XJrCKQU9rmvBT>?d6sHNW5>GPby!v3?XFaAEy#^q;0=!q)^W&!G zOHXm6pC%*39;|J8O?Vf%$F(XPn)YjwQ#eM(`Q%0?EeQD~D_ zmXXH9XC3ZOZFk0HBWph0V%)Cq_r!;hcDr3bi&vo-prR3zY?NGP_$X3!sH_Mx{x~nk z@G$O=jO&k`!-9*?dzHy*wXk(gg(3kFE0gRkazSah4*YiB8`gS?xqs(jo^O0b-ga!K z)i?$gi}pxsqsE1E4fol5@e*9{oo{*vsnupLN3HqBwSJe2SDp4|#^MU!Nt`qd^8eXIl>(p)! zTcrqOx!@|8`Rr#ww-}u!Kq{m{{Kn|B)_w_9v?|sPlps8Cz!ga)GK<@?;R7s4CWP#O zZ39Wlb(f&1O;6VG!CpYOkvvy)7u}Bh;W~-PNAwwj(w38p|DN+tZ98IF75ux3ZhC6M-+ruO*pi*AEf*0CexjkYC?Dg=$-2G=RikCPLAHwq5;?<(^( z<(0AFK%P9cvs{3OPwI21^oP$8H>0Eqpn!{M`qj@bWZ_v>Y%D97d)o-KlZ>@%obhKi zW&t6;jZVL&;5Y;VVdHlDZ*VTifyibi{JhT@|Gvv}Al1?8&-k!Uo5D^5r^h;JPQgRA z^hC`^aoRZ02yLI~=QVZ(9>#syQvTj55P~FcI<)T^k$}|)7bL_~4mK-TaV$pu$HIk$ z*PNvuvPg^2w>x(FpHuK+bh?6z!7533ntO7F7{6>1awPXr}3*k?R1NiNyGd<#2}0;V@| zWcZy)gQMgHO82V5NZhk363P0;Gyf{XMKa5=Fhnjd=S|65%3C6iNVO{ZI`a;zuY8o@ z&O)NjblfSAR@)N%jt#dyi$Ptm`b3Zc3rXD(0Dbo3a_#qHhRCigBesjp4g?RED?(w< zYo$FMz>9W&B7d}mimsbpJLY*y5?F+x1B#CtKExlx}x?ypIT%t~4;xQZCuA-)9N5c?+fVCS^; zic~Exv7T&s4!9rZK5+Q{7nzfH@aosl+xE*pa995lO#qUjaw4@t27&(}i~mnniv{tSS`^1uv-7GycWZ6GsA!JTIxV=VZVg*!+&@!i=C?sYa5hR z)GjIw~habXas2#c&xqNfBQV$h9d2?<)MVjk5Nd@Ldn(&X@R zOH|g1B{JD0cu>R?BvmWi%;tF^h?`7ywgyJFQTvJ|{i-5qw%NtQs{7*&{DRTWQWXtDf!aB_u z#P&BZi%)=H^ne!ylF^VDRq=pKvLTX>X0}kPf-RdAU}+WF5L9=bj}8VL(u@+=?KYP3wLhjU$8hmP?3NOUJ9(0^dWC(HK8Dd zRBA0fa+Zgpkz2viTF6RgTSS0P7#7#zUd(OATifCxQ4j-78V7e8^G&CR;dsKI@f_{F zjgpQR3!IIMv@4((WimM6Vz$Gf!~nV4L=6Hl)wNFo+<*6 zvORy}hNn30%RCs;rHxM_5+5Z z0Tm6p#1QIJs!la=rfz<%;CTVRr9UET@JK;aq_c0ojsVe=uDBKb+K})FK%x{~N3I2? z@*nVitEd+AI|L@9{dBCq657F#obI6yunj4VKZJ1+m6SU1 z=igMFr>^xX9E(rzE&fuLkO`5&Vl`$%s)7pbwc=WN3o^jUXE8bDdhfY_MvSj-X+)q> zG$0nep9IBqFHy;5Q2^2)+W^j*&qO08_R$b`Ll=goLL%>c_6(OD!lW?b8o_IUU4$!7VLt-jGQB4*s+ArU-^OCF%2w743 zFDSkbZoEbaqMt3NG761G3+Yjfkbf{zBi<4U!HG0yA*U}DGnv1^PK9s+@xHeX3#27#Ws*xAy=60HZjmi*E8j+@9Drf?RNpv90 zuhU$sYWQ1#<`=M-OM11#v*0$A%$J4(Y0qP4HHd@NH-UlRs8a#`vvL$cQL_hU;_~Nn z$238of^1eUV({d0n?pT0iUeU=i{f)mIN@v{WV*o{ZY4l+-*fzGJn)>lr^2igve)r< zeS%43)Ok^%s0k%+;9jawte(m%oYRC#VU5S3+G1HS&`d|?BmA?=uFwFrba-ar>Kra# zIneC_L!?;f&L6zIDfn2cIvdQ&7G6A7Uz>fa*hIwS#aL-Ym9dD=ZQJ@Pk*@;cv~$0Z zD|UPeUOtTSc%1SEHnsl;_5k0H#A%gE=|c~Ais<`jW~~QzP1xsDR$rhhD{S2t%V!pe zXl4rW7d>C<5j0xD?|FWcQ;9Yez@GM z$BGhJYVt0BgD}Rl8^XZ7gA%OaD9wqG-3^i#R48?GfEQ|S$JZJzpReD-v!YomtM z`D2%Tp$>L$g78E~sIpI@NsO>WGPT0#^oG{1(;B=sdR%CQM$~=7l_HmRcP1Aa`clgw zR{+}*e>P-oM*dK(*qIhzAtcIqM$cq(%N&0yK6b8)9Qz{Hv?8XIyCq{|51yY9EWg94 z-7N2+kmb+W5W39~V~lA{xHO~Y1We(alg2YJvB@~y&>>~DBLAGDC3?AsJWivTt%6R^ zMrktEzO5g{^yBGP2M&kk36TBr=eh<_RTQZ^_RByB_ll_>#yDTpqj!DE4d;x7GgCdR zg54A0SH0w(P7ox;0!N+r$d*=SE^}2HKHGGf*S!?o+^~)D^qagqHbbXfn+Povc(w)U}4u=--5QsbBWd2p-Fg#&Z99l<+&5Nim&BTZ_j2qr+e7(EMu`M#G7 zS5%lOWfBPOevMbug!pWaZrjozFgPdhTqm)N#&D0)JiPcY=0VkhS9}s4^cBEL`!K zvDVjt+IVC6D zsXX`H{(3$n^;}06LJRk&krWUS0~taP5O@m~d_;!54Jcv`gg$SNqoB&mV-6<1nf5UG<{>X07X`*YDJKSKsD($=>BW zai8r0mBB$C8)kI;@9NW!hvE|W``rlc`|l5^bl-8`T)hS-Bj7Z^+fNw0*Cn|4?zzXD ztzXL&heg?S{-dJ9YyQEO@vH1x9)o2Qb&q+p{_6r|hBzy^_}!Evng5ut(aP{xqCm%= zmrRyuFiCOUsxji~_PztftsrFh5{`RTX>>yU@=M`RB6Ge;pq1;?i&*|+pNUez1+TXj zzp?|~>{oY*1#T#)`7tv%fl-c@TCRZa;!(9!jL9J~Kw(6_DZVnG02H%@@WdB zpaU!CoCL@5r;L^hhn$Nr3)f_^rU^-m>nUM}L5lpUgC{`0I*Fi>3eCJo2HO@ZdtUV0 zNqM8HZYYjB`LK&pU2zLO%0UmV~NQX&OyN@z8^7Rz(nCG|I`n5*;INCbOZTrc@}^fWx=Eqz(q9R9BF~YRDi1{0B*{7k5`0td+hmb zP#1f!7dv3E&Ve`Ie%h!9ZMPY6Rk~TQASj5HLlc)MlU)sxDNK5h5m;BM)yj#SnC~D& zAi_)OK{)`}7y)B&i+de|b0UKcqWNTFf0wLRngpdU`J+MEBthAvfH!KuHVCR?eMOagGNz`NwTlpgBgEhe6DBH9*B0X8$*8 z0fPK=(9k~vK|TEgxO+ZX{XW_O#2*2l1bZEsXut$;FZi9Jr+h(UXTay$36gs%*u^Ky z1|-WcRyo(g<4XC)0rQusa)b;=#~k4!mw+9_d%|7*L7M~MY{0zQ{zG2=GSm#FgyqhV z3{M&clRyg6fI(;1WyKWA1S$o&CdqLd(mxzoLDgpFU z2u~omt(C7Ve9+;v-gBtHu8BZ&8)guI^12`{bfB`C#00~VKx#r?w30$#{z9V4T>(2+ zd)G5SVql#C34;6hCTrt(NV}WOa55ky3;CsR2KPGi(FZ<$jKE7wvKpzRW#J$u8v+?@ z2G5cNZXv2EP^5UC>)ipxz$v4BiUSa}Q$%nu&tZat{}B2_Gq>I{d5z1nUr7 z*E$C8cli}#?+6QEEG(Es)rsHb~hyc*vQg zLe%w!iUeqEqO?8ORUrv=87D)G(@j8M;~oriCgSXuV$S?C>912OM2mYI{NCaSiDLg1 zlycin|4!2?|JJa*YhO(bSBc=^gF8S2%zdilz~~2Gof>O$3=B*JEpH3mlg-Ol@x@VwJD)*uiU`e~bv)p`p|-kE$MMigFFH0* zu`4?Q!rv}_|E_pd`i!Icv?3|8ASu-3oO(v}4-i7@zHzz~HY0 zNr4Lj|1`kWDDRpH#Xejhx$$~>%5ZXFjg1IepXRe@UU(Z20;jR;4ToTm8DHf)7M9NN zCwZDL=t;ApKLQ60`R$|>VPx@*#IyULv?v4(**OO9-eheLA!@!-y|Z5um1H$4;|Zk8 z$*Qh*y(xA~C#;xR3kBPm6a)2;g4C!ne*N&12-GYzM`5ks!ce#++@X>S4;c!M_@EFa zSg6qm?*}9}S7XtRtsw$%lo3pBm3WjQFe^cr4bD+ZEI{2Lw#8c%A6m5E(?N+7?=vnm znwWMXydEf2&5gyPoknF&S$O1pcrZR#5r*~{i;ZHHQB`OHK7BC(FtL_QWVU)bZ3s0c5dqGEX>j9)iAuXjl#0DNoB@xwQjWEVi zfjA;)O~BCOI7Dl~s+FQbX(r=$g6`z521;T>SXhYMFIoq!H~dXg88n99iA7iFd>PdW zEE&jcI@cF)mS^GCUorAhZU$#XC*-6rcpcXH;_-*s3SDXxnsz0Vao^#xV(>75Sc#}I zq~(@(^zb&J-`e#X^R``N<+V)YL6Ah)x-85po;-A@uEjbyoJP4V)cs-IQ=!D(;J3R?)`3PY_&y(Iedi1k4P(Dh;9i6f3pMY z3FYrCyf}r=ATf}s;fff<3}aSKsnfJlqKKIc-`ey_9@hJPopNA!SRP4?pSAQ~ZOGtx zC5)F?GlrP|#1f|6aqQRY;WK-m`lMmuNg|6dO1V?1SQLbmDUVg!8LU<~!)(gGkf0r`~!P`v0Kpn`0~cqd05Zwr$(CZEM=8r?zdqa%!8e zHm8`{p4#62Hk)j+$!@aAyLtcJ`_19!p7S~9`s%2$SPkldmRgZ;B5TLHuFk;*ov2;5 z!?6*FO?l@p^ny)t*_7~hn?oMcegX$a1`N=x2(D^5(BzJ>l3QSv&Ayn14VR3_9;v9oUFpQo@hII-k`nxa9H?ZGWvx=x3pcc+kvaVy z#NKojiF4G1%XI^mywtXXOkct7wQmPm_iXym+pzEI2_Vgwn3GWpgB@j$&=>e~!weeV_sv z7VtVzc@SGZSZyg|VivfMxnNv9U{Tyj7=8I6W0BcwCBzqEJoV>;V2r8bPy<@vV+mwUEdtvs;uJ%-H#s5n=g~ zceQaC*ag-W@*LceRN@DNm8PUU%$xNj6z*<^5YZFC`M(tp0i z@+=Udnt`BKmtNPz7Ux25t`TA z76f)9T$xHI}L-e=SAO&whCieyuqu_({JHS^| zIfV&+U6`rnQ@;n>`~xd51+SPCiWsuRF1kJt1SrsdpH-u8FHLprnde{FZ%`xH!#1P4 znrKa}6iP-9L!KCIpxr5o)9H>;`&*kmqTvDsUZtM^7hAkWSfs-wd1_m4;6Ya%Xf8xG z?VU&-e18`g?q5ZBR1e#`KQg&~n7R8~QF|$#gZaHG6axN)+ztN>X@ovVZjt@qE1ot( zrn&aBD4=x7~0=jfr9m# zHcsaBxt3ob9`c;%VUPF)t3RJS3cC-zIdTKQH$K)Aji1PA7|EibO6p#r5~oPs3hY^K z*Vp?)B6b<{;ty7d#cuEAMYFgtxZj}$+l(0@e!XDtFbEmys*emxv)l@A2Q}R*@W3le zntmwvmlkQPn$^^knM`N1Vb5$>F7;s>>oOOyrTY~!)E6(}M$iaFa6BgB;^0c1woseS z3K4;j{N-%$>s1@$i9a1b_)GbXV9@-QJ05aVUAvZvYlS)WKMHwE3^?eLF}$a>sO`lhNA?Hr=yDbttUNcF^~E2P z1@i!#*~4_f%7>dl4v~ipFwB6ID-l7&H3lRvA493E-E-WYUj;2~MNJ(Vi+Hb49jtud z)Ta~$L=27$1IoB4l)dW$3W%baYayd0fvq8~WC4i7G*KOY&%wdcSi}+j5pKlX9$6>y z>~02Lka)kA(l{ExGaU^QHYnd=6}%DdP$cnv`Eb;}AjiHxdRSb&AfUx)lXTqr)mt+{ zQT_?c1L17d!fv3i;=1S zV>+t~_3h~2N27Nf3QELs6Wcw7R^9p&c3;X}e3N}u)V+_1k?)7I+9iZYfoC#n0Ac+p zTVFOEva|*&*B0qJi}W4Y!+cHfC-jX%y8`GFWeQDSAZ7foW9s>=3ZECn$38xh0GZs0 z-mhnbU@abi-Rj9IZX=%$+;4-In^#^G5>w%VD$Kk30C8SXD zNq|{@^dNoPR}O?mmfcXwE8CO+Pj(i@9zVn$$oFRW%Q(2%gnL=5NOQFbCe&?gQRLJi zXQ@&KyIhi0Z_OOU{w2ZFgcTz1{q6jr1IENCq`w!GDLUI$HD)cPd>$TT>Rxd~u(}Jy zaXR-?Ts)oQ`)nliXE%NvPjXNZqjZRdHQLM6Ojcj7yzLVB;*~dGKE|3S8F;haDsLR> z2O0IS>F8Ai4!9*0J)aerC>uu(C8yON7X%*MF3IHQqu|@OTiSvVB<6uQ;2Y2loUcYD z32($L;Lifj*TGX`cFuam`!B=r2%^WP@`aEQ>6xl;Wrj1B2RGX&|27{k>2mmMXpMOh z|Z*RD(TOLc_apPiewct+P*z9j|R z3Kd{*aL|S_;&J~=;CSo>0H^GG*s3oYfK0au+fhRc?B5POV3!v?b=YQd0=)w0Oyy9z zXm(ZPC-n)G3g10)+qulct5c=Tl>h6%TX-*K!stRtX@=#$U*cAo<#u_~_07~HR?Re{ zbl_Ym8x@fpgUD!@Bl*aqe5}94cfKaMJ@?1f=#l>X;WS$Cbv1DFoS~$n}#HUmnZ7S~F@@)sfeFMf@n3Q>OQVfml-@_WkCV zrl&*oj~ScsAj!4dpMiPCvCI?LsGnvb-km*Nthd)skd&f7eQ;(><*-a^=q~#8|OeI z=`<3=WxO}haI+0w_`|P8I6+54vgd|Tf{s@_-SoZ+7q5=vY2ei+<>@U7K=7ESy?lxkdvkto%goOlON$_5Ml7GVp)>zgKEj_TRk9!w|9SAN%v(j+gW8Bq1rl zspcA+2s%bu;1#hj^+z0T@LuT8xwh+4y+2q@m4#_^I=wfWBUjD(2=moaKJ!V(jAvTQ9~m?n`YJGifL6CH5w?aEeO3 zZc~Q^F4`$uZRRkcZKDZZ{K+42xc2zNQHg;fqX(REWfRJJ*^IGDBjdeZbY}wMWs1Hu zXa47nyG`dmbbvxi!pk7e`{94A64m}~FDg>$hYx(NOB3~E>&CUeC=^eE%gCIVr0Oc> zQ@Bk17po!t#GA2uM~MEBcBHIdjy65J9vr#vMDxiqT4}neZm5<9B|VPGKxA@D^t?6@$A_H2kb~PvALkg?RE%A_{N{ZVp)Nb`$Ab!@bHeaH< z!@oLg6fJ^4IgEZ1vKKM5C%=6paYEugFAx=M20hU~0RM(=%%2>e8;-f6L5R$Uc}e^(hUrC~lD zO{ddwI9)b;$H>1?WlsTUTIUvtc_Ma)cHbH`eQxtCx<86(S-M%!WA{ES5wxXayZS&` zxcVj-f!RYNI)0su;MEZ8NKdnd9yJ4mp4;W zWA#NUeUDw9&4uZX9|d^e(+giFW=UABWn1VD5ueNtDz-`5E{(i8NFg4h>lr8#^n;G% zodenJ3vatNUf+@oJq!Bq`4*>mRS=`;&mp46j7!ibjLBrJvQR0y&v|`DBnf@|3Xr$> zQM*vfgs}N#h|$)&H>e%UbTD=dlC)~AGG?yifNoZQ!44IIVt9PLu=E! zN5zxpR;4#xN)~O%O!jn8muT>lZNb_IqN-z>c?QJD1BF~!IDB{e1DmHnJd+t7T?ar; zJpo6CA}VxKP7OFVPlg;a{EXu1dX$>Z{8Yf`pVm+LU3}ABm8jPNr)6%>ILZ_Yqt^xI z=7c+(S4aMKvPt9MZ?qV8?$C{q=xj);9-D>WB*C$7UF<%9>U-_78L_=M4=1jjXW*$t z*b4ci^6g(=M7Q`~p+}QW5@tasOLaWDA~t>=7d>MiSK?-ypw-zfxZx2`V=*JBRfz1b z^Ezv1%2Euo1ipi1on7M+PS>&emToz?STbO2_OO>vtTi+5#=fzY<~@o z{UMZU6LFqhxiw-o2j>WFW-dd$8G3u-mNo;mFk)g2) zW>z6rVK~!(x$V{TYsdnB4DY3R@d33K`@cOZj{FLh(5uD4B!mB? zh5w@U6Gn@9vdr&@9_^sAcUM(|TAZ?WkiJOUcqf?+NTs^7MNB#vyg`O>sYvR;jSh>| z7VzQeec`M%wSwEv4d@^5+jX+Gup+iOAq`0*2a5^Er|6D6GFNWUPQk@dQ{9EelHSBE zM6qsf|L^|X|7_D05`IAk=<=O|%&cmCw-Wy^ElY0k`wF4&?&5zJ#sAT={QvJ+CVoFB z@qhL#kEQn7K|n}A{#Vb^&o-4p+x^IsoGfFavbMCU>Pf54?m}Ol5dy85axT~+)b#I^ zi350W11me%@f7nuEBhd?+duP!W3Tu)M-OxSrZuc0QJKKRMZ4w&wYJ60WWAoc7Wb88 z-=REWX0rVr>?=trE6wq{;=SDD&PFzNGNTR8E6=_z(3sJ^1}_Gp|JR=*(!?#*XF-O` zD}ITy*~Ryky(cK%ZX7t~(8F~rq%YaM?V7$H6ALcOmgh%i?99x}|B65%+6x5d9;JF# z+(?gZw$Nf^N7P5_T+bCTdZSXlW}53x+|lvYuikwG>SC#H_=I3B+Xo>#h2?Y=mY0as zZ~L1RQDETxLyxWC!aldf*_na7J@Lv@@V`~3z|hZelmuW;w=495Kfk;B%x8?7B zZiSj`h2;VYYHuueWq)DP-VQCiwYMnJ8YtQ2<)mG4Rf95TLYWvD3dc=JoEzicK$!tTe+V?N$xAA9`74(KPQ7)Oo_6 z*>c4po|EK&6xIH#T8YBqw#Ric1fdqG9v3DHSmD6kqUh9Wbzc!pTcf(2Gs-7M_X#NovK{jFL-osro&L3scM#7LiiN znmj;WkRp0iXAOd3LnXap7EY*G0t?rFy*MqVD7!0bCxMtKoiX~#iGp%+J_r>iW7rk8 z%8@w!biiP?aD0bAwE?)X|@KpvwW;-w<+CGPn2;S~ZVhnaGC{hfzI4ROZy0yaA zAOAd`)GF&mQ5adkY#R-v7_wktr{#e2B*$1}i%C34v>}3&5wmu?p6cX)tX}iaIZ7I) zx>3OZ5P<{{*Cp$NMEe02Q0DpwaCE>CY9OJsg5<<^CNNms@iGNK4R|2)`M{@?z*=kp zs_5E(4@oQf2MJN+w#E%+sFkmM%x8#c*#)MVTgO0*;VrFgfzrgTF(lA`| zYhWkQzS2;WM%3n^iM1w5nEf6oHnBeS!w4%7s|)`#!~_;YJ0Kq4j03vEJ&X*!=Nx7P zaK^N;VXXf}&(e*`(G7Z{PxYC8g;_R1Ptg$7U0Dl#j{qVF4%mIzbMFIPv=(jA1qi`2 zDTD1xhpEK%Ld6n;&c=c*;gSX(z!Xww+t4RNkQ6c|*LJ=Wan(w;8(l!mR~B}P&u`_1 zNkcF0hS3HWJRK_PO3(*nvNwb$>fRSTU+a8RM#Bfm?&`2=Pl zhcQ#Xyd{sv5@C4<9)lU7(jW#yX@Q;BB@{^ZAP5D3@%VxKdbGUm$Owws-^uy`x(H4@ zgn%?n)CM7FAden7Lnkn=mkks=H?_^TMi}ytDo8_-gbvJ-;dEx(qd*H>WQ0*y*P|mE znd39DfM;pMVSJDv_38An1?pUvSzy`f1klu90yw1}67n3rY}(13>#qMQq! zjT*AlR0 z30$)ewcjp>`}gw`!|N7;#ak_N0Ny&VqG@LW5@3Xi#|A@Dz+Zw1Bg>|`ph9(J&vetJ zP;?aVs3s*nP>gDmL@hyq#k~!GMhKP801ZXi<~{BT2!Nd9CR0G=g$ja2aH>|Ch90ob z#sya{056jcJI&-_@_4%E1@)<72a}crtpCvLE-g2?86+55**Oa1qC*6gml6T53hZ|W z^lJivbcH>y;X(QSqzq9il8FX?LbH(r^>l4Y;D7h4TL)D%0S{mcJIZE*tf?Ss=Bgvk z3Dn_o`$<~~UuJunRbVX%>l;W+TsUn`p-fIC_pEJdP)I`79muvd?v5y;I;9{51mPsa z7S>vR_c!f}Si+Uhe(#_Y;RID;qRlOQl+y!~#}hQNdl>@@`h zP}Kw`+D5&dbd=q@_|ku+tpJ>?72LRr&zzaIK={j6%ThvW60xl{FC z)}Q)=uiVIsc?zpDP}l2L5P&ZX(fJ&ZbiW0}&bGYeVR^-4@#i@*D;o_v zidyV)7$8HA7AS`Mt>T4IZ4F~Z1*EfO@ZJGWvvsl5gWiKg4Mvoz_&S-1YY~h zjY-{HQIZ$3q_mP+I;)mpLWv53gJmRv+bu)L)J+E)`=a$Lf5H9Qjg^vZzS#(jpVqSm z8)6ADEeByVi>C!klTQc#s4`p8VdZD&n8sMYIpMcg--+4>o>cApfevhf4)e&u#WoEe zf^}X5HlYJM#9_#qvd%#tkYEux8)2JY<>sOp z#oB$K79S#~@(JuCIf`&GjSB{eIL)pF8ZQ@O9;cb{zWokZ`Na~Guu_sA|Han4L zqsMdmX>pA=cBDPAQaJbX)&2TKeW#%qIGGUbJB}H4-0_pq<^^l6axxGTdr-HNX>o_G z26%Rq5*9-W(%p#l11@rp1hr!8{5_oQo(J7ltwuU$FwOOQbPZSFwep&QDn4iw3EtUy zubpJgjvcN3A@us;UY?)jwQopQMGiyKx|g=GKUaY#)F-Bxs>WDe5qolYA> z1>*`HYNgq9e-G~7I9W*h^A17zC=*=gg68z->!+70x070&6stOBypIgrdu4i*dy}ONo{Pv z0;yaF??rUtchtzNgVMq%V=`oEgKgTcn2OFE<|C`S?g`4pPD1h4v-xtFXuu>=DzW(% zrQBjh^vT-c9L-bCWLy>$5J_{cad>HMN%cdst)dmz1`mR0ze(1)X_DlwZF&nd)vEV- zaTC&p_J7tA8~d{OSHukA6L*w!dbs@>^2$A8{-ohqq7d=sll$^Tku**}{$`BFuvzl- zGm+_YjA&|>i_&Ns)&(o3igGNt8Q*^V27Afg@q{*NCUmBMtiL%Rk)PK(Bnr z$c}0B1{BQ^m%zr;Y1>=^8O@3XK{-rII$+#2!ugumKG$5&OO(M+u5PP1pQhpW5~|HA zNLbYmg8@%K>T$nKPHMWsmDi>i?uCXM9E;*FrN>WMH33)6%12!b)J(wxnJFD>Hvk*US*e7L^8w*zS&pn#cFwo?656y(t_1m!oz^SB-`spuTnH7U*x=q!gL1HGR9FF1WQ zO-w)wfXIl;&>R1pGX_>4zlpcrz6fyimNs9Ih2oEpKOxGMs<_8k(&yM^ zMe_f^Dk80EOeBYO|4U5E=Z~~vYZ`Dx>MEU5efTME>CiCjaQkzEKm`vBGb3`Swt9@# z7!B6&C0}d72Vq&JCW*R<)%oJ)~;Y;EU{j(h)l=G>Lyjz{Kv!H)K)3} zXt6veb*cs&tG9j*D|g=ku|fwbXqSu*4YJ89MbW6_P^mZa$n#6a;%wY|l&{_rRqV&s zJ!!kMmvPauvZfnehfU|zHJ;Z$oNVMz$lw&_)+yaaI?6@8IJ#W<0LYG~cVr#(ox>|? z{N2nNcSQLdHN{(t;cnOZL>wCV_*lyP6x$4taS4~!r6&4H#qQVVx6zxQt((A#1>Z`(e*^!@O&@Cm1PHx z8jI|I&bH9$@uvArZ#79ngn{b0L3X$Kp*34%-SoRQ#gD8ALwXQ&p|IpY@{saSjgmwW zDz0Wa_Vm7BkvRRf{EPO3ia%-0V_>kI%?{gW`kXw{F3JFuqj_zCH(Zh_&SOR^g{o)M z3`#5!%+b(Q5MX&O8bZHx@OI*3hneIv;Sgy%PMXBRK@M(3-eP*XH!`N8O)UL(oQ$=o z>xs0`foD;^-`Z>y#Rki3#Ac;4rG?-0cR)Saka}=f8qxpO@$A^4JOvfz#z0j#Pp@!9&XZ{ zXU5{+#gJ0PE5WIpY_^7SyB4}_YN@>TDz#b8(2SU-d-71=Ta?U(;Ab41QnpiIBvO8g%Ls7F6Q+T4} z857AUI;wPE)cfkhhS2iLlEld2>%6B!d`DdiVylcqm5_bqCcci5WB7fwO#GOPl>|eDNUlQL0iG3gbRrFQHM)6MV zxpZ+kTfSAq-=#SQ0KxF$@6j&O%*DI&4ylV=2z^eM@Lxow4$MK^j5Ex$T~KnYoVHAY zrGYl%>@u3$RDl)j4YB_!5Vf+-x@!!$kM~2%)ueW1r5^vhiP%mLS=s|1HvnPOXo~An zKJBhM2|SPzgXzy{oGpIdl*24Pw!3O}S~ zup9K6Gsj>Oo6Q&$(qC)iO*0Fl7r&V=VNo-U`?Ag@B#?Ebhy#x5E=Ty_lrG;WOd0Ee z@WMv!QIcgOpFuggDGP-QwV1)!mF7jxGfzVl&7Y1jHqtR8(xR;M^wH6g zLR7|!$~sd0y#ybj7)LhXy9tSSAyF3Yl-7+YU8EHDK*}$j>Ij;(B##RLQg9~OS0oFs za>F^8u_J*oR(f7~R&Ra``Na9fEC#Srb!<25BF1Kf1^Iw1Tmtv#1f)?+;aoC-R57Y# zND8ZAkr6NLoMVObI13*r+%W9|){(3YPsWaXf|f5b6s~QA1=b}2#mi|U9UD1~*+h`- zY(-Ouc!!Bj8P8M7I$FVL3i>FU%sSIbQ21m`RKqOw)*(i9`CQ9gRq4Rw_sJXDb*g+4I*wm#6tRYIA>O6R=KQgj|Xz^CZbK-Y!~ zHUTlE9`tT%_h9)Nf_5$8PV=LCozX`2N{@6r`rG2UzwQDyFa=P72 z>}*&1V{%Zb(qbOTVGiquO2!`%Z@hGJdAh2onU!Zo@6Rfs(uQ?uVA)&x(Ii{b8RZ0} zI%vpidQ=uZV{Upgxi?WIt%_uu0ZxSEg@}fbAk~%fW1Ld}-vosm- zaj25XvPK-Ty=>uBmiFnv<9takf1#Yj*tKAz4Q}p0mK{G4$O*x4N?0Wmh(+MMBeZil z6t{Sy;dQJ5y+@2fyEvKlSO2;5^pPH(`j$~eON1t2(y>6!#W zm6{$|y^ZZOTZ|(r;hW7Dz42?%i%(%qHcKQGY}gEyl?d)ZrSIV7ksn-pA4{oB>;Hq! z+S}}ia;n?v`q6PWh4j0LpGx*lHip`iNS&J@pBH&bfo}Ud!36CkCt6TcX_Fk+k8zJl zxk)%SD*`4==2Gh+(b5{Ha233$rBMPysO~+OU1@S1?=~NvO;93_&_d`X_a9EYHLQVu z%+Zo)wEvDgjWOPFadWYX!<5a4GNl*yn(|C8x15b*)`cwbUgiS1 z=amrg-NTz?J606}YI}*1Yt5nFrkBY^l5iRkesiT_r@X3+KS*f%!@C}zA++m39medw z4EjcL0cr{Vrf1V-C-H;Ri-gdp^5XSRQ3T%|n~91VP^H4GPJ^T85rSv^>WLk~U!mKn zP_1wh1nCyYJ_}W@c#X2TGAEKnW*}X`OqJogX)9 zJ0EYiMctQKp-F~~IE2I2}x zZN$zyO}kcjlLH~min(G9qErG{FS4~)jn`lGebOFLxf2y5;pHkBau^kwe%%&Hn{60w zEo;a#MH76aKa?r7p*m8S2hLwRqTL#As+da;NPRD3i%DU5;Twfx}8v;C#=_tG`X7CgNAOj?W*8AHu=gM zC9;$Rgr>D}A=j;&u(dNq-anal+FjD?paqy|MJPagUCuN~w!y6rrM}Bh_Gl1{ZMjV| zFV}u5Uog45?hC1sfs^Xjv)$@(yJI~)VBL(|k+-XqX2a%e)34hKuM2y(hr6=dPqig2^_nhG61>XZUp4Rym@ zbxM?;96fsoRC<5U37pFLFNnnnn|}dhR)BGpq{`Ads(e%hEvx+CioM>k2~wPepNqmg zM8R&-sIi0*O~1q^OFh$TgSsx^AgGb;{ORz)^6{FEy{?t};J$`LdC1RI|K5;le7+>8 z2K*}si#F~R#r)pbWed`Qe-##ROM<8T3)-L#;4veXX?k|&*wJP^$)@J^E!#$nm)`N^iB6V?K!VYQ4V8osF&|sc3IIs zzw2^53W6n8-9yMd+Anv7!1TLlIR$mu$#=&PNN0|jzlo~qfXf`fQ}%MJ5^ujG^;A>a zZv``^%>DS@Vl*f446(tuF8Y!`7mGNs=|zag{H~!uI>Xjewgj92)wZ7ZdW61i?FHRR z<4^2*Wa99DA+byNMHt2}jAZRn%4Lq>vSJuMupWTjLj1vqH~We5Z6jlb?(`do+Jm@y zuxI$eZ9iLNQed(N)^ms=0uH=6kOd<~ViH2AmB~F<8EH0dRK0TADuVGk)Tg1cp$o(u zSK4-CtEE2`ufB6j`NgL;gmpr=I^1gRKoK^k)cM(A(t0=I3^1}$EAB&?Bohx_R?cx> z_f7FOcYNp)Z9+|&xD+vjoCEn2?8W${-bHvY@36KHZDvC8+W4)MvgMCNW7uVNhWOo% z6Nixx#tZ&-)O$@|(6>M1DGP&md1E-eG4hDh>w#el5;&iGs9b^I%?d`2JaMbGBXyQPvhVDg{FH2`%!kE?0*0Rf%n!<+r$i0xHpBj$@S zUpNG&U1mE>#FmY;>FDpVfh!^PuaaLQU!h*|$JkEKT2fd;@C@cc~(QS@$y51__?qA>J_P7AZ&o}so$`HRW*OVc8hMtZ#V}vR|w*3 z2o$1t6as~ipY)lQ8Tw2Y%z9X=eWe@oGFyJ9dIQG##V1j_-fny(XtVtCUEqD*oTm4p zEvf5O%f%Zm4f<0LBE2Vgv1u51NAXw*j+A}f>JZZ8)1)CKXG0_$cr@>~#~%Wi4qs`4 zwNWB3=q9Jhu2mlFdWK5%|7iM8tY@{sZXx~2y zAA;5`^4TtAz5J4$605ntUl4W)e(=-aUEj#(zWe;o*!I0r;(pOZ;JB zynmbo4~^$=&aPvxQYo}lLUHfYu^EUn4_ZnoFLQN-$(l#iDA-RL`5bJi$47E1rMR(g zZ&wNUmviF%Wxc19x=;EYgXcEhB3Ve&@8cna7o!0&q%9Ad+mb51fqQ}lpUsz-U+F`E2XF(FRMHckI~rPPIqm?hAmJZyaXkq| z4acO<)JR}PNF7xkfa4V*M9(EE2zB&wq1%M=&ws>&r;8R@n`y~6V+gmmG+Qt8EZz)g zK#&LOz66tZjIjs3!D#*cImKns*%~X`#`$ygv?NYg`c5uQKqfdqN6dV?cx?L@%B5sR z+vibj2SkUXv-CqDa~KJKy@G%en4?{(b*GKM7mJqNjk{ybQ0}sr?IK?ZT^af=V?x88yEK;nD=kfxjb#X^B!E|C5YfO2rm(3Agm{ZisGTVFID{_8}(T$Pa(RJ)_ubR64P5 z*NG!RAmL~idxv;C;V7x+zg)**>Ic{Y-3@xUkJ$!=NW(?-%;8Nic@$8|17#mP)rWK5 zZ*k!^fOY=Ua?ND`N_S71DG>MixmkYkO7w2d9R22!n|hP>+O2u23MKTNvuOT#^>tqQ zgFgrgi@O}*RP!#D<%_ed-2KRK_sF{w;X(>xL$DXmMSygj+xUXWUcU@&qisE(xqTSx zdjT*1+^HKpU|1MwX;NKp5u-|ibrrl`#IY0pvi=UxU>)?Hz;z$+4Nm=}zDuNo+Frz8 zrWpN0w9n?$D6s_+EI;(x|23)a+#LhsI)RQIVBZK9R|m>-v(zQM_8agbZdji0FzCbo z4#4@xjgreBeze!*fyviAgLh9GAQf|@olD2 z8&vk#e#>+Q<%*@!$7md*M{tBN3C;LLt||Yei&;S!&xc`vnK2Co=&FhJEfK`m?f3qNt#zr+@NlTzOIjWVSl3%&0pzms6_WD!#QX6{;t>^vJ4vlbN^ z>_6_^nzi8f$2Rz2mWempMV0L*o@{C)u~R84`L+h{1`g<@Kb%lzlmApM%sS#kSV6CI zY`UL$Tr;-8u5R`9khgI4q6XrTi(HTDT`FXugRD>{sKn|hWk7?Ynqa8DvG@*lIi{~< zq?s4z1}1(O6|NrS88Otu;~=gM6$dhi(_;Tav`hw%FGBx;Erj{#yukOkvtZeXB&)H; z03oCX`h!;E_c;GC0;6-ptt7^l7xM-u_9z7n&gZFk`PVHqS>eo&mi~d(@Lb($@#DtD zERfQJs`@ws1^NtAF&2Nghhw7Prb{*+`S36n9-?k}C$Y=Ub=819W6W9dvy|$Q@peS~ znFhIp;F3ey=R;Z!aE}patWJIT_HItnI-Hbj+>r5f;qF5$oYNApEESd>{mI?LI@{^noV9(*#g8C$*x$ zVa=w0t05NNAk7M0F8qbwuHRZ%|KUW+5D)%sS}!@1~L}{{TG^? zO+elF^#LqVODr1YHXck~!V*sNCD^-g9owcdL7A>}3i?m%0&=Tw)Hv6)|KTFNEVn`o z_~Audnc$~v!|%6c2gN736af%*ZZ zB&2b;0Soado*5f93wTDPjL14QGMcrT>Ew+XNsR&{bmV}z=WpO~Ie~c6aZdh?ziA2_ ziG(93LX4O6+7inm$3fN}tR<}AhRX4d=RBJD*7Db_)TasM2jx;$tntDpm?j1qSHdX!HbnXd)ks!aTm0MW zosd3nLdxH-;mO7g?eOx}I|&7^o(*?FY3Yc^;gt4+DZ8>yW2_h`;S^pb#ukV4SCR?h zLE>f^9U+88W(|o&KP$PbW<++1U+vdK%}u!FylopoJdU;0iixr3~T`@e0{^gf6Wl*Xe~9&E8>16Zhy~i1Px6VM*6m?Zz#k+B&GXz&`J<=h1w@SWO7F9gxQaL-l^TCg zz&ZOX>v)Ss^H*XYNTQ`?Y7vxRZm3f2n(`7BE$PeH=>XLRf7*8<2Kf<(ojXgPZf&D( zSz&yc6UT}Bs`&3*eZOz2NpmcBUg*o{13uUrc`@#s1{CyU=II(yQVLP#pqdelL^6|6 z6yl0+ceEi4iKqG8VKn|pp5`l)1T5(f@kUM+(-mG+H&bmzv98ncR@$I+u*(8C8Frdw zx{Sogg%Z0Cpi;MPVB6)j!IX0C4-)c44hsWB?hfc{L^$p5_|-aiS{?8*32SRk{GwvxLcT;LK>@bG5dAV0{)!#)iSFZIl;w zo*S$By?3FF)2PY1<xL|bELx$GG2DGHJagn30;V!!=e-0|tdM)8?a)jxOr7opGgoAGY#W~GmHnSUJB zhNKK}2k>ZHU#$Nk(zYVJMm2=Loj~4ypIyw$K$+IN@?t$K>*|xz%Y$yp$bxb*5b?!_ z5q;8PXOo8Tt~pPYZ5EHFBRzGl#Gz?=Cw3!BvE%4K;^y)b!+~*%VhJW*0s2?)V_iENJT7Hq4TG-ngs%QcUE209Q7D zy%L7b&%-LAtBRKqaLHJTuY?O$t}6bs5?lziLmLHp%%555a|6+0s*JxrWpzohbm6>- z5Hi@keaajF)$-e)B636qJqg_N!3yC$W0u@=9I*cIga+oBgqf^D2W3e zTBtit+ib`46X8X^P2L^_WSZ*Cw*Yvr{nd5Ce!x@MW=Ss1XbK~q0^wFmN6X+NARgOg zr?Z`hU%*+KHpbUVD|+2ffo_Wy@}nnw_3DFovA|CzvsxxHbC=0Dg0k;L7o)a^b6MO_fLA&tsuvr~m=4W6_9Y@2p)FJ5Qx<+Q+xhUm9eA9U3>bw$Y~9{%_Ag|*>ORDV$; zm&5Hj>~r6H{3PlZQZD$0(yr$OE3jUxY5YsB(tBS4nJqD97aIjyq&9GX(^>TNayz{3 z60H<1?Ob!wcq48V`n!ofu3ehW;K)@<>Q6me&FSbegD0rN0?GzClH6R$VWbD{=n#pv zPBjkv_c_APhf&m!LUZ*_)~+gkgXmc$#?gq>^Rxl|jTDwfdm~Kj#6Yx;9SAH;}s{Ek%{=7 zRDDM);xO^jke7~=XuWBB&Ix7yIS<3K+Lwk zMCjg~2;?88!yzB&c7oFE?%CucrB%L`*O~~o<16`PHxfre*~{w(s+_!!JAJ_)817;q|ImxCqWX3l zH)Bp*UXb&e&K;0*o9l51cy1?4GNQ``K4JdN?YcAn_wCF(b?{obp}_u8oFzE`k>~wd z2WfIK?uw2Zcb*}dspOHp;ZM7gT-uszvk<APF5NOEuXla z8_Wmq{8sJuik|nvBb0S7i~5iFPh4Ij$NRJa9wZFZiXWsk5dBB=rvg%OT&-c5mB{Br zj@|9ss8fx+V3}dkHpiwy^F|zL3C-pJw#)WD@EO4`$+kt}VFUMXzwE^mqK z^ZOLwurtPA*@i7}-+;K)j7gL2$QCPf5ln6lm7}kr*ZDWO4 z2^6%cur4E3Ok1~Rc#T0kPf>HW9`Tn<0@UHBrsXB&&lX&m&W_-^1+I)N zYaz-ihq_n-+dj2kCvcXoVu{1j$o<$o-^+P3T5qHqF5NwB$$YVScAj>Qeb1W8eD=v{ z<`{5jqEiO{$aas^sjhiS)T%%$h`*Szdep<&d_|dS5^His3*rVO@pfFRu*R43+iy)> zJqeag#KW$`-$TY)!qugf(!K46nVSg}PPc1Y*o!9ONBd_d#_OAA0|Q7mB73rsMfo_ z?WSzG45raL-=6B`=KJl6%h$pJD}~a|?UboeEl^}Dt+kpjJI6;n3|)OmGJm|m&!jh% z-s?!Pz~pe_XMQMky)}R&)k~jeCl|M#1@Wp-!V7(NbRI@P{38J2FbFTo-%c7vcae5J zKk{jggX1HKHU|Gmen;*QOOB(EYILCDL=QEVSyYWHw!|w!V$J2uJUF-_BXzP~1l>FF zbmpL>f_z@H5

rKmT!9tgy&WWT^K$EJ+4*x)US7M2Ym{h;@C#6zl58{d^gMvK|g< z-xuqp%rwF*ewtPRw=EV2P#89aqrl@eZiN5z4$nOubzT(UgdQO;;jX;#_kQI0wPYurc68-4 z`4N^LBq;4$Z&kL?bPA0qw=JU@aF@d!@E?qOeiKeX z5=J>PQsrFP-HS!e<6Ac5t2mCpWE!7~z%XrkEJKN0p9?J+-yU*;n}JoRSn(BKO}Yp)>m zWt$+H>JUS5=azh6RkLSBt0Rde>l)WAs$w`n9`f^gK&b7M<&Nfj17Aj(H%<$|H|H}k zOIzxK(iYv{p2J}K^PW%hf~}MlCcXl38_tn^?yyYR0(~=VA??8zaQ6Nb(uaSae+s_U z`V3|(;upy=W&E~_xp5;<-ZEQHxaEubW(5iRKD=(1^Kk-OZ=bt>;nC?6c}^(54RLDm zVAcn!ZqB4!#DTDNMU@hp=W`~*K-5jEx+QB2GO&b1)YHILwdgGtNz)(f@QdqFZCX_! ztFSKxIl5olsoYRVoe#H6mGgHAf)=c%GoJFaq`%-M&MDC0u%UVM9i{EQ!*(PWzleXX zwkky6Jix0+b0qV6Kv4~T^yO8dkoCND#%mfxA9ZjET^z(SWG?IVCahJFNEn;)ho~sW zv7oG?F50Jmfvqy5I2*Ia6+{|c; zi4aFbZMZ5+rnu44_cgSNI(@2(Ocwjfn`oX()8N8yalxQ-sNls&E~J_N}rFk4atDWnAu;{CTL;J}g>(lycaRleuYz(SDxM z?SbvTM$x^;A0TF5LA%K$k#hKqIjibEzHc8iV3m=Jxn$Te(U{RK5x@*r>$@bs_)=e= zgq)bs-1m#$$6lSeK7>WUKlf8LG`AA)-+ z3B)hp2xR}?Fhuucn;mRD0N_6_`u`h-_}|ANf}y_z{$Dsm^-V{_{{x5k&90ej)sx|& zaJv5RoHQi55P+0IB^O5}r+_CATV5TRW;N03*d?-#(N{LINWOb=agy`Mb@KyzQQ|1u zKS=45ips{K^L_i4MQ)m-LxoISd3THm0S=v{IsFP^e_##!P0^zp539>|zW`62@op&3 z&4Q#xy`NU5?`4n8;8KVy#e zAua_{&BSD_&UCph1<3UhlxKY)Bm1b1w###oU6u9{_8pq&>o|0SH9d*oGn(`Ya086^oZ=@^GoW2tB zx+T|O^5urxDE5rji{j11YgCeuepr1=qAH`ZBr(7`(=F*>vX zF+v;}5&Z9a#@~aZhCRz_;;5o}TUV`dp}P$&Nk)_o(Zh#RO-}RvS@C{kS+nzO{1E1E zdlUEuM&PVl;WQ*d*0Y&P+!}l8bGMN>*Y5p-UynYQ8#t*CGus!@t7m~r@IsG*@}l&C zt+Z~*64X|O>kcIQZsmWJ|Lxe%uQ3aLB1RM88#^lS{3R2RmZBBBY_)H^1V0NuM=iZXUxuz|z`& zIy)A};x~KO2Pb_OF=Hz2Pma#v@GN!_+w{9ZcCjjZnMg11GJVl&q}q4wh{&6^c?)UK z3CLu%U1qdmWTxFta<7>pg69ejn9FSX${%ly zDrXjy6@QZbmX@y73v$Gv$~PISb6XN*6g&?l_`n9K&1B7+Wth>AxI|uANx(~YL1t41 z!<-EWF66CrB3cp#-R!ryn^cbS7N9M3wp54mZ0ii&H};`^*ZVDajNanjRfFQ96W*B0(Q1@rIP!;d#oFH`dw9 zLaZ>v@Js3H=YaUUC!7>G7aAVkB%e z9CkB=2uwI}tabnfU=>#374%NT*-y;dNlvs5H*EwrR|HE?6|xR9Upq0E2HS1*DD(pr zQ|U_^ndB$tT^r>TW}rsII7%IkwB0Tbwb)n!w-stDpDLdoYMYF?#>?%EaA!VQ8u(>U@DwT3GYuTYkDhe&pOkTL=Zj z40r|gs;Y28Rw2Hb^VH0q;!Z@2DZ>=)0BqVPi3$KuvY`Aspd{{rK&Umk?f^p3Alh^x;!II?wB;^} zj~z<&2VhWJ!;D@B!GY~XMA{cq1;nz_E`3AxehVuhEkcG0)itbskEWCXJ|Te6@Jb(l z0c>?_5h)<%86NZrCt%BkQ|rkk&XOUPP@Sk}Je31TaR!Ff13XD)ds5p@ zq_`TAsRtEv;|E{^e(^FuA6mo^Y^k8C0{yc`RYwK2p|y34hk;W6=BjK1Il(BFNveDQ z2IURTWbnwP3N1t4Xv}m3w*QtZs@KYuG+`+0Qf$*iiY@(v(ks1tzZ1<16qwy*0y-Kh zm}aZnz_Z=B)STA|Ms;3m|X`ESqO5g#Cve4n<8S`tNy1U@j4Y<8?xzXJZo{deO@njNC5> z&o&2hOW$mQ_z@7dmp-o?hk}~BYzQP=yxI)KFG(-5rX%SDukvUu38^_Is4{N%R0e7EEB?#d6C91 z@J~NNcID@PeKs9&?)kYd5}L}6Q^SSqr=W|1k}*T*v>bYPf;B4&JK!fA#hcv-yp=hL z?%GvS8LlQPRj8=_(iX)aX|WM9^L)5Vw1Z5tM)UP&oi3h66=L`w?Z!Y(4w%K?;MLEe zAA$izVjqlZ*84wJFNI@Mt0s{?D>%lw;H2>DZCWniWH1N{1ZKY9y6LCGs-nGL(#b6= ziNvdpYXNJ39r7OO%cS;2dJSCeG(x@p7PcMUw;^~kI2?u6vvbPiH2gwQuG@;1Anj9 zfc8wQWhHFmkW=L}H=-NO2V0mR+)YDZmUT$v6r=|#tsjuJo9jm%YkCEIUa(Y14EQth zk~m(*03%Jf;+(#$ejAGrl5G_tMEC$GJi5cYbevioO;RCBm6}tc%*{kzl+idnU;cfI zHJlnI9bP2u+fj6;FfC2n95HX`7>fDeALx+>L#)6L5|Tda+al`830Pue0c zO<;jgbRFC^)7CHE`*2t_g zm~8l9%W0Mm0NCcjbgb#>yG3D*dkIQeR?FvC3F+b%>PM|GR9nBNHuq8B7^$7}4tNP_ z`1nOEJ0NOBko2^$O^F;G+%)R|GoM2zi$(EZ_ltMsU1KcO-g6B5pfWnUOO{uxJp#32 z$f9I5L^S`Ag9$7>LQyql&A>QC-;h|$V-b{S};?O%~u3j(Br+> zoh(cnAwglZki)fk7tAw{^;j+eC5|3|uN*V*#_5cN?upPH~rI{hEQQ%Q@dW{U;T(abRjwG7DUFyU2B_zdJwuRJB}>4pZuw>2nFQc;TVAi z_QBtd`p_p@;Jq~ASI_I8&+g|B)!?j#0Oc@w)|mMVpf!;x8CgpTHyxrbVe<2YEYZuYs?Xh1a*z# zw~D@U@S|GIPra_8o_ZCMSK5p(8R*`UBTme@2c^emv>!h`pa_=~gUUJLFhtbX-Qz$g zw|{(1kJwz#+at(r=Aygr6GVv;h!SIAj_ifViD=nUIY&R#z;N7(>nbWTJtmh9yRc*O zceBMF&>L?W&)fo8VH-=q!wQo zF$3-Ua`X81sZudxH>Beyuz47T2VBV5QAn|O=)4?k!Nr>2Dc{M{VcjT`&l(eK?ezn3y9jwDn;Sdb8Lw3`#Qt{-B+MKc;?%eAm}q_`vzJeI%f_zT{#pSXRO zf!G!^VFHdqueHDSjDURBHkqeCntyhV8Ni!CsL(54#|hlZlQ@E zgWE-I$W-a5Q=v(%scukoN0A*&`gOblStetfL(krl$MGKR8DUCfPiN(qmiFcoo-wE+n@oxnop`5h#PW63(M z=9;+5k91LrJs&^J(QkPUe7tI=8TOpTo7_0g=iq5bZpPLOhql;K4o@AMjv-DC{UEPSqF(Dhn| zA!u^=`?PH@Roh#n-f$CZ>D}x41RxG^llz!XBi?GMVbyD2N_T}lnorTv$ig~PrqR}7 zaIc7=B!&+y?1n_UmA78YKxj=H1h@O7KFr&um2$(srtO?xwmU&J{?nS%;k&6qG0Lnn zprtoFX{bU2PLGJYtdoBXO(3joQ;mkTz@ekDM%^i2AZ2;EG1HVsk&Y`smU~R9UzUJM z@rlRakD%Uc;l?XGPeYd*4-Z^!TsKQ6n-|-7`EYC=y=56y`tH3)yU$)x4`T+68;>F) z`X2?3kw(0KmvQVWExLS933A&}3h~t=N3jMR3$(3Kh#&?Fp>MKJ3~ex>INWBc7m)-u z>2>BS<4&LLM{;NxNcX=`pvXqGB5K8xnofPr`^iHsgHPSspeSi#xKGSYfx^+qwqHcb z;sXvf_87cRA=kHawHz7IJ>l>EHKD);isnc_36HAe?fIeL$J%j6{;?c@)3muqVjK~2Z z%BY0FMBoH$yliqE}r<@<;_LkR88(h+}K zmzokBTG@IqY_+lF(3m$vo7lgA2+qQ?$h3H6T z`ekP-N(mDeGBJKRC;=c zh2>RGx#WK{bThiuD{<2~J=+TM*l^>cAt9nfY3NE`mL#_gdIE{9bifVal90}g@-@Tq zoLH$L;}JNF_hb~fWNYf*qCsk%XtTO)ABn6`eWgSgmAEUaCcc!iE9s{yuTQR=R@tTU z_rX;Kk7OxvV_(#MjqM#>KM%?fG9Mq9SjRzZwV~0w8augUiMjh;@slF~?WdlkeH?>P z?uIs97w$}WTZx4pu!$)y+e8HwjBLq;At59`Qpm?TYik@Yn(dY@1pCc&gy=(>k@~XP z8X?3-t_>Rxu>&W%wuNmCyY#3?$_mk%j=889Nghj;JVCwIah_G!1mwqQzT_5Hx$ZuS%#AJ|nS1yUzaA?# z4UW@j zk`A~EqXS$`+O3AA=q-(Z%xzRIQglcd?H=hyD6F^oU^-tgzWw$is@@h8;gy*-d`7(K zFHc%rB*Xns&y8YnVjt@CB1=dFcBldR{OCT`(5O$P%w#|7cqT>H*<>LS7EHUFKtW#- zT%z;&npxx(_2+L4z5!UBO0DoFmYw>R{5`IEC>$(NipYZlv3i-s?dy18Pn59K!zzQL z*uzqg-7#mFWbB$cM{49pnU+s-VEmtn_0vBB_zG=de*n>4(kj-^R}}a-1xiS`ET{DH zrzgL~rLffx{CfNbbA406Vry&Y_MGDQnaggm-*hRWUOzKgIkr+kU&GZVLRYUV+vNes z#JN;#J85Xp5@8^JnnU@e=!sE=iU~&dQ45zZE+LCw zw=vJ?ekJU>S5gW2ZI9uE(Mm+UzkD5+z57Vt;FPDRTd8gm)%J?vWAb__Nt>@)oiKzz zn!ow8Kf&qLeK>Sisw0+*mvsr_<75SgJ}GL7nyVFl&}nJr{L@BdZ*~C;$B1?7>BtlN z)t*+&%_k64hwiHQJyNKl@ex*knx&;8l6|UAG@CDSJbdDt4OR$YC76;Dn2nfG!zZ5| zQ3~K&C^#6mE#3UUY>soErz2|?*YS(Y#ZAprMG}2k2s;B`<)1Ui%Y{9JVH~-_W<01< zK*p00WrlnaWTOt*krG2zkF1VbW?un zX)#U@`R|Erapxdk6wQIxJducqc#`EE`}~1EK`29Y?|hbp{8XO`EQT5)5Hmh!H%mSq zBMugZkB9cegWJBw5K+khGW+eca$OlYQ<9nvz0h)DW*@+*6Z-8lU4W-gyMO#apTj47 zjn!%amv4byH_@4B;IzkOSjE6J3p!8`Zo3_3!3G7q@}Kb@9iZDGUYI};x4HwjTw^?X zcGn$oIbU!PMF^jYCb`X8=v7lAcD(>1Rpaj?4qh@sTZGlH-%84M3>tj=3%|4zrPr?q z%qDy-64biMB~*RE*9CaE61g~0o#uk+s5akpFksh_=Z~#XFFACD@?4C!B3(Mw3)IK1=#j&o<7m+%WI*QCEv(bJvcdKaJ79^<&Q-pV(l_2 zw&i8fn5FaY#7t8>Lg*&)q6m~_8hN3~5?K7^p7QV6iF@|Q4EKAbIOgn{#ld{bK}i(K z(nP^snsYU(W!+G5g3#wIepv)7gAh&sLUX~jBvb=~BOE@qb!)e!^<&FvL?p~6*FTxr zBnH%nK%kIQ`Q7TWztv4~-`zK1W}h@ZxpZX(rQe?>EzHl5=H&`St9eZrtVYEsGP?Zm zVw8`1Vby6%`N{OqtP@50a0VUNF+0mZ|u}UHQ|}GZr&Wc{wA(X6P3FR`fR1`h}u` zYFsz7LV^<2t|>pZiWUqLw@%EoB&3mvminvU^y2K9qZDu1Z0jKf_}=yM8luE}0WeuQ z4l}q|1`GGUlUOA^_}MzF;d>XfTSCYxoCFh@I(;1+r}1l(osacYb6xz@?@WcTxPCYe z(eRCismC052!;PMl_=BTm+ZC@vGgzBTYq*va zwc(#_q2znsMRaB~pq+#CBjt&il+UvA-!YhhdUxcRXjbn1Wk(LPquz;Yrv?&KJ682y z(c|5(qB#X>^^T&7H@8^fzmphNs5Z3r9J^SJ2g-Q~OPo8lr_l59c9_evv+dRHpr{rBs7r4(p0WA;8OjuoyG=i30MqpQ?2Ft@2tb1~PX}#PUex^T zTcDC8~{@BQHvYCKs;^u_teO50kI~uERU;S8$T>ym=Jn_dhI_muB>nML4<9 z2&@T)LGUOky!Sd3|d^9`!u}B<`}Yt9BiX9bg+x#R_s;DH@F| zDtE4_SsW)azK(%!Qie?+Dao63d)< zRr}S7&q#+4m&i!~ZVv>z`xLC8i$WlTP4fd`H#K_qI2HU9j}Y-NWfLK@R^@~zmlCV4 zKjC<9j?Bv!8Yd2|`(}%JAQHLPX4Cd-+cHS^G^0A?inN2z8-~yLlC@4DGj7qC<$`_V z8ttif^Gd&XIH}@LPlO~mogd(I$A?Z}Hu(_vDn{e)A<-lYJRrg+h08ZU?tNO%_AzHM zz%J!=U)klIR=Q_^-Q*lg60(k!ke_zd0~XAd<{{Y&Q6ebQTmQO#Az+ap6L;t0rSvwF=9rCPj#BH+GmB7E~lGoz$1D@=Inp;kvnLbwmdMo>Z2fjF* zW0H~Qzlc&CSLxzU<(A)=g=FL46T}FXB15XsMJg4*XlV^xqXda)u??x zYq+Op5mxRU?mBn@ehJe_g#!|EY9fgg;l>N6OYJB*^le;tn;6(9jS#d^mRlud+A8r9 zmOzy@YN*>NnCBc(Ad*nl4YaQMZVs*4LX9PYz-+1gv@L^zQTkD_p~xl`#qwJ-wx+s% zer-i*E(ZbpI=Bwx!A4hYKK~j0P6uM@%)KqPm3(mB$NJuA;drH3I`-4uw6mpnFg8~&;k*0O^)yBVGeJbWMd4d_Mju@K)()_y-j!0(@x7r)@SV|0uZ00sd zHP|$-RPzk!Z)GPsl47 z*wFT+PUOg7yaqQF7}Hr(*~(wr}lyuk?`pIGGpr=yg$10~#t67gLF znPm}|T*$WA_~I&ayTx<_q*>Rt+xEJ@v9zvky_G72F0SY4?W4>f7yL9+%_T%5Wwr6J z@1E3}zlv&i*b^AKg|^HRB>5=_&>gCGl~;zGzHc_ef~*s`4rv`?-G9KJDnc^h<@u@` zLU_25yDK;q^Rv;ZfAKJ67-9T$0u;o1sqt4RhF0gX!z}dqG{^g5R8B6>OKJ3A&d!f5 z-M8%kRtc=oAiY_9tm^O^eN!ioPxSng43-%oLOm@o#zp8n!|QEHjxdvf2=}pzOSIax zo+3LzG@O^}i-b5e)BSS(raH#1{{icCGP@f>lX~FI3w{r09)D%( zO!2Ae?605or!p+F_Ta6!G;}qZkpzH(3zJ5F!Q^%uh4j4rp#-V~BfZb>jG^6>mjo>l z$Gka+59nR)4@`~npIfV|g?cg-MZM}YX8yQ&>5$ywV_cK6%UiHyZ+pD6HI`71*yJqWhPMSAyI9un+`0dh;9o+@9Mw=8yl+LF# z+ZmzsC`7tw4kt^Rnx?GaxyB{vqJGEikDldobyc8SZ-e zf`a21xGWn!JkSIh8f3cG?ZY(4BB)!o5+$K+hc}llGs7NKbYzAd>l$@yqU}d)LNB_r zB%SpJi0$UpgALtsJ9H#b4GmIg4eP?WFHi!6ct8=_l9qUw>k2rUCQR)VOA;g)5Y$XK zfS^13=(XtgJv4}5YjE-T$^jOtCGTX_p-Y1yl|yD}q;e`GMb9XI^5tn|3P)-dS}aItt!{ zEnIeCqULPYKp!h&#LQ;^^+AlLd&mH_&<(3gn?Cx9e;3JugtsBFH=j1$m1$TsZlt|k zPO6r>MG_Fs#ujb%mo<9|qc0}vBIK~g=c9x5qY?t#hO|z#H>a&mME9bwL@En3dkopL*P22^K)qYR7HqH}rx#(BziZX`P_xaK7YY#AYom(&$XNQNd^c^m8#xqZvj zG3#mHYXc=n`KV*PHQt%J2hh0uiOC*ZW%)x97^QNO*^d>uiTM4}e~y=d5fh!GczT(Y zNH}P7gi%R|GQ+eRk61{Y>)Qiea&`V5w5<}9!_QlzGpaJ7w^&$9D={gq*Ki||Y`~DL zzhs01(2M#Hq5U|VoG>J0h^qwDSyDP|@!O;T7TDrcdDprm z$fUebk6Qq%t$BWe-Z0FdWwlfs;n@0-+A#TUiv9MMAF8c+-_ik3HdSLm6i*8|uP`w} zhu(U|PzU|hG<@+UqK$dJ5*x>yyps5pmD2q7ptGP7@}_-glmM7`2j!Ghovr&NA!h)I zO9G!z%o>enbybEY4qo(YM-1>w2Gwv>`Va2-#(B-&X>QKEZ*a!_S@oXSQVco_or*Io zFuof~^3$UDMJc&a<;uOM3c>9IOoM~MNH&gjqJTxhk+{)65x`sYp~6k{5Uc`m=w9t6 zIEM9|YYXlhRCubz4J-v&5D=XIKw;5p8X{gp^WF(`W%Lfu{*6^2hQJjFoShW`!P2(e z@nf&1eJSyALok1$fvVZ;SRqvxT#KvL7`)VXkM-VMJF;&S53XZfQ5VU>=fvAzbHp)? zbDZZ`6sPeS74*fms8dFJ&w7}QN7Q37kXaOD^N~I-muHbZGFx+ilnS#Bf9i)EIBp@@ z>?=D((0|pVs`FC|*~Ia&BQS?f7_*OAOveP@5)Zc>W6k{fUt2*G1)d9DdyXVhnkW6ZhuTp2X zlmKJm3jaY8>BP!U_}4D8o?Ggi@a;2kZQX(`f| zzh64;VA_l%(G7osYhiE0~0Z1ZFHT(jrx&wbSdfF&5tG z2%5%_xmpnKtqWcKdn7{eJYes|6@fUe({>ooqX>*;WK}+h=HBA)vF61I9|ax$fg_b&J*(YOdeP79rVj1e@j4t z9y?6wHQXoEm^ocSeZ}iqbyeW29YU#Jyv~6pkmbucVt zjkDzsSgohS&~-PF{te0~@3Yt9P&K%0m9pCv6$oW8_L4x%b@&d#EvA(!PgI__6>zA> z8{1G62(>AfU?{BhL!lI>=jUGEB%2R7ygvm`Hyb~i4kEkU%(#wqhV-{e1v0-gJUj%( zO2txrL5{32NNy*C9U1?$PWwJM1#22gWk!4q()>d~>%=Vq&*jPRS+zqza|}3p^bKj2 z?0i~!S9YcES_?>Z(nWx3L)d-K{T&PG!pVH8(ZlDS}cXrdP*-;j;kn9$KQTK zgls(lUNl@vpno4qBXlzRY1eTR*~S)T_LwfLrVVRVJj-_cTJ)m_{FTA=#iZ8B*JP(i zo*vYTqx@axK6qE?Y`vrs-v_&gPha5$9GJ*`kSh8p;u{(~vsz%^r_~-9jbR1On701< z`iQus!}wma$E+!Gr=R!Kl!|or*wqv!n0UDK!p>erSKx(WUgTRp;64A ze$Q1`JHhaqwIdt;l#}sM7O}H zOaggJVceO)KA7tIr`2P4*V*TJNKWT0T+v_@WG|YFDy3H^JdBr5*{CWlrcrn}1~uUi zl-5}<%rY%A0!qpUB@7>myx)bAVsFv*xbg~O*i5k(CkjXg1dwSGBW*kg&%eYLSs>c- zQ*i`CX>C5JZMQ?mfBHkV{xazgzHsPYenZoYh9o$pUYYgD$|d?*Dq5ZtF;Wv${(L#S z$1;3Ef}W-@4VMw9=J@zu0?`x3bIcrhI9rshEy_}>L)j*vs>&5evA6!?N@XB&g;XNz zuc{Sb+v%&!W7{CYB{-a0_X`{PZ#HlPf_`*x)XHCz5J{%3JFNZ<3IO;&yEU}tZABnD zIWAR^+4VPL7<`TbB@uC%b6R+B!eHSlZqIpJ^QLMvN^#5IduN0(b>jX;OmTRoj=7YH znlYhPhRb5r!b=LT9rC-8O7M}OA2ylJ7*WV|jDKXq11WJa+KM^EZc_BnZJjBk=1gi@ zxy19qF5}`EU$uhrF!~+%{h#kc^jF!ltHlr~QOx`7)T;ziRS@v_r~4GRuE}HgFp159%y45UZ2IL6RJ&azDx4G0~ISvIkS}(;3swGhOvMCL9!b z{3i5$GJwU{mX?FTV-A5*>>5Vj6RLr9tX}EXAGOWMf{D$oN|d40&lc^P(?1hUjaP}l zMJ$&YEs0tOV}HS{@mCUlu!o02^I+Rjz3{P4aNms@ugUAcA9{vKtf0kmH>V*ZOeB4z z&ykjBJbtkOPLc+%mBQbjr^?hXZFi$=2 zf{yv1k^+VIl4@q1n)c47AB~&PPY;Te)lTc%; zx5WTvvcF25KyY082^BB{P^l1SOoA}qoG4v{TC+xS>uO(kragoze-9gYDqoM1(m0YHaUgP9!%gA?OwQz@X-GDz;tWzh4?$sow@)shY2yf< zR%W&m84=*&))TQHsrO_A{@!|SQYFSI!3gWnI|QHiM^{O}d(-e%ugVd^q?W2D0J>nr z_wNo^sEbIAMUmc9HW23xL>tF=YHY%6m~o7n)C7MB41*4;xDEl9p>QnZkBDud5BkuF z1!SEDYQ;X@#_oi7-D8cj4@m^#&8!9i-pd3)jfAMS4_l5idmB{qoMyi>cb&IkUEXaL z9MD5=pQV-6$H|;LVeuHG&LoTJqIYt)FPZ_`gR!osQ!-O~IC&L0x|ctmB~rpbM@UPXgtd86Zk|7HL^t1Fpdvx@ zf}C#=&VCPKhjm}40=X*Vt2Y}s2)CaXvBedA=*Q*AJ4EZD zDmjF(N)on=DezP+L*kV$!#Nmf(|%aOn+&?Fkd>dm!Da%wy#B>?Kc!Qr=-~>6xO}>T z^h@=rK|ed$6roIyQhM zwY8+JHF>ph?Y#VSc3!m`{Ey)Uw~1QtmOn4uU$u81+ow;QzHRw+WMTPgvUOqUHsv&} z&Cb|UB|Pg(w!WvU6YjN!P&SIPI?D@X68WKA%uaZCHj{GaCZj9?v%PY zxi9L&F6J?E-;pQp1GudtQUi&K?9#P%RaDtzapm?fxdEC_yMym2-!m7v)ilplbdkL0 zsJr_xrPyW^j7zFD|)k*X>LLe_Ee24BR(rm9o}|9C@EB@o%a#G zMY|L~uBhw|-gCd)2P40K%ic{;9^Pstyhz%BIPYM3Pl-ddejG)9RVEMrSEHTPX9&#q ziu9|wfq2L?*Te6f=$rv4K@ljt?8*x|#B1n7@s}$%s8j1;|KxgPR3C7ej2#eQ$WYs_ zke|Zu;vB|hf{R}x#L=x6&O@CZ4z*N!ehr*v6N&V>V39}u>V+MS03K~1DQ~c-n5ITC zvv%UPDqNGFl#;M252|=Rj2iq3&)vK*1vHJD$%ghe*uq>S!A^|2lVwmop~GoFxm3kr zHgxH>Xo!)umv=A&4MVUC0#K)Hl7>XCu$~6q=5{}C)5ULH1i@^RCy%Sg;Wm}|KG-15 z4kEt)!*1UWRMSsakbvr8>}#?MLgMFP;%z3QiZ^OiPppr0`L0>Ntrj0P@xK4D(z}-k zq%gN5u;r69Cg$Y@y4*{4`(u;_%wpD#Yype>L$3K^)w@k>uy9R9j6(@{WpYa})@< zo38dkP*Xg>Z(N)p?O}$i76!Z@?Vq+H1Cm=>U!18p`MuAIM7nqS1#`+m^Dfvc0p zP{Y?=GBf_C(VUizzj2nD%$* zaPb^Pjw=}5{FDtE+DARYe$t0sO?>vH)s#c(=}4l54{{}|X#W0d%bjNv8m>66!;?^C zM^_mFpV?zrtNz27uK($)wChFp$1DhaD&LU2r_50=CCyEMF-D!uH>a-R_Df^Id@qJO zYL!vl_6?c{>)k&ga(X|r)UUT z!sI2%1{w)T_@V$CyX;Q$7g!@q^E;FmOck)i4MteoB~AC2|7!2M$r4F5896CyGB2A==N)su93JQXt^xmbYfD|cGq)9IhU&eDi*K^)G-@Wg9 z@B8QdPUg3Ec9NBqWF;#*d#yc*@s;c&eWckn{d>M&)ISdq?0>i+pQ(Z%EW{>np75e8 zHc341cOY`QWAxJCY?#@-jEW?pic;CTpI203eXq9B;-Qt}<%5w5+YRFkN0pWq_8Zo-69;W{9I?@mg!_0a@w(-v|r3RlU>sg1M4p}1o zrpP=u6vTG$prs^@6J8We3n|V958`xbc5rCE)VfxWD5>s)9h)MlG8|_`tkgEQu76~# zUFMc%@@@Y3oU|XD-QH3;6~q5sxscVQH2f%Lw&r24i!>Q;*|IJBqdKv26Xez93mOab%-`?}?`o#243AZ)JH~bP$zD8OS2>)q2d^ZBC)W{U zgZ^_x;Q5@qPqj$&ZW~9|FrCo*HZeT-ef63AT!y*H820kWF%upxqHi;^+X$u zYMXm3cjLZB`iLV4!9;HuylJ_4+9Njiy<|K3qr){Bo{VI>BC@D{p(uQDc?!Vkg#d;F zJQaC~_d>eUGN^$1yWXrP>0e1VvuCcrEW&Q#z4JMTn{SDAF(?<6$u-v~dR4+oH?Jng zb+pu$tb!S(^U*#Qi*;1w)E+49s2C1;3HC0XO|JI|J44AA1j986_hsWyK z+xKW`Y~6>i@$~JWXiL=zgo7iG2GE>%zAU$}D>nTe%o3)q2a+tnv`7M+!YkKHjC)SZU}6MvO1R@@OvRobnk9|iW-tY>EERi*UQ81vhawaUGDICJqM z$Q?b7NMT*}K()kdRoW5U>Oq>)^GipL3@oVyiDC`xtCrUe5%+^lPp&xGoEa5CD^A9G zbbYgIN?#199+sw!ccW)y@Km2_eaG{KjiBgxoNpJdmBtmZmi_pUc@$S-@2ajXoR>F%ZUK-vfQglJ^$R)5Dc z2)V7}#DGV%ci}r%Z!d#tW7dc0ZtUq$CR=p3?dL1hO=&NR8#W*8a6UO?9}a@L1h!d8 ze>zkDWI)YLJu%zOSC13H+Yp(*HeF#Gn78ukBKO?P38CclLB)>Ts1;VSy5M>c6_|3m z@X|ZU2O^;5!20lAbJ5SIdHa%!#$9|;rpW1o*8_9cWEvAT;2K35fh!lJDaoS zFK#`~a@Hu*$a8HKTE6lZf-_?mOLzR)`Z;dC-glx+Cy9l2oWNP9j z#(XPkxc%*FQdhTn0%6vRK-x~nz%Coo(W&tr1lNZVcZ{ORe!SUKr^<6AX?_E&3_U*G78&0c3~dmt0mDhlPE)FbOSZoCd{h&mcL#RB{#!(|p*h^5GF zx^BX9M9ZIqZlQ=B3NJ<~`*xr9-cz)r5FgXzIH%Zt!LPNk&SfXfYwT`kb8c?0`<%lX zX~Y>^4R&)8`|G}`R^R&&elNOYH-_?z!NF2zA>4Og+tg!| zwYeF6s~7PC{d|--_{5G=^sFEd`bNy2#DYRfE%*3N9;WcGnd^~t_ z)#YWPeD=(v)E#ybCvWO*d{(vHJnKtj&f1A8dc-wh@zM(yRP3vs{14kWw=6 z^jWU=`!7vZaC>>`B&;X~S;_URE@PKF{DJ||W%^eneZG!A5f!YW6u%O&KiT^v`Xn-i zvj_2maK%rf9OJ9gQq57?O4Fj0J~Rmu@<3QGM6Wq(|aj;1}q( zC4*k_l=jD6ESw+0i*9*6uE07S5tF=jXsOd$+W&@wL9$M6uV90?fpY?n^QL0+me3u? z`jghb+&Rd0b-L~;$u|A z!(i*NjLm@%dX$)qi%p5P1_b4LgNT}`$O&5Z>HdtM^X-%p_p#N%vHSQ+LHpase6(VuN7}Y44@^YHYY;E5 z8^tzpZh11X<|OycWu;|MVj<m+<1=T*6)5TjClAOLq}xq3&Z*8H7~sK;c2IE;<*g=?@lwoQz3VuZ~RR{H5tQqv35 zyL>0b)FoCLr|;2x-7ya#yX3m*7vA&TnHR@*Da++2R2jqf_v^L2{WA1bN&xpr0*QU; zX9v6(^_Kj%SWfO#U?=O?m6irtKhKg+NWqz$8>|9KComist8V9TxyuD$ih|F5935tkD?;c!RUA2cQ)AZss zsFj>s97~4YSBgUmZ-1CipFz|yL^Pm9j9xE1yu)o2CZ}fh;;>)0`1JD)a6|%jktZA9 ziDp~J;#=#&4$jL|>?UWDNUu}TG+H1C#Av^8x%TS35GBzk3^_?lpY2UenR=y9a-FO7 zz3g7TyOy`^CPFsXS*~`ge@WAPwO@H{8Pw?w2d-MqLj+MUY`au&*Dy80*TL+1KUWw> zj~VP@dK3YkI>gUT0I%UNhW8VSz~J=xbw;P{mRE+yZTo{qOWL=d2iQ^x37zPrn^Qij ztv|BD?f*t~o2WrWrMwHuyW^Is-AfNUxlk-I%-SEWHOv^zOb$$wELg9&cvKK?WONq3 z)F64P^HEFJ*_>?`icWRzx@(Z~Ykp1scHTJS@`JXAVmzx)(HqLXeYM#LRWP%v9m0~d zS2Bx@w6YeLz|h+Lf@UnI_6#?rg`Y-s-mAAyCbVZZR=$()O~d!ez!N2$tFP3@&r){x zlbYd+d41tbN13-nPzpE_8H4`E=D8*+yW0&=KX>CJhHiFbJGW8L`~<<`)BM9R383hU zZT$GQ8&{IH!-VZY-)x^Q8i{uqPT=$UsUBAFk?zv*(9VO zto05Lz21EAbE<|28cC44OuLxp7_7cAW-c`%cG8qaellYFFnfSQt!7jv^&~!5*ZK~<#q6MHzfL-!lg!rPnqkfc51yXcor(< z#Y5Nl!}@Dv%_GRl-5BcbtuKBmRy^?8Q9@6Mm1Hu$I#3KhbaSrXqaJy9{Dgwii@G@n z?Qu469_!d_bH?r4-OzYG-{U|Hsv!j;ht@ui(TBJ)e6U)mK_YG9Alc;_u=?SoMP4&+ z{suc0xTp0nV^ZGrg0!8&I%{*I)TyNj>!D9gLCP%eQ09A>^~nKAW)E*BZwNsZpPSp) z=RE}eRkjm$aLTDf%fZuyX|-wlhI52fskJ>uYC+GX)Jxt^>tq9nSV7^*=44Pn4i`e9|l;SNHV0l%Lnu*CLIS67MMfSC7PKMbCG+%_X^C z%Ro;UdA*))ZG45s<BprcZX-)vqT3ErcdUE&@%x%ohm z)2^B@%~Fz*E87u|%c>^)TjAC0)1AIVhu2#5%Cy$^?|ddP4$w&1$QtMUxJ2|gQfZ*R zH1jbhPvyu%^uWxxc@N)(BF48bY>Q=B!kLz{dQWLO5cmwy z#Txq;=R-|t9pneg3XVQLi=uF;EQsg-4(5i?x5l{rGxV+lL0Wq~uMzLQ(t&TWU9Lak zc`nc;a)m~3UY3g_msyh(Y`9Hb9NqL(qT>AgS>v>@Gw8+GSB5vxe3H{Q&I^+dAcsGs z-sB7q`gWN|_!;|*RN9AZ|7bmY_dfE_*h3-RmaG%sZq*n;9VOyQo6;4%6Dv7}MUKoj z;q9|I*v(oRIJ_8D+oSvsPh}yV;Ln;t5up@d-$NF=_0lu49aXRjI*MTX96=}ZIB$EU zW5rH^rin`8v^rLe-M1o559)QinC1qgl>EUanj!do`_|YZeFETIpAYZ9;>+U?iBlCA z;pJa;uu6g6dozU}>fy*MVSf=<%!dBrxjV#3m9swmp`+5WI8VW3!2p(^$?@*o6vnt&w z*BiLI^jvJZi?KDTwwj4Q`~R_lOBa;%^O=uR}+YTaor+uYwC2wmHcXY z(KW2jyKk!4 zE!KvL&o}N4o<><~O@Dcd>$xoqf2c~eJewP}7~^TX;CYJ8xKR&uc^QIa9{e3-gAWDm zJ~4COK2hK%HL*K4s=0g;HzYjBXbyRC_C<}K`{QO!6sG(g8-V|DsyOq~hXIQ&RDHyUboxi(9_)&KPLFaJ8 zFvo}2WK$WJwlkTO?=89CwlK*lnlB^Idl&ccLb}=rf=~=#{%ig zNu;7wvRwyT_LRT@KtMDWSkZ*x0$&V+X52QbHxiu4c$ZR!m2Q1IOWypTp0H>7)X$ma z^jyto>XNRr*t`8S4WDq6>rA!6(wkv{*!`V2-F^z;=#CKLsj*MCXB&iXQLT6Yt4o?w zLj)L&%7T45(e<~gWD46z)9-nmiLDx8XgAY-zuEPT(2R&(qO6BBLPbW-w{LgfLrsFH z`KpV5D$E&Te(h2R?>$Y*>%z*t26^bCVer5#o;u~~_PcrE4AnWWul~p*Je7%kTQH45 zJUfJd3`3g=hqqJ+&YPN$*|L&(EXO&IsTA^Gd=(7d(3Fz8&yqKOsuW-eEH@3ta<1PZ zaTNil%M?vPMMRw6;M|`|vC{3*S<`0-<6VAU4u$qI4kkxI7GJx#5ky!RrRxnN!X@Kc zt~#0~*(^MIc@$mEuMriJ$^ewSj9S)_X7@1j{ld%_hg^oy=UR?a0-)D?rN66zt_QNlT8Brdo2{xC1Aln!3lX>>zz9Iuj+4#VQ5XpESzB|(NSm8!< z{D<7xJcA%TVhKyl;K!nVl_PntN*b?je!~JoI^M>Y$VPK{@lq4RC$@m=yjgF&dHnKa zUqjdD$t2rf!WM z)Y&&X9mIuj2dX4ey7dh;k!eiG#m4GHVypt675a4aO487N)U@V8KR&P#Q1*8lH|T!; zwlJZH?b*|}iFi#~MR!Vz?&5U3BHI%vGYa*W+q0eNX^8t)z8@Lo9&ge|1p%LaH6W>W zrE)aZ3LDf!gBT1>ZDi4dF3u!EWh57z+$O^OuUvZ^10Rg`zZ1v&__DHe)U%QhRI!bi zRpr|jk_fnTaF)g772YezWUp`|GA)1ZN$C4D!IAmgz-j8G5OBKO>)nT$)*Y!vaA-y4 zt-_7(awFEqgsmqpdqq;a#VphkPx>77Es}-m2KPTc%Rk+kj+Brt#hup9m4^)aZ!?t1 zZ<{(ZhX@Hy7kB5%WW2dYO8?fL>>Dci$%*mC2>MBv6!_z{5#BrJnrie%rk8v8Y;byQ z8EOUwIk$7g)eeLz7o5?8?@HB;OV^YO+o`=M13M@_&T5jbN0%1|FX}FZoUk^2su41B zOS$hQc)mJh9lPx0vnMFo`~b$#>yH>%v$&WoZZ%#QarV+b&04jiB$mYc(`MNjIrSj~nhH_3vJeCn2*yrLSl08-^Uq zNG6?Np+^GzE&U2M-iqYk+_d$#^o=;eQ>Hwh_t~OKi*7W1Q2K}}J@|l?(n^L7#mdf5 z2YEnOLNoGt%e0MDfRej$B#L&!G4xur&I;Ja%98&1@4GWzpTyByuGKH>WflSyN@Xdl?+UCzCjQ#Na~ zJXy6ZkiIx$EuYX{kufN}Cw{Wc0}n5^(&4TW?;7gW^F3O9)P{VJjLxrnzdawWFMMXO zr)29*fd(HVZ-u%2^Cr`g)mj#R0Mj_Q3}nJB8hD-K@N!}s;u zD9(F-fWSh}P1p%#z8G&tXp31TGT7e`vto%5XQHdbwyuZ}pOn@H=ioxKZtgSVpp<6Z^w733k@9uGzZCi2({ z8tDs}*`3@5u$=q^OLiVQY(`Bi3Xoj9N!xqcl+(F%hc5b%IQ;mRDrv{Qk`z36(Wt zr)o+TvpRwsrew3_7DIGo{9ZQ^mdRG>0W|51v?(-3*`2Vs;-Q2W)dw85++Ptf53$+8 zhm~-KitQJoHVO`#EP1DWSxfO*HC;A}xaLp=iS1kO2mM!NBrnw`*c?$wZ_bjCvU+|=>rhMJ59FWD^u^c*d!cq*7hzgWwYiYX z$Wh}s5D}GpM5sz6y(YCwA|=c^30*tNX|O5ewShtw(HTYA*86DsVf` zpv%DA*-`ny@8co`Pck19lf(j%JHka{YCF5TjgSr1#Gx!2qMt99F_15_#~r?`OQE~X zoicZpKQgv!=6aXr+X%7K0xb8(Q5^j7g<+AxJ?QZL`u-aw?&0N z?7b)~rlBb7b>W`-8RVLeJyx1TX5&X>V{s9MGdVfrBToc92f8}(Ioz#E-!Y|DGJTej zRw%{GWP+wI#z4`4mq7H=Z3pV0unH*eiRUve-O^DRM|Iy>rzhs(#07mEMpQ!_LI%b? zz2;qHA7Ff`(SoEjLbXpVRt$}|PFn7xJRFRf*@Njq(oc*NSYzm`Xx@B&u~6o&BZxVd z9MlHe_+V4|yLQm%+ap*o<4wNDT=8Q$qydx$yezPd@}_TDf_78PI6XCw6UWZk3uRT} zEnUB)8j(n@9C$O0U93sQ(9-U;y(aW(k-1LGjS>gOY(oiZDWfk;(fORJ+oCrf@2zR} zGPvI0$fnXH3~kKkl)(lTx|>9l2NvDxXX1LbuOvQR7Zl;^GM87w{am#QhxTgns991} z)}hPgb@#@pwrQPO2`6VA(&a}=mvW^Yw#+nf3<-~TaBF5;1!o_j%^d$7QQjejxW`g@-84;>zDwyg|I1L+%i@!2zW`wwNL zqlw-iXy!++3@sX2iPNbzx>sB?wMcua^!20MnOiTeRLy_`B$V>_2`9Lp2108ONVP`T zyBpPLF7~?S+0UQtC2JS=IjYCqKi4zCTb3FRA+oITdCfrQ!)cg{sJc{So$XZpY`+b6 zTHMUB`g2%-LV_2p|HmG!Wi!6q~m*oDBxIN5Q?> zVBRPfgcF2>p)sZ|-p*`ZNVvU=tvkqSi*WN| z^Y&y@g`*J8wkQWSeIK~HJD}?5iDCmZZM^}(A4S+Ak-}^So_=f|wk`;70B{F3yMP}_ z`Y;nMHYCc^6>je>%w}qf@MiPyM8TEV)STh=Zoi=0&V6x>S$M0omm zi`d$GyZE|z2Uufh3;TPxLuhF+$sTZo&yUML6F4}2B)A|wY>`6F4oDXvUr7-cduw-F zAB4R#93_kd4EO_*$6p|Ea{fZ;ulB%(aB=;L^_OVp&iw)HFY!P>La^AN&j}!q5OC`W z@q^exydX%3FvJ7+-Vi5n=L11N>>((SVh?cxS38J11P&3yU9A8zmnUtAPrN`ivX?w za0Gw?5DFj;KsJC!02mp6AMVbcD3?G_AWiOIfTNsT5D#K zczXkqpe)P;frt0oi~w*{kVq3}IKY8mVNpQnLA^0u1g5QjTtFC0?*~VJ#+!hoU;7Dz z`d@Jr02zqI4+nf)D2Tv>G4y|iF=b+U^}kZ4B=!%I|LNrvxIdJs{Ll7c+WMot|H%2X zy+7msXWEPNqYO|7Doim1V0>b#`!kH;{TUF(#9`WuahQ780RLy4A-MnV@Ey~B900)o z@m%{^CMNFBW&Vu&pC}XeA7x_vpJih1f0PL>Kil~;?mw293$XuvhiNAs08BeEbIZS` zzxnSI|BdeN^fBf9Yx>GR+52ym^K1Pv_y3yy%>Ofe!hef?=&$1OFWc~I{lQx4N1t;6 zAHyq5AOB|#)5hl@;U|s_xc~2rbqbK54S*;BWdN8uVHyE^Yz(t7Va(Y0HH?YFg#Ur} zQ$8``_X8ipD@-~j;^#R2F}^SeKwy6M)6Y0V5dGh40~CVc9|QD9mH$ihX)tyA7mfl* zAcnpa)^GF=$bZW){f|y!j3K=K9^4NDNCS-)1j)PrFxYIsUtn*Pw=mbcUrQ#-6MM@P zrGxjTbf?L}W8C+#MRl}^(p8Yh9Dryz_GQs)q2z!kP$H~wC=?RiO&O>0l`oIn)_9~gm-^$S^o9Zx zecOnJ2bE_V+dp{V>j<0_*h=W-Vj)bL7<+33TMiN>?ILh&G_2X{HPEX%3Qo`M#wXC_ z3DE`&B9<3LWnS)x#j-FHiM2Xa9mK*8yUL9(ygS0?LkriH9J^oH-0bJrANd}JkPv+{ z1XE%pXg6Wu6Kd6Cekv4{aJRD^Pxz~31hsw_+I!Dl0BtofK%phh9_YR7ca9=%XDG)y z>GY9<$Zi~+!lVr4cVg*l6;f}fH=d3@*h8?=m9pL=7)*mgUwUL+7?WMoahM&JfUwTH zb#By_**%F?mp^)BWc6;tw=#SUD`Zhkzj}xDJ7(-b3)n?jMbCo%J4=Q`i*x2(`TfeU z@Gs{BLt!@dhdcP=Hz&e=Wg-|cfQeAZwal9eo^3<#zI%1c`XxStS&*<8k@{PWckzoBk@AN1mZatJ)KR-9) z@ZqQt?vRq^)f7pyYy4z}PgZfX6u#BpU)w9-wrnQZtNd`(6u2#L^RPN&QsZ)1PVW;r zVEKqNDcDEG%wO#EXFr@TLPaI&Y>i}d$G=XG*xWyY-<%Mc)2rh|59NvXz9!MSNN{zC zr_T68zAJAQL;9tbcjmzFHxvC?$rE^+N@UfoK_ zT9#KlCY1KbeR||4_pr~0*C?DS(^)VtY%~rDPowP$b;i@&v_jiXvUfBHRpT2@@I2Xs z&U6VAk~=HF;cqWU&{JuZN{YOVXe9VjUb6+;bSiyw&1}OTa!{!{^Qw4dI6F2^XJkr~ zjdPL^j^)*VwP-Pz9Lthj3j1Hg4W2@R8GYD2t#pMP(0K)aR(EUrH+rm80fb)Z+ zRDZks`}#Kr{^r2n9Qd09e{H;6$3r403tm|rg{2nUp>i-W!`!o?Bp<&81sUL%xEdfeG*z7flCD1^*Xw*qyff28+9=nwS%!XEc$dVP1wsNDdYFu?hT`teHrf!<%)+Wt&XN^Pe^5h&g; z!1V`$6RK_LGJrOgq? literal 0 HcmV?d00001 diff --git a/bbot/test/test_step_2/module_tests/test_module_apkpure.py b/bbot/test/test_step_2/module_tests/test_module_apkpure.py index 59a32a66f..17b8e0685 100644 --- a/bbot/test/test_step_2/module_tests/test_module_apkpure.py +++ b/bbot/test/test_step_2/module_tests/test_module_apkpure.py @@ -1,9 +1,10 @@ from pathlib import Path -from .base import ModuleTestBase +from .base import ModuleTestBase, tempapkfile class TestAPKPure(ModuleTestBase): modules_overrides = ["apkpure", "google_playstore", "speculate"] + apk_file = tempapkfile() async def setup_after_prep(self, module_test): await module_test.mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.99"]}}) @@ -35,7 +36,7 @@ async def setup_after_prep(self, module_test): ) module_test.httpx_mock.add_response( url="https://d.apkpure.com/b/XAPK/com.bbot.test?version=latest", - text="apk", + content=self.apk_file, ) def check(self, module_test, events): From 574e88f9ccbdfe5e4b03f28f042be002695a382d Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 16 Oct 2024 00:25:19 +0000 Subject: [PATCH 226/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 159f0a057..55ff7999b 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -13,7 +13,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.82.8", + "version": "3.82.9", "config": "", "only_verified": True, "concurrency": 8, From e81a549e7a45307b47c256bbfef3df78f373e6aa Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 16 Oct 2024 12:08:10 -0400 Subject: [PATCH 227/254] fix queue bug --- bbot/scanner/manager.py | 4 +--- bbot/scanner/scanner.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 3cb1f5fdf..8cbe098a5 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -15,9 +15,7 @@ class ScanIngress(BaseInterceptModule): # accept all events regardless of scope distance scope_distance_modifier = None _name = "_scan_ingress" - - # small queue size so we don't drain modules' outgoing queues - _qsize = 10 + _qsize = -1 @property def priority(self): diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 35cbaf220..97550179a 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -288,7 +288,9 @@ async def _prep(self): self.debug( f"Setting intercept module {intercept_module.name}._incoming_event_queue to previous intercept module {prev_intercept_module.name}.outgoing_event_queue" ) - intercept_module._incoming_event_queue = prev_intercept_module.outgoing_event_queue + interqueue = asyncio.Queue() + intercept_module._incoming_event_queue = interqueue + prev_intercept_module._outgoing_event_queue = interqueue # abort if there are no output modules num_output_modules = len([m for m in self.modules.values() if m._type == "output"]) From 1a3510f7c063b06b4ee96b1e1c43ad771114c525 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 11 Oct 2024 13:47:45 -0400 Subject: [PATCH 228/254] unify api_page_iter subdomain enum modules, paginate shodan dns --- bbot/core/event/base.py | 8 +++- bbot/core/helpers/misc.py | 2 +- bbot/modules/shodan_dns.py | 16 +++---- bbot/modules/templates/subdomain_enum.py | 47 +++++++++++++++++-- bbot/modules/trickest.py | 32 ++----------- bbot/modules/virustotal.py | 18 ++----- bbot/test/test_step_1/test_events.py | 16 ++++++- bbot/test/test_step_1/test_helpers.py | 2 + .../module_tests/test_module_shodan_dns.py | 21 ++++++++- 9 files changed, 101 insertions(+), 61 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 4c7496649..b390f4f1e 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -352,6 +352,12 @@ def port(self): return 80 return self._port + @property + def netloc(self): + if self.host: + return make_netloc(self.host, self.port) + return None + @property def host_stem(self): """ @@ -741,7 +747,7 @@ def json(self, mode="json", siem_friendly=False): """ j = dict() # type, ID, scope description - for i in ("type", "id", "uuid", "scope_description"): + for i in ("type", "id", "uuid", "scope_description", "netloc"): v = getattr(self, i, "") if v: j.update({i: str(v)}) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 9a42732b0..4979453e1 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1289,7 +1289,7 @@ def make_netloc(host, port): if is_ip(host, version=6): host = f"[{host}]" if port is None: - return host + return str(host) return f"{host}:{port}" diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index 9814ccb5c..21140831e 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -16,13 +16,11 @@ class shodan_dns(shodan): base_url = "https://api.shodan.io" - async def request_url(self, query): - url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}" - response = await self.api_request(url) - return response + async def handle_event(self, event): + await self.handle_event_paginated(event) - def parse_results(self, r, query): - json = r.json() - if json: - for hostname in json.get("subdomains", []): - yield f"{hostname}.{query}" + def make_url(self, query): + return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}" + + def parse_results(self, json, query): + return [f"{sub}.{query}" for sub in json.get("subdomains", [])] diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index 95a040b1c..07f249324 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -31,6 +31,11 @@ class subdomain_enum(BaseModule): # "lowest_parent": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com) dedup_strategy = "highest_parent" + # how many results to request per API call + page_size = 100 + # arguments to pass to api_page_iter + api_page_iter_kwargs = {} + @property def source_pretty_name(self): return f"{self.__class__.__name__} API" @@ -61,10 +66,31 @@ async def handle_event(self, event): context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}', ) + async def handle_event_paginated(self, event): + query = self.make_query(event) + async for result_batch in self.query_paginated(query): + for hostname in set(result_batch): + try: + hostname = self.helpers.validators.validate_host(hostname) + except ValueError as e: + self.verbose(e) + continue + if hostname and hostname.endswith(f".{query}") and not hostname == event.data: + await self.emit_event( + hostname, + "DNS_NAME", + event, + abort_if=self.abort_if, + context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}', + ) + async def request_url(self, query): - url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}" + url = self.make_url(query) return await self.api_request(url) + def make_url(self, query): + return f"{self.base_url}/subdomains/{self.helpers.quote(query)}" + def make_query(self, event): query = event.data parents = list(self.helpers.domain_parents(event.data)) @@ -86,13 +112,11 @@ def parse_results(self, r, query=None): for hostname in json: yield hostname - async def query(self, query, parse_fn=None, request_fn=None): + async def query(self, query, parse_fn=None): if parse_fn is None: parse_fn = self.parse_results - if request_fn is None: - request_fn = self.request_url try: - response = await request_fn(query) + response = await self.request_url(query) if response is None: self.info(f'Query "{query}" failed (no response)') return [] @@ -113,6 +137,19 @@ async def query(self, query, parse_fn=None, request_fn=None): except Exception as e: self.info(f"Error retrieving results for {query}: {e}", trace=True) + async def query_paginated(self, query): + url = self.make_url(query) + agen = self.api_page_iter(url, page_size=self.page_size, **self.api_page_iter_kwargs) + try: + async for response in agen: + subdomains = self.parse_results(response, query) + self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"') + if not subdomains: + break + yield subdomains + finally: + agen.aclose() + async def _is_wildcard(self, query): rdtypes = ("A", "AAAA", "CNAME") if self.helpers.is_dns_name(query): diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py index c17aa6160..40f6ea704 100644 --- a/bbot/modules/trickest.py +++ b/bbot/modules/trickest.py @@ -28,39 +28,15 @@ def prepare_api_request(self, url, kwargs): return url, kwargs async def handle_event(self, event): - query = self.make_query(event) - async for result_batch in self.query(query): - for hostname in set(result_batch): - try: - hostname = self.helpers.validators.validate_host(hostname) - except ValueError as e: - self.verbose(e) - continue - if hostname and hostname.endswith(f".{query}") and not hostname == event.data: - await self.emit_event( - hostname, - "DNS_NAME", - event, - abort_if=self.abort_if, - context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}', - ) + await self.handle_event_paginated(event) - async def query(self, query): + def make_url(self, query): url = f"{self.base_url}/view?q=hostname%20~%20%22.{self.helpers.quote(query)}%22" url += f"&dataset_id={self.dataset_id}" url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname" - agen = self.api_page_iter(url, page_size=self.page_size) - try: - async for response in agen: - subdomains = self.parse_results(response) - self.verbose(f'Got {len(subdomains):,} subdomains for "{query}"') - if not subdomains: - break - yield subdomains - finally: - agen.aclose() + return url - def parse_results(self, j): + def parse_results(self, j, query): results = j.get("results", []) subdomains = set() for item in results: diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index 98f469e39..14eec2a9b 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -15,6 +15,10 @@ class virustotal(subdomain_enum_apikey): options_desc = {"api_key": "VirusTotal API Key"} base_url = "https://www.virustotal.com/api/v3" + api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")} + + def make_url(self, query): + return f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains" def prepare_api_request(self, url, kwargs): kwargs["headers"]["x-apikey"] = self.api_key @@ -28,17 +32,3 @@ def parse_results(self, r, query): if match.endswith(query): results.add(match) return results - - async def query(self, query): - results = set() - url = f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains" - agen = self.api_page_iter(url, json=False, next_key=lambda r: r.json().get("links", {}).get("next", "")) - try: - async for response in agen: - r = self.parse_results(response, query) - if not r: - break - results.update(r) - finally: - agen.aclose() - return results diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index f7ab9db2b..e52eda3b4 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -14,10 +14,14 @@ async def test_events(events, helpers): await scan._prep() assert events.ipv4.type == "IP_ADDRESS" + assert events.ipv4.netloc == "8.8.8.8" assert events.ipv6.type == "IP_ADDRESS" + assert events.ipv6.netloc == "[2001:4860:4860::8888]" + assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443" assert events.netv4.type == "IP_RANGE" assert events.netv6.type == "IP_RANGE" assert events.domain.type == "DNS_NAME" + assert events.domain.netloc == "publicapis.org" assert "domain" in events.domain.tags assert events.subdomain.type == "DNS_NAME" assert "subdomain" in events.subdomain.tags @@ -67,8 +71,12 @@ async def test_events(events, helpers): assert not events.netv6 in events.domain assert events.emoji not in events.domain assert events.domain not in events.emoji - assert "evilcorp.com" == scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True) - assert "evilcorp.com" == scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True) + open_port_event = scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True) + dns_event = scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True) + for e in (open_port_event, dns_event): + assert "evilcorp.com" == e + assert e.netloc == "evilcorp.com" + assert e.json()["netloc"] == "evilcorp.com" # url tests assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True) @@ -78,8 +86,10 @@ async def test_events(events, helpers): assert "api.publicapis.org:443" in events.url_unverified assert "publicapis.org" not in events.url_unverified assert events.ipv4_url_unverified in events.ipv4 + assert events.ipv4_url_unverified.netloc == "8.8.8.8:443" assert events.ipv4_url_unverified in events.netv4 assert events.ipv6_url_unverified in events.ipv6 + assert events.ipv6_url_unverified.netloc == "[2001:4860:4860::8888]:443" assert events.ipv6_url_unverified in events.netv6 assert events.emoji not in events.url_unverified assert events.emoji not in events.ipv6_url_unverified @@ -193,6 +203,8 @@ async def test_events(events, helpers): org_stub_1 = scan.make_event("STUB1", "ORG_STUB", parent=scan.root_event) org_stub_1.scope_distance == 1 + assert org_stub_1.netloc == None + assert "netloc" not in org_stub_1.json() org_stub_2 = scan.make_event("STUB2", "ORG_STUB", parent=org_stub_1) org_stub_2.scope_distance == 2 diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 91067324d..956c0e7a5 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -212,6 +212,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): ipv4_netloc = helpers.make_netloc("192.168.1.1", 80) assert ipv4_netloc == "192.168.1.1:80" + assert helpers.make_netloc("192.168.1.1") == "192.168.1.1" + assert helpers.make_netloc(ipaddress.ip_address("192.168.1.1")) == "192.168.1.1" ipv6_netloc = helpers.make_netloc("dead::beef", "443") assert ipv6_netloc == "[dead::beef]:443" diff --git a/bbot/test/test_step_2/module_tests/test_module_shodan_dns.py b/bbot/test/test_step_2/module_tests/test_module_shodan_dns.py index 448158572..373122048 100644 --- a/bbot/test/test_step_2/module_tests/test_module_shodan_dns.py +++ b/bbot/test/test_step_2/module_tests/test_module_shodan_dns.py @@ -9,13 +9,32 @@ async def setup_before_prep(self, module_test): url="https://api.shodan.io/api-info?key=asdf", ) module_test.httpx_mock.add_response( - url="https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf", + url="https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf&page=1", json={ "subdomains": [ "asdf", ], }, ) + module_test.httpx_mock.add_response( + url="https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf&page=2", + json={ + "subdomains": [ + "www", + ], + }, + ) + await module_test.mock_dns( + { + "blacklanternsecurity.com": { + "A": ["127.0.0.11"], + }, + "www.blacklanternsecurity.com": {"A": ["127.0.0.22"]}, + "asdf.blacklanternsecurity.com": {"A": ["127.0.0.33"]}, + } + ) def check(self, module_test, events): + assert len([e for e in events if e.type == "DNS_NAME"]) == 3, "Failed to detect both subdomains" assert any(e.data == "asdf.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" + assert any(e.data == "www.blacklanternsecurity.com" for e in events), "Failed to detect subdomain" From 0d9c3015ab44e7f943d7180e179905232b2ddf7e Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 11 Oct 2024 14:04:11 -0400 Subject: [PATCH 229/254] fix netloc tests --- bbot/test/test_step_1/test_helpers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 956c0e7a5..50af57990 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -212,10 +212,12 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): ipv4_netloc = helpers.make_netloc("192.168.1.1", 80) assert ipv4_netloc == "192.168.1.1:80" - assert helpers.make_netloc("192.168.1.1") == "192.168.1.1" - assert helpers.make_netloc(ipaddress.ip_address("192.168.1.1")) == "192.168.1.1" - ipv6_netloc = helpers.make_netloc("dead::beef", "443") - assert ipv6_netloc == "[dead::beef]:443" + assert helpers.make_netloc("192.168.1.1", None) == "192.168.1.1" + assert helpers.make_netloc(ipaddress.ip_address("192.168.1.1"), None) == "192.168.1.1" + assert helpers.make_netloc("dead::beef", "443") == "[dead::beef]:443" + assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), 443) == "[dead::beef]:443" + assert helpers.make_netloc("dead::beef", None) == "[dead::beef]" + assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), None) == "[dead::beef]" assert helpers.get_file_extension("https://evilcorp.com/evilcorp.com/test/asdf.TXT") == "txt" assert helpers.get_file_extension("/etc/conf/test.tar.gz") == "gz" From 16237cc35735610e6958c06fe18e8c879e10b4e0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 11 Oct 2024 14:24:05 -0400 Subject: [PATCH 230/254] netloc stuff --- bbot/core/helpers/misc.py | 2 +- bbot/test/test_step_1/test_helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 4979453e1..18a41cd53 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1260,7 +1260,7 @@ def gen_numbers(n, padding=2): return results -def make_netloc(host, port): +def make_netloc(host, port=None): """Constructs a network location string from a given host and port. Args: diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 50af57990..6a74d7b70 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -212,7 +212,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): ipv4_netloc = helpers.make_netloc("192.168.1.1", 80) assert ipv4_netloc == "192.168.1.1:80" - assert helpers.make_netloc("192.168.1.1", None) == "192.168.1.1" + assert helpers.make_netloc("192.168.1.1") == "192.168.1.1" assert helpers.make_netloc(ipaddress.ip_address("192.168.1.1"), None) == "192.168.1.1" assert helpers.make_netloc("dead::beef", "443") == "[dead::beef]:443" assert helpers.make_netloc(ipaddress.ip_address("dead::beef"), 443) == "[dead::beef]:443" From c7416c4f302b2b824451f4860a51dc40f4b7e047 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 11 Oct 2024 15:09:46 -0400 Subject: [PATCH 231/254] fix builtwith test --- bbot/modules/templates/subdomain_enum.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index 07f249324..30267cc10 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -112,11 +112,13 @@ def parse_results(self, r, query=None): for hostname in json: yield hostname - async def query(self, query, parse_fn=None): + async def query(self, query, request_fn=None, parse_fn=None): + if request_fn is None: + request_fn = self.request_url if parse_fn is None: parse_fn = self.parse_results try: - response = await self.request_url(query) + response = await request_fn(query) if response is None: self.info(f'Query "{query}" failed (no response)') return [] From 835cbb8ed058020c7a08ef0b5ec101829aa334b3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 11 Oct 2024 17:36:51 -0400 Subject: [PATCH 232/254] add .port field to JSON --- bbot/core/event/base.py | 2 ++ bbot/test/test_step_1/test_events.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index b390f4f1e..8719d53ae 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -766,6 +766,8 @@ def json(self, mode="json", siem_friendly=False): j["host"] = str(self.host) j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts) j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()} + if isinstance(self.port, int): + j["port"] = self.port # web spider distance web_spider_distance = getattr(self, "web_spider_distance", None) if web_spider_distance is not None: diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index e52eda3b4..95fff0e55 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -15,13 +15,16 @@ async def test_events(events, helpers): assert events.ipv4.type == "IP_ADDRESS" assert events.ipv4.netloc == "8.8.8.8" + assert events.ipv4.port is None assert events.ipv6.type == "IP_ADDRESS" assert events.ipv6.netloc == "[2001:4860:4860::8888]" + assert events.ipv6.port is None assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443" assert events.netv4.type == "IP_RANGE" assert events.netv6.type == "IP_RANGE" assert events.domain.type == "DNS_NAME" assert events.domain.netloc == "publicapis.org" + assert events.domain.port is None assert "domain" in events.domain.tags assert events.subdomain.type == "DNS_NAME" assert "subdomain" in events.subdomain.tags @@ -77,6 +80,8 @@ async def test_events(events, helpers): assert "evilcorp.com" == e assert e.netloc == "evilcorp.com" assert e.json()["netloc"] == "evilcorp.com" + assert e.port is None + assert "port" not in e.json() # url tests assert scan.make_event("http://evilcorp.com", dummy=True) == scan.make_event("http://evilcorp.com/", dummy=True) @@ -87,9 +92,13 @@ async def test_events(events, helpers): assert "publicapis.org" not in events.url_unverified assert events.ipv4_url_unverified in events.ipv4 assert events.ipv4_url_unverified.netloc == "8.8.8.8:443" + assert events.ipv4_url_unverified.port == 443 + assert events.ipv4_url_unverified.json()["port"] == 443 assert events.ipv4_url_unverified in events.netv4 assert events.ipv6_url_unverified in events.ipv6 assert events.ipv6_url_unverified.netloc == "[2001:4860:4860::8888]:443" + assert events.ipv6_url_unverified.port == 443 + assert events.ipv6_url_unverified.json()["port"] == 443 assert events.ipv6_url_unverified in events.netv6 assert events.emoji not in events.url_unverified assert events.emoji not in events.ipv6_url_unverified From c0b63679d60c5cb6c63f4326b87b47d8e561ee63 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 11 Oct 2024 19:01:43 -0400 Subject: [PATCH 233/254] open port tweak --- bbot/modules/internal/speculate.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 53d22aede..afb89b149 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -112,15 +112,15 @@ async def handle_event(self, event): speculate_open_ports = self.emit_open_ports and event_in_scope_distance # URL --> OPEN_TCP_PORT - if event.type == "URL" or (event.type == "URL_UNVERIFIED" and self.open_port_consumers): + event_is_url = event.type == "URL" + if event_is_url or (event.type == "URL_UNVERIFIED" and self.open_port_consumers): # only speculate port from a URL if it wouldn't be speculated naturally from the host if event.host and (event.port not in self.ports or not speculate_open_ports): await self.emit_event( self.helpers.make_netloc(event.host, event.port), "OPEN_TCP_PORT", parent=event, - internal=True, - quick=(event.type == "URL"), + internal=not event_is_url, # if the URL is verified, the port is definitely open context=f"speculated {{event.type}} from {event.type}: {{event.data}}", ) @@ -169,7 +169,6 @@ async def handle_event(self, event): "OPEN_TCP_PORT", parent=event, internal=True, - quick=True, context="speculated {event.type}: {event.data}", ) From 856a362d9a1b08f1e4d70d1d20aaef2870fd8518 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 12 Oct 2024 22:14:46 -0400 Subject: [PATCH 234/254] consider .internal attribute in outgoing deduping --- bbot/modules/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 89660edb2..e930a0005 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -984,8 +984,11 @@ def _incoming_dedup_hash(self, event): def _outgoing_dedup_hash(self, event): """ Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True. + + We take into account the `internal` attribute we don't want an internal event (which isn't distributed to output modules) + to inadvertently suppress a non-internal event. """ - return hash((event, self.name)) + return hash(event, self.name, event.internal) def get_per_host_hash(self, event): """ From f63e11238678e36046b9c798e9406cc1428e7f60 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 12 Oct 2024 22:17:55 -0400 Subject: [PATCH 235/254] also consider always_emit --- bbot/modules/base.py | 2 +- bbot/modules/internal/dnsresolve.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index e930a0005..d3ac389f9 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -988,7 +988,7 @@ def _outgoing_dedup_hash(self, event): We take into account the `internal` attribute we don't want an internal event (which isn't distributed to output modules) to inadvertently suppress a non-internal event. """ - return hash(event, self.name, event.internal) + return hash(event, self.name, event.internal, event.always_emit) def get_per_host_hash(self, event): """ diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 53b317d9a..38cea5dd1 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -16,12 +16,6 @@ class HostModule(BaseModule): _name = "host" _type = "internal" - def _outgoing_dedup_hash(self, event): - # this exists to ensure a second, more interesting host isn't passed up - # because its ugly cousin spent its one dedup token before it arrived - # by removing those race conditions, this makes for more consistent results - return hash((event, self.name, event.always_emit)) - @property def module_threads(self): return self.dns_config.get("threads", 25) From bfb6799aaa57875dad6c18de9ac0a6f065d9e199 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 12 Oct 2024 22:20:15 -0400 Subject: [PATCH 236/254] fix hash --- bbot/modules/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index d3ac389f9..956d59c98 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -988,7 +988,7 @@ def _outgoing_dedup_hash(self, event): We take into account the `internal` attribute we don't want an internal event (which isn't distributed to output modules) to inadvertently suppress a non-internal event. """ - return hash(event, self.name, event.internal, event.always_emit) + return hash((event, self.name, event.internal, event.always_emit)) def get_per_host_hash(self, event): """ From dbca7edd278cc269f0d0cf4e87210752d3ade2f2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 12 Oct 2024 23:35:18 -0400 Subject: [PATCH 237/254] allow merging targets --- bbot/scanner/scanner.py | 24 +++++++++++++----------- bbot/test/test_step_1/test_presets.py | 5 +++++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 861ff4d03..34ef29c38 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -130,20 +130,22 @@ def __init__( else: self.id = f"SCAN:{sha1(rand_string(20)).hexdigest()}" - preset = kwargs.pop("preset", None) + custom_preset = kwargs.pop("preset", None) kwargs["_log"] = True from .preset import Preset - if preset is None: - preset = Preset(*targets, **kwargs) - else: - if not isinstance(preset, Preset): - raise ValidationError(f'Preset must be of type Preset, not "{type(preset).__name__}"') - self.preset = preset.bake(self) + base_preset = Preset(*targets, **kwargs) + + if custom_preset is not None: + if not isinstance(custom_preset, Preset): + raise ValidationError(f'Preset must be of type Preset, not "{type(custom_preset).__name__}"') + base_preset.merge(custom_preset) + + self.preset = base_preset.bake(self) # scan name - if preset.scan_name is None: + if self.preset.scan_name is None: tries = 0 while 1: if tries > 5: @@ -158,7 +160,7 @@ def __init__( break tries += 1 else: - scan_name = str(preset.scan_name) + scan_name = str(self.preset.scan_name) self.name = scan_name # make sure the preset has a description @@ -166,8 +168,8 @@ def __init__( self.preset.description = self.name # scan output dir - if preset.output_dir is not None: - self.home = Path(preset.output_dir).resolve() / self.name + if self.preset.output_dir is not None: + self.home = Path(self.preset.output_dir).resolve() / self.name else: self.home = self.preset.bbot_home / "scans" / self.name diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 26c5505b1..0494acd43 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -169,6 +169,11 @@ def test_preset_cache(): def test_preset_scope(): + # test target merging + scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]})) + assert set([str(h) for h in scan.preset.target.seeds.hosts]) == {"1.2.3.4", "evilcorp.com"} + assert set([e.data for e in scan.target]) == {"1.2.3.4", "evilcorp.com"} + blank_preset = Preset() blank_preset = blank_preset.bake() assert not blank_preset.target From 406a58cc133235c1d56af2406eaa141ff85a05b7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 12 Oct 2024 23:53:29 -0400 Subject: [PATCH 238/254] don't make netlocs for ip networks --- bbot/core/event/base.py | 3 ++- bbot/core/helpers/misc.py | 9 ++++++++- bbot/test/test_step_1/test_events.py | 1 + bbot/test/test_step_1/test_helpers.py | 9 +++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 8719d53ae..689cd9f0d 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -25,6 +25,7 @@ is_domain, is_subdomain, is_ip, + is_ip_type, is_ptr, is_uri, url_depth, @@ -354,7 +355,7 @@ def port(self): @property def netloc(self): - if self.host: + if self.host and is_ip_type(self.host, network=False): return make_netloc(self.host, self.port) return None diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 18a41cd53..c493bd4d3 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -621,12 +621,13 @@ def is_ip(d, version=None): return False -def is_ip_type(i): +def is_ip_type(i, network=None): """ Checks if the given object is an instance of an IPv4 or IPv6 type from the ipaddress module. Args: i (ipaddress._BaseV4 or ipaddress._BaseV6): The IP object to check. + network (bool, optional): Whether to restrict the check to network types (IPv4Network or IPv6Network). Defaults to False. Returns: bool: True if the object is an instance of ipaddress._BaseV4 or ipaddress._BaseV6, False otherwise. @@ -639,6 +640,12 @@ def is_ip_type(i): >>> is_ip_type("192.168.1.0/24") False """ + if network is not None: + is_network = ipaddress._BaseNetwork in i.__class__.__mro__ + if network: + return is_network + else: + return not is_network return ipaddress._IPAddressBase in i.__class__.__mro__ diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 95fff0e55..5c1f9677b 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -21,6 +21,7 @@ async def test_events(events, helpers): assert events.ipv6.port is None assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443" assert events.netv4.type == "IP_RANGE" + assert events.netv4.netloc is None assert events.netv6.type == "IP_RANGE" assert events.domain.type == "DNS_NAME" assert events.domain.netloc == "publicapis.org" diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 6a74d7b70..d13f4f0aa 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -94,6 +94,15 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): ] assert helpers.is_ip("127.0.0.1") assert not helpers.is_ip("127.0.0.0.1") + + assert not helpers.is_ip_type("127.0.0.1") + assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1")) + assert not helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=True) + assert helpers.is_ip_type(ipaddress.ip_address("127.0.0.1"), network=False) + assert helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8")) + assert helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=True) + assert not helpers.is_ip_type(ipaddress.ip_network("127.0.0.0/8"), network=False) + assert helpers.is_dns_name("evilcorp.com") assert helpers.is_dns_name("evilcorp") assert not helpers.is_dns_name("evilcorp", include_local=False) From fbb54fdf37fd2eb411bee7dfb7563432b0a124b7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 13 Oct 2024 23:28:18 -0400 Subject: [PATCH 239/254] update scope tests --- .../test_step_1/test_manager_scope_accuracy.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index 1d1f8d3ca..62a03c0ef 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -575,7 +575,7 @@ def custom_setup(scan): }, ) - assert len(events) == 10 + assert len(events) == 12 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0]) @@ -589,9 +589,9 @@ def custom_setup(scan): assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888"]) - assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888"]) - assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"]) assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889"]) assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0]) @@ -603,7 +603,7 @@ def custom_setup(scan): assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.44:8888"]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888"]) - assert len(all_events) == 29 + assert len(all_events) == 31 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110" and e.internal == True and e.scope_distance == 0]) assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0]) @@ -618,8 +618,10 @@ def custom_setup(scan): assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888" and e.internal == True and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == True and e.scope_distance == 0]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888" and e.internal == True and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == True and e.scope_distance == 0]) + assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["url"] == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0]) @@ -660,7 +662,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888" and e.internal == True and e.scope_distance == 1]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 10 + assert len(_graph_output_events) == 12 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0]) @@ -674,9 +676,9 @@ def custom_setup(scan): assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33"]) assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888"]) - assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"]) + assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889"]) assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888"]) - assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"]) + assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889"]) assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889"]) assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0]) From 7d6e25ad1c81d0e638e4fba75e9134e9c2cffcea Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 13 Oct 2024 23:55:19 -0400 Subject: [PATCH 240/254] fixed excavate test --- bbot/modules/internal/excavate.py | 2 ++ bbot/test/test_step_1/test_events.py | 1 + 2 files changed, 3 insertions(+) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index ad34b6efb..3e55aba7a 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -875,6 +875,8 @@ async def setup(self): yara_rules_combined = "\n".join(self.yara_rules_dict.values()) try: self.info(f"Compiling {len(self.yara_rules_dict):,} YARA rules") + for rule_name, rule_content in self.yara_rules_dict.items(): + self.debug(f" - {rule_name}") self.yara_rules = yara.compile(source=yara_rules_combined) except yara.SyntaxError as e: self.debug(yara_rules_combined) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 5c1f9677b..fbee2f915 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -22,6 +22,7 @@ async def test_events(events, helpers): assert events.ipv6_open_port.netloc == "[2001:4860:4860::8888]:443" assert events.netv4.type == "IP_RANGE" assert events.netv4.netloc is None + assert "netloc" not in events.netv4.json() assert events.netv6.type == "IP_RANGE" assert events.domain.type == "DNS_NAME" assert events.domain.netloc == "publicapis.org" From 0f7c2663056388d2e608c7e8d5638b9112ccd4f8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 16 Oct 2024 15:49:03 -0400 Subject: [PATCH 241/254] resolve conflicts --- bbot/test/test_step_1/test_modules_basic.py | 13 +- poetry.lock | 540 ++++++++++---------- pyproject.toml | 2 +- 3 files changed, 281 insertions(+), 274 deletions(-) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index e2f55f8dc..3051f8de1 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -380,15 +380,14 @@ async def handle_event(self, event): scan.modules["dummy"] = dummy(scan) events = [e async for e in scan.async_start()] - assert len(events) == 10 - for e in events: - log.critical(e) + assert len(events) == 11 assert 2 == len([e for e in events if e.type == "SCAN"]) assert 4 == len([e for e in events if e.type == "DNS_NAME"]) # one from target and one from speculate assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.evilcorp.com"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "asdf.evilcorp.com:443"]) assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "evilcorp"]) assert 1 == len([e for e in events if e.type == "FINDING"]) assert 1 == len([e for e in events if e.type == "URL_UNVERIFIED"]) @@ -397,6 +396,7 @@ async def handle_event(self, event): "SCAN": 1, "DNS_NAME": 4, "URL": 1, + "OPEN_TCP_PORT": 1, "ORG_STUB": 1, "URL_UNVERIFIED": 1, "FINDING": 1, @@ -431,16 +431,17 @@ async def handle_event(self, event): assert python_stats.consumed == { "DNS_NAME": 4, "FINDING": 1, + "OPEN_TCP_PORT": 1, "ORG_STUB": 1, "SCAN": 1, "URL": 1, "URL_UNVERIFIED": 1, } - assert python_stats.consumed_total == 9 + assert python_stats.consumed_total == 10 speculate_stats = scan.stats.module_stats["speculate"] - assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1} - assert speculate_stats.produced_total == 3 + assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1, "OPEN_TCP_PORT": 1} + assert speculate_stats.produced_total == 4 assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 3, "URL_UNVERIFIED": 1, "IP_ADDRESS": 3} assert speculate_stats.consumed_total == 8 diff --git a/poetry.lock b/poetry.lock index fc05ed650..d084aded1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -74,13 +74,13 @@ files = [ [[package]] name = "anyio" -version = "4.6.0" +version = "4.6.2.post1" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.9" files = [ - {file = "anyio-4.6.0-py3-none-any.whl", hash = "sha256:c7d2e9d63e31599eeb636c8c5c03a7e108d73b345f064f1c19fdc87b79036a9a"}, - {file = "anyio-4.6.0.tar.gz", hash = "sha256:137b4559cbb034c477165047febb6ff83f390fc3b20bf181c1fc0a728cb8beeb"}, + {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, + {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, ] [package.dependencies] @@ -91,7 +91,7 @@ typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} [package.extras] doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.21.0b1)"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] trio = ["trio (>=0.26.1)"] [[package]] @@ -289,101 +289,116 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.3.2" +version = "3.4.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, - {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, ] [[package]] @@ -430,83 +445,73 @@ files = [ [[package]] name = "coverage" -version = "7.6.1" +version = "7.6.3" description = "Code coverage measurement for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16"}, - {file = "coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36"}, - {file = "coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02"}, - {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"}, - {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23"}, - {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34"}, - {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c"}, - {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959"}, - {file = "coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232"}, - {file = "coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0"}, - {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, - {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, - {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, - {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, - {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, - {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, - {file = "coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778"}, - {file = "coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391"}, - {file = "coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8"}, - {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d"}, - {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca"}, - {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163"}, - {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a"}, - {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d"}, - {file = "coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5"}, - {file = "coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb"}, - {file = "coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106"}, - {file = "coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9"}, - {file = "coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c"}, - {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a"}, - {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060"}, - {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862"}, - {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388"}, - {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155"}, - {file = "coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a"}, - {file = "coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129"}, - {file = "coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e"}, - {file = "coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962"}, - {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb"}, - {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704"}, - {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b"}, - {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f"}, - {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223"}, - {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3"}, - {file = "coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f"}, - {file = "coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657"}, - {file = "coverage-7.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0"}, - {file = "coverage-7.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a"}, - {file = "coverage-7.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b"}, - {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3"}, - {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de"}, - {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6"}, - {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569"}, - {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989"}, - {file = "coverage-7.6.1-cp38-cp38-win32.whl", hash = "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7"}, - {file = "coverage-7.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8"}, - {file = "coverage-7.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255"}, - {file = "coverage-7.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8"}, - {file = "coverage-7.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2"}, - {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a"}, - {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc"}, - {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004"}, - {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb"}, - {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36"}, - {file = "coverage-7.6.1-cp39-cp39-win32.whl", hash = "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c"}, - {file = "coverage-7.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca"}, - {file = "coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df"}, - {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, + {file = "coverage-7.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6da42bbcec130b188169107ecb6ee7bd7b4c849d24c9370a0c884cf728d8e976"}, + {file = "coverage-7.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c222958f59b0ae091f4535851cbb24eb57fc0baea07ba675af718fb5302dddb2"}, + {file = "coverage-7.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab84a8b698ad5a6c365b08061920138e7a7dd9a04b6feb09ba1bfae68346ce6d"}, + {file = "coverage-7.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70a6756ce66cd6fe8486c775b30889f0dc4cb20c157aa8c35b45fd7868255c5c"}, + {file = "coverage-7.6.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c2e6fa98032fec8282f6b27e3f3986c6e05702828380618776ad794e938f53a"}, + {file = "coverage-7.6.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:921fbe13492caf6a69528f09d5d7c7d518c8d0e7b9f6701b7719715f29a71e6e"}, + {file = "coverage-7.6.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6d99198203f0b9cb0b5d1c0393859555bc26b548223a769baf7e321a627ed4fc"}, + {file = "coverage-7.6.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:87cd2e29067ea397a47e352efb13f976eb1b03e18c999270bb50589323294c6e"}, + {file = "coverage-7.6.3-cp310-cp310-win32.whl", hash = "sha256:a3328c3e64ea4ab12b85999eb0779e6139295bbf5485f69d42cf794309e3d007"}, + {file = "coverage-7.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:bca4c8abc50d38f9773c1ec80d43f3768df2e8576807d1656016b9d3eeaa96fd"}, + {file = "coverage-7.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c51ef82302386d686feea1c44dbeef744585da16fcf97deea2a8d6c1556f519b"}, + {file = "coverage-7.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0ca37993206402c6c35dc717f90d4c8f53568a8b80f0bf1a1b2b334f4d488fba"}, + {file = "coverage-7.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c77326300b839c44c3e5a8fe26c15b7e87b2f32dfd2fc9fee1d13604347c9b38"}, + {file = "coverage-7.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e484e479860e00da1f005cd19d1c5d4a813324e5951319ac3f3eefb497cc549"}, + {file = "coverage-7.6.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c6c0f4d53ef603397fc894a895b960ecd7d44c727df42a8d500031716d4e8d2"}, + {file = "coverage-7.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:37be7b5ea3ff5b7c4a9db16074dc94523b5f10dd1f3b362a827af66a55198175"}, + {file = "coverage-7.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:43b32a06c47539fe275106b376658638b418c7cfdfff0e0259fbf877e845f14b"}, + {file = "coverage-7.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ee77c7bef0724165e795b6b7bf9c4c22a9b8468a6bdb9c6b4281293c6b22a90f"}, + {file = "coverage-7.6.3-cp311-cp311-win32.whl", hash = "sha256:43517e1f6b19f610a93d8227e47790722c8bf7422e46b365e0469fc3d3563d97"}, + {file = "coverage-7.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:04f2189716e85ec9192df307f7c255f90e78b6e9863a03223c3b998d24a3c6c6"}, + {file = "coverage-7.6.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:27bd5f18d8f2879e45724b0ce74f61811639a846ff0e5c0395b7818fae87aec6"}, + {file = "coverage-7.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d546cfa78844b8b9c1c0533de1851569a13f87449897bbc95d698d1d3cb2a30f"}, + {file = "coverage-7.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9975442f2e7a5cfcf87299c26b5a45266ab0696348420049b9b94b2ad3d40234"}, + {file = "coverage-7.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:583049c63106c0555e3ae3931edab5669668bbef84c15861421b94e121878d3f"}, + {file = "coverage-7.6.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2341a78ae3a5ed454d524206a3fcb3cec408c2a0c7c2752cd78b606a2ff15af4"}, + {file = "coverage-7.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a4fb91d5f72b7e06a14ff4ae5be625a81cd7e5f869d7a54578fc271d08d58ae3"}, + {file = "coverage-7.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e279f3db904e3b55f520f11f983cc8dc8a4ce9b65f11692d4718ed021ec58b83"}, + {file = "coverage-7.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa23ce39661a3e90eea5f99ec59b763b7d655c2cada10729ed920a38bfc2b167"}, + {file = "coverage-7.6.3-cp312-cp312-win32.whl", hash = "sha256:52ac29cc72ee7e25ace7807249638f94c9b6a862c56b1df015d2b2e388e51dbd"}, + {file = "coverage-7.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:40e8b1983080439d4802d80b951f4a93d991ef3261f69e81095a66f86cf3c3c6"}, + {file = "coverage-7.6.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9134032f5aa445ae591c2ba6991d10136a1f533b1d2fa8f8c21126468c5025c6"}, + {file = "coverage-7.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:99670790f21a96665a35849990b1df447993880bb6463a0a1d757897f30da929"}, + {file = "coverage-7.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dc7d6b380ca76f5e817ac9eef0c3686e7834c8346bef30b041a4ad286449990"}, + {file = "coverage-7.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7b26757b22faf88fcf232f5f0e62f6e0fd9e22a8a5d0d5016888cdfe1f6c1c4"}, + {file = "coverage-7.6.3-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c59d6a4a4633fad297f943c03d0d2569867bd5372eb5684befdff8df8522e39"}, + {file = "coverage-7.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f263b18692f8ed52c8de7f40a0751e79015983dbd77b16906e5b310a39d3ca21"}, + {file = "coverage-7.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:79644f68a6ff23b251cae1c82b01a0b51bc40c8468ca9585c6c4b1aeee570e0b"}, + {file = "coverage-7.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:71967c35828c9ff94e8c7d405469a1fb68257f686bca7c1ed85ed34e7c2529c4"}, + {file = "coverage-7.6.3-cp313-cp313-win32.whl", hash = "sha256:e266af4da2c1a4cbc6135a570c64577fd3e6eb204607eaff99d8e9b710003c6f"}, + {file = "coverage-7.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:ea52bd218d4ba260399a8ae4bb6b577d82adfc4518b93566ce1fddd4a49d1dce"}, + {file = "coverage-7.6.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8d4c6ea0f498c7c79111033a290d060c517853a7bcb2f46516f591dab628ddd3"}, + {file = "coverage-7.6.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:331b200ad03dbaa44151d74daeb7da2cf382db424ab923574f6ecca7d3b30de3"}, + {file = "coverage-7.6.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54356a76b67cf8a3085818026bb556545ebb8353951923b88292556dfa9f812d"}, + {file = "coverage-7.6.3-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ebec65f5068e7df2d49466aab9128510c4867e532e07cb6960075b27658dca38"}, + {file = "coverage-7.6.3-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d33a785ea8354c480515e781554d3be582a86297e41ccbea627a5c632647f2cd"}, + {file = "coverage-7.6.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f7ddb920106bbbbcaf2a274d56f46956bf56ecbde210d88061824a95bdd94e92"}, + {file = "coverage-7.6.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:70d24936ca6c15a3bbc91ee9c7fc661132c6f4c9d42a23b31b6686c05073bde5"}, + {file = "coverage-7.6.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c30e42ea11badb147f0d2e387115b15e2bd8205a5ad70d6ad79cf37f6ac08c91"}, + {file = "coverage-7.6.3-cp313-cp313t-win32.whl", hash = "sha256:365defc257c687ce3e7d275f39738dcd230777424117a6c76043459db131dd43"}, + {file = "coverage-7.6.3-cp313-cp313t-win_amd64.whl", hash = "sha256:23bb63ae3f4c645d2d82fa22697364b0046fbafb6261b258a58587441c5f7bd0"}, + {file = "coverage-7.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:da29ceabe3025a1e5a5aeeb331c5b1af686daab4ff0fb4f83df18b1180ea83e2"}, + {file = "coverage-7.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:df8c05a0f574d480947cba11b947dc41b1265d721c3777881da2fb8d3a1ddfba"}, + {file = "coverage-7.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec1e3b40b82236d100d259854840555469fad4db64f669ab817279eb95cd535c"}, + {file = "coverage-7.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4adeb878a374126f1e5cf03b87f66279f479e01af0e9a654cf6d1509af46c40"}, + {file = "coverage-7.6.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43d6a66e33b1455b98fc7312b124296dad97a2e191c80320587234a77b1b736e"}, + {file = "coverage-7.6.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1990b1f4e2c402beb317840030bb9f1b6a363f86e14e21b4212e618acdfce7f6"}, + {file = "coverage-7.6.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:12f9515d875859faedb4144fd38694a761cd2a61ef9603bf887b13956d0bbfbb"}, + {file = "coverage-7.6.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:99ded130555c021d99729fabd4ddb91a6f4cc0707df4b1daf912c7850c373b13"}, + {file = "coverage-7.6.3-cp39-cp39-win32.whl", hash = "sha256:c3a79f56dee9136084cf84a6c7c4341427ef36e05ae6415bf7d787c96ff5eaa3"}, + {file = "coverage-7.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:aac7501ae73d4a02f4b7ac8fcb9dc55342ca98ffb9ed9f2dfb8a25d53eda0e4d"}, + {file = "coverage-7.6.3-pp39.pp310-none-any.whl", hash = "sha256:b9853509b4bf57ba7b1f99b9d866c422c9c5248799ab20e652bbb8a184a38181"}, + {file = "coverage-7.6.3.tar.gz", hash = "sha256:bb7d5fe92bd0dc235f63ebe9f8c6e0884f7360f88f3411bfed1350c872ef2054"}, ] [package.dependencies] @@ -584,13 +589,13 @@ optimize = ["orjson"] [[package]] name = "distlib" -version = "0.3.8" +version = "0.3.9" description = "Distribution utilities" optional = false python-versions = "*" files = [ - {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, - {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, + {file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"}, + {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"}, ] [[package]] @@ -703,13 +708,13 @@ dev = ["flake8", "markdown", "twine", "wheel"] [[package]] name = "griffe" -version = "1.3.2" +version = "1.4.1" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "griffe-1.3.2-py3-none-any.whl", hash = "sha256:2e34b5e46507d615915c8e6288bb1a2234bd35dee44d01e40a2bc2f25bd4d10c"}, - {file = "griffe-1.3.2.tar.gz", hash = "sha256:1ec50335aa507ed2445f2dd45a15c9fa3a45f52c9527e880571dfc61912fd60c"}, + {file = "griffe-1.4.1-py3-none-any.whl", hash = "sha256:84295ee0b27743bd880aea75632830ef02ded65d16124025e4c263bb826ab645"}, + {file = "griffe-1.4.1.tar.gz", hash = "sha256:911a201b01dc92e08c0e84c38a301e9da5ec067f00e7d9f2e39bc24dbfa3c176"}, ] [package.dependencies] @@ -728,13 +733,13 @@ files = [ [[package]] name = "httpcore" -version = "1.0.5" +version = "1.0.6" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, - {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, + {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"}, + {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"}, ] [package.dependencies] @@ -745,7 +750,7 @@ h11 = ">=0.13,<0.15" asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<0.26.0)"] +trio = ["trio (>=0.22.0,<1.0)"] [[package]] name = "httpx" @@ -1076,71 +1081,72 @@ testing = ["coverage", "pyyaml"] [[package]] name = "markupsafe" -version = "2.1.5" +version = "3.0.1" description = "Safely add untrusted strings to HTML/XML markup." optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:db842712984e91707437461930e6011e60b39136c7331e971952bb30465bc1a1"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ffb4a8e7d46ed96ae48805746755fadd0909fea2306f93d5d8233ba23dda12a"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67c519635a4f64e495c50e3107d9b4075aec33634272b5db1cde839e07367589"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48488d999ed50ba8d38c581d67e496f955821dc183883550a6fbc7f1aefdc170"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f31ae06f1328595d762c9a2bf29dafd8621c7d3adc130cbb46278079758779ca"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80fcbf3add8790caddfab6764bde258b5d09aefbe9169c183f88a7410f0f6dea"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3341c043c37d78cc5ae6e3e305e988532b072329639007fd408a476642a89fd6"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cb53e2a99df28eee3b5f4fea166020d3ef9116fdc5764bc5117486e6d1211b25"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-win32.whl", hash = "sha256:db15ce28e1e127a0013dfb8ac243a8e392db8c61eae113337536edb28bdc1f97"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:4ffaaac913c3f7345579db4f33b0020db693f302ca5137f106060316761beea9"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-win32.whl", hash = "sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8ae369e84466aa70f3154ee23c1451fda10a8ee1b63923ce76667e3077f2b0c4"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40f1e10d51c92859765522cbd79c5c8989f40f0419614bcdc5015e7b6bf97fc5"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a4cb365cb49b750bdb60b846b0c0bc49ed62e59a76635095a179d440540c346"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee3941769bd2522fe39222206f6dd97ae83c442a94c90f2b7a25d847d40f4729"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62fada2c942702ef8952754abfc1a9f7658a4d5460fabe95ac7ec2cbe0d02abc"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c2d64fdba74ad16138300815cfdc6ab2f4647e23ced81f59e940d7d4a1469d9"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fb532dd9900381d2e8f48172ddc5a59db4c445a11b9fab40b3b786da40d3b56b"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0f84af7e813784feb4d5e4ff7db633aba6c8ca64a833f61d8e4eade234ef0c38"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-win32.whl", hash = "sha256:cbf445eb5628981a80f54087f9acdbf84f9b7d862756110d172993b9a5ae81aa"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:a10860e00ded1dd0a65b83e717af28845bb7bd16d8ace40fe5531491de76b79f"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e81c52638315ff4ac1b533d427f50bc0afc746deb949210bc85f05d4f15fd772"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:312387403cd40699ab91d50735ea7a507b788091c416dd007eac54434aee51da"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ae99f31f47d849758a687102afdd05bd3d3ff7dbab0a8f1587981b58a76152a"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c97ff7fedf56d86bae92fa0a646ce1a0ec7509a7578e1ed238731ba13aabcd1c"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7420ceda262dbb4b8d839a4ec63d61c261e4e77677ed7c66c99f4e7cb5030dd"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45d42d132cff577c92bfba536aefcfea7e26efb975bd455db4e6602f5c9f45e7"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4c8817557d0de9349109acb38b9dd570b03cc5014e8aabf1cbddc6e81005becd"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a54c43d3ec4cf2a39f4387ad044221c66a376e58c0d0e971d47c475ba79c6b5"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-win32.whl", hash = "sha256:c91b394f7601438ff79a4b93d16be92f216adb57d813a78be4446fe0f6bc2d8c"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:fe32482b37b4b00c7a52a07211b479653b7fe4f22b2e481b9a9b099d8a430f2f"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:17b2aea42a7280db02ac644db1d634ad47dcc96faf38ab304fe26ba2680d359a"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:852dc840f6d7c985603e60b5deaae1d89c56cb038b577f6b5b8c808c97580f1d"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0778de17cff1acaeccc3ff30cd99a3fd5c50fc58ad3d6c0e0c4c58092b859396"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:800100d45176652ded796134277ecb13640c1a537cad3b8b53da45aa96330453"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d06b24c686a34c86c8c1fba923181eae6b10565e4d80bdd7bc1c8e2f11247aa4"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:33d1c36b90e570ba7785dacd1faaf091203d9942bc036118fab8110a401eb1a8"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:beeebf760a9c1f4c07ef6a53465e8cfa776ea6a2021eda0d0417ec41043fe984"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:bbde71a705f8e9e4c3e9e33db69341d040c827c7afa6789b14c6e16776074f5a"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-win32.whl", hash = "sha256:82b5dba6eb1bcc29cc305a18a3c5365d2af06ee71b123216416f7e20d2a84e5b"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:730d86af59e0e43ce277bb83970530dd223bf7f2a838e086b50affa6ec5f9295"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4935dd7883f1d50e2ffecca0aa33dc1946a94c8f3fdafb8df5c330e48f71b132"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e9393357f19954248b00bed7c56f29a25c930593a77630c719653d51e7669c2a"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40621d60d0e58aa573b68ac5e2d6b20d44392878e0bfc159012a5787c4e35bc8"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f94190df587738280d544971500b9cafc9b950d32efcb1fba9ac10d84e6aa4e6"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6a387d61fe41cdf7ea95b38e9af11cfb1a63499af2759444b99185c4ab33f5b"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8ad4ad1429cd4f315f32ef263c1342166695fad76c100c5d979c45d5570ed58b"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e24bfe89c6ac4c31792793ad9f861b8f6dc4546ac6dc8f1c9083c7c4f2b335cd"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2a4b34a8d14649315c4bc26bbfa352663eb51d146e35eef231dd739d54a5430a"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-win32.whl", hash = "sha256:242d6860f1fd9191aef5fae22b51c5c19767f93fb9ead4d21924e0bcb17619d8"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:93e8248d650e7e9d49e8251f883eed60ecbc0e8ffd6349e18550925e31bd029b"}, + {file = "markupsafe-3.0.1.tar.gz", hash = "sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344"}, ] [[package]] @@ -1272,13 +1278,13 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.40" +version = "9.5.41" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.40-py3-none-any.whl", hash = "sha256:8e7a16ada34e79a7b6459ff2602584222f522c738b6a023d1bea853d5049da6f"}, - {file = "mkdocs_material-9.5.40.tar.gz", hash = "sha256:b69d70e667ec51fc41f65e006a3184dd00d95b2439d982cb1586e4c018943156"}, + {file = "mkdocs_material-9.5.41-py3-none-any.whl", hash = "sha256:990bc138c33342b5b73e7545915ebc0136e501bfbd8e365735144f5120891d83"}, + {file = "mkdocs_material-9.5.41.tar.gz", hash = "sha256:30fa5d459b4b8130848ecd8e1c908878345d9d8268f7ddbc31eebe88d462d97b"}, ] [package.dependencies] @@ -1340,13 +1346,13 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] [[package]] name = "mkdocstrings-python" -version = "1.12.0" +version = "1.12.1" description = "A Python handler for mkdocstrings." optional = false python-versions = ">=3.9" files = [ - {file = "mkdocstrings_python-1.12.0-py3-none-any.whl", hash = "sha256:1f48c9ea6d1d6cd1fefc7389f003841e9c65e50016ba40342f340ca901bc60b9"}, - {file = "mkdocstrings_python-1.12.0.tar.gz", hash = "sha256:2121671354fff208fff1278ce9c961aee2b736a1688f70064c4fa76a00241b34"}, + {file = "mkdocstrings_python-1.12.1-py3-none-any.whl", hash = "sha256:205244488199c9aa2a39787ad6a0c862d39b74078ea9aa2be817bc972399563f"}, + {file = "mkdocstrings_python-1.12.1.tar.gz", hash = "sha256:60d6a5ca912c9af4ad431db6d0111ce9f79c6c48d33377dde6a05a8f5f48d792"}, ] [package.dependencies] @@ -1907,13 +1913,13 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pymdown-extensions" -version = "10.11.1" +version = "10.11.2" description = "Extension pack for Python Markdown." optional = false python-versions = ">=3.8" files = [ - {file = "pymdown_extensions-10.11.1-py3-none-any.whl", hash = "sha256:a2b28f5786e041f19cb5bb30a1c2c853668a7099da8e3dd822a5ad05f2e855e3"}, - {file = "pymdown_extensions-10.11.1.tar.gz", hash = "sha256:a8836e955851542fa2625d04d59fdf97125ca001377478ed5618e04f9183a59a"}, + {file = "pymdown_extensions-10.11.2-py3-none-any.whl", hash = "sha256:41cdde0a77290e480cf53892f5c5e50921a7ee3e5cd60ba91bf19837b33badcf"}, + {file = "pymdown_extensions-10.11.2.tar.gz", hash = "sha256:bc8847ecc9e784a098efd35e20cba772bc5a1b529dfcef9dc1972db9021a1049"}, ] [package.dependencies] @@ -1925,13 +1931,13 @@ extra = ["pygments (>=2.12)"] [[package]] name = "pyparsing" -version = "3.1.4" +version = "3.2.0" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false -python-versions = ">=3.6.8" +python-versions = ">=3.9" files = [ - {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, - {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, + {file = "pyparsing-3.2.0-py3-none-any.whl", hash = "sha256:93d9577b88da0bbea8cc8334ee8b918ed014968fd2ec383e868fb8afb1ccef84"}, + {file = "pyparsing-3.2.0.tar.gz", hash = "sha256:cbf74e27246d595d9a74b186b810f6fbb86726dbf3b9532efb343f6d7294fe9c"}, ] [package.extras] @@ -1961,17 +1967,17 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments [[package]] name = "pytest-asyncio" -version = "0.24.0" +version = "0.23.8" description = "Pytest support for asyncio" optional = false python-versions = ">=3.8" files = [ - {file = "pytest_asyncio-0.24.0-py3-none-any.whl", hash = "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b"}, - {file = "pytest_asyncio-0.24.0.tar.gz", hash = "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276"}, + {file = "pytest_asyncio-0.23.8-py3-none-any.whl", hash = "sha256:50265d892689a5faefb84df80819d1ecef566eb3549cf915dfb33569359d1ce2"}, + {file = "pytest_asyncio-0.23.8.tar.gz", hash = "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3"}, ] [package.dependencies] -pytest = ">=8.2,<9" +pytest = ">=7.0.0,<9" [package.extras] docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] @@ -2573,13 +2579,13 @@ test = ["pytest"] [[package]] name = "setuptools" -version = "75.1.0" +version = "75.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-75.1.0-py3-none-any.whl", hash = "sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2"}, - {file = "setuptools-75.1.0.tar.gz", hash = "sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538"}, + {file = "setuptools-75.2.0-py3-none-any.whl", hash = "sha256:a7fcb66f68b4d9e8e66b42f9876150a3371558f98fa32222ffaa5bced76406f8"}, + {file = "setuptools-75.2.0.tar.gz", hash = "sha256:753bb6ebf1f465a1912e19ed1d41f403a79173a9acf66a42e7e6aec45c3c16ec"}, ] [package.extras] @@ -2672,13 +2678,13 @@ testing = ["black", "mypy", "pytest", "pytest-gitignore", "pytest-mock", "respon [[package]] name = "tomli" -version = "2.0.1" +version = "2.0.2" description = "A lil' TOML parser" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] [[package]] @@ -3071,4 +3077,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "c96e967a9cebef670770ca025ed60ea5124f64e2bf2833b4c5ab215fe582a2ae" +content-hash = "c25df7cf571a94b66f847340ccfcf4f06131e97df02562f7f33ba8346a28d4a1" diff --git a/pyproject.toml b/pyproject.toml index 9a1f1f9f1..666bf2982 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ pytest-timeout = "^2.3.1" pytest-httpx = "^0.30.0" pytest-httpserver = "^1.0.11" pytest = "^8.3.1" -pytest-asyncio = ">=0.23.8,<0.25.0" +pytest-asyncio = "=0.23.8" [tool.poetry.group.docs.dependencies] mkdocs = "^1.5.2" From d7478cdadae14bbb37bf511e1aabd4a55ea44a15 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 16 Oct 2024 16:03:44 -0400 Subject: [PATCH 242/254] remove critical debug --- bbot/modules/internal/excavate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index ad34b6efb..dd1099562 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -791,7 +791,6 @@ class LoginPageExtractor(ExcavateRule): } async def process(self, yara_results, event, yara_rule_settings, discovery_context): - self.excavate.critical(f"Login page detected: {event.data['url']}") if yara_results: event.add_tag("login-page") From c16fd46855bd2aa9e1c78986c07e982f4d0ed9e1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 17 Oct 2024 10:35:41 -0400 Subject: [PATCH 243/254] comment out clobbering API keys --- bbot/presets/subdomain-enum.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bbot/presets/subdomain-enum.yml b/bbot/presets/subdomain-enum.yml index fc4ae4aa2..d4230c3fc 100644 --- a/bbot/presets/subdomain-enum.yml +++ b/bbot/presets/subdomain-enum.yml @@ -13,10 +13,10 @@ config: threads: 25 brute_threads: 1000 # put your API keys here - modules: - github: - api_key: "" - chaos: - api_key: "" - securitytrails: - api_key: "" + # modules: + # github: + # api_key: "" + # chaos: + # api_key: "" + # securitytrails: + # api_key: "" From 27c819ea6872df70eb1543bae9f0c39ffc149498 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 17 Oct 2024 10:45:30 -0400 Subject: [PATCH 244/254] fixing but with dnn installwizard detector --- bbot/modules/dotnetnuke.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py index 2207600e2..d36b4a014 100644 --- a/bbot/modules/dotnetnuke.py +++ b/bbot/modules/dotnetnuke.py @@ -155,24 +155,25 @@ async def handle_event(self, event): # InstallWizard SuperUser Privilege Escalation result = await self.helpers.request(f'{event.data["url"]}/Install/InstallWizard.aspx') - if result.status_code == 200: - result_confirm = await self.helpers.request( - f'{event.data["url"]}/Install/InstallWizard.aspx?__viewstate=1' - ) - if result_confirm.status_code == 500: - description = "DotNetNuke InstallWizard SuperUser Privilege Escalation" - await self.emit_event( - { - "severity": "CRITICAL", - "description": description, - "host": str(event.host), - "url": f'{event.data["url"]}/Install/InstallWizard.aspx', - }, - "VULNERABILITY", - event, - context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}', + if result: + if result.status_code == 200: + result_confirm = await self.helpers.request( + f'{event.data["url"]}/Install/InstallWizard.aspx?__viewstate=1' ) - return + if result_confirm.status_code == 500: + description = "DotNetNuke InstallWizard SuperUser Privilege Escalation" + await self.emit_event( + { + "severity": "CRITICAL", + "description": description, + "host": str(event.host), + "url": f'{event.data["url"]}/Install/InstallWizard.aspx', + }, + "VULNERABILITY", + event, + context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}', + ) + return # DNNImageHandler.ashx Blind SSRF self.event_dict[event.data["url"]] = event From 3f1e5745b9bb0dcc568566abcd32d1e545895783 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 17 Oct 2024 12:34:17 -0400 Subject: [PATCH 245/254] better handling of custom secrets files --- bbot/modules/badsecrets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 4f82736fa..0295bfc2c 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -23,12 +23,13 @@ async def setup(self): self.custom_secrets = None custom_secrets = self.config.get("custom_secrets", None) if custom_secrets: - if Path(custom_secrets).is_file(): + secrets_path = Path(custom_secrets).expanduser() + if secrets_path.is_file(): self.custom_secrets = custom_secrets self.info(f"Successfully loaded secrets file [{custom_secrets}]") else: self.warning(f"custom secrets file [{custom_secrets}] is not valid") - return None, "Custom secrets file not valid" + return False, "Custom secrets file not valid" return True @property From 918f3b799081de3b72fa933e72e49f250e64ea6f Mon Sep 17 00:00:00 2001 From: GitHub Date: Fri, 18 Oct 2024 00:25:09 +0000 Subject: [PATCH 246/254] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 55ff7999b..e185e265e 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -13,7 +13,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.82.9", + "version": "3.82.11", "config": "", "only_verified": True, "concurrency": 8, From 7d6a57e179e7da062f333ae2cef923e688572f0d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 18 Oct 2024 00:40:11 -0400 Subject: [PATCH 247/254] handle bad chars in matched data --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index dd1099562..b60f1192b 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -154,7 +154,7 @@ async def preprocess(self, r, event, discovery_context): yara_rule_settings = YaraRuleSettings(description, tags, emit_match) yara_results = {} for h in r.strings: - yara_results[h.identifier.lstrip("$")] = sorted(set([i.matched_data.decode("utf-8") for i in h.instances])) + yara_results[h.identifier.lstrip("$")] = sorted(set([i.matched_data.decode("utf-8", errors="ignore") for i in h.instances])) await self.process(yara_results, event, yara_rule_settings, discovery_context) async def process(self, yara_results, event, yara_rule_settings, discovery_context): From 7595ff70bfc3fa137e75bfcbd3dba9cc5c20847d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 18 Oct 2024 10:00:08 -0400 Subject: [PATCH 248/254] black --- bbot/modules/internal/excavate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index b60f1192b..6651b0b42 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -154,7 +154,9 @@ async def preprocess(self, r, event, discovery_context): yara_rule_settings = YaraRuleSettings(description, tags, emit_match) yara_results = {} for h in r.strings: - yara_results[h.identifier.lstrip("$")] = sorted(set([i.matched_data.decode("utf-8", errors="ignore") for i in h.instances])) + yara_results[h.identifier.lstrip("$")] = sorted( + set([i.matched_data.decode("utf-8", errors="ignore") for i in h.instances]) + ) await self.process(yara_results, event, yara_rule_settings, discovery_context) async def process(self, yara_results, event, yara_rule_settings, discovery_context): From 4bd16b79d041dcd4998ef5432df32f42ffc2f65c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 18 Oct 2024 11:12:00 -0400 Subject: [PATCH 249/254] fixing bugs with generic_ssrf --- bbot/modules/generic_ssrf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index 0aa61a3e5..c6bd38544 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -137,10 +137,10 @@ async def test(self, event): post_body = f""" - + ]> &{rand_entity};""" - test_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/" + test_url = event.parsed_url.geturl() r = await self.parent_module.helpers.curl( url=test_url, method="POST", raw_body=post_body, headers={"Content-type": "application/xml"} ) From 481bd3598ffbae398833cb6eb2d0e89a8da0d463 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 12:38:28 -0400 Subject: [PATCH 250/254] fix preset bug --- bbot/scanner/preset/preset.py | 2 -- bbot/test/test_step_1/test_presets.py | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 74d4ab48a..a8fe471b4 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -844,8 +844,6 @@ def _is_valid_module(self, module, module_type, name_only=False, raise_error=Tru if module in self.exclude_modules: reason = "the module has been excluded" - if raise_error: - raise ValidationError(f'Unable to add {module_type} module "{module}" because {reason}') return False, reason, {} module_flags = preloaded.get("flags", []) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 26c5505b1..644ee3638 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -527,9 +527,9 @@ def test_preset_module_resolution(clean_default_config): assert set(preset.scan_modules) == {"wayback"} # modules + module exclusions - with pytest.raises(ValidationError) as error: - preset = Preset(exclude_modules=["sslcert"], modules=["sslcert", "wappalyzer", "wayback"]).bake() - assert str(error.value) == 'Unable to add scan module "sslcert" because the module has been excluded' + preset = Preset(exclude_modules=["sslcert"], modules=["sslcert", "wappalyzer", "wayback"]).bake() + baked_preset = preset.bake() + assert baked_preset.modules == {'wayback', 'cloudcheck', 'python', 'json', 'speculate', 'dnsresolve', 'aggregate', 'excavate', 'txt', 'httpx', 'csv', 'wappalyzer'} @pytest.mark.asyncio From cc83d6c5476fe1e2a53975adf632eee6582e36ab Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 12:38:41 -0400 Subject: [PATCH 251/254] blacked --- bbot/test/test_step_1/test_presets.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 644ee3638..481eaf1b3 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -529,7 +529,20 @@ def test_preset_module_resolution(clean_default_config): # modules + module exclusions preset = Preset(exclude_modules=["sslcert"], modules=["sslcert", "wappalyzer", "wayback"]).bake() baked_preset = preset.bake() - assert baked_preset.modules == {'wayback', 'cloudcheck', 'python', 'json', 'speculate', 'dnsresolve', 'aggregate', 'excavate', 'txt', 'httpx', 'csv', 'wappalyzer'} + assert baked_preset.modules == { + "wayback", + "cloudcheck", + "python", + "json", + "speculate", + "dnsresolve", + "aggregate", + "excavate", + "txt", + "httpx", + "csv", + "wappalyzer", + } @pytest.mark.asyncio From c6f445bd29faa56bfe0e5b7ef569a1fb1bb06114 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 12:49:15 -0400 Subject: [PATCH 252/254] fix tests --- bbot/test/test_step_1/test_cli.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index bf7476d49..f34b7c147 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -351,14 +351,6 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config): result = await cli._main() assert result == True - # enable and exclude the same module - caplog.clear() - assert not caplog.text - monkeypatch.setattr("sys.argv", ["bbot", "-m", "ffuf_shortnames", "-em", "ffuf_shortnames"]) - result = await cli._main() - assert result == None - assert 'Unable to add scan module "ffuf_shortnames" because the module has been excluded' in caplog.text - # require flags monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "-rf", "passive"]) result = await cli._main() From 80a65a189624f39f028e069c0a20e7aa248a2101 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 12:51:41 -0400 Subject: [PATCH 253/254] rebase dev --- .github/workflows/docs_updater.yml | 40 ++++ ...rsion_updater.yaml => version_updater.yml} | 0 bbot/core/event/base.py | 14 +- bbot/core/helpers/regexes.py | 3 + bbot/test/test_step_1/test_events.py | 16 +- docs/scanning/events.md | 95 ++++++--- poetry.lock | 198 +++++++++--------- pyproject.toml | 2 +- 8 files changed, 233 insertions(+), 135 deletions(-) create mode 100644 .github/workflows/docs_updater.yml rename .github/workflows/{version_updater.yaml => version_updater.yml} (100%) diff --git a/.github/workflows/docs_updater.yml b/.github/workflows/docs_updater.yml new file mode 100644 index 000000000..fdc585718 --- /dev/null +++ b/.github/workflows/docs_updater.yml @@ -0,0 +1,40 @@ +name: Daily Docs Update + +on: + schedule: + - cron: '0 0 * * *' # Runs daily at midnight UTC + +jobs: + update_docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} + ref: dev # Checkout the dev branch + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + pip install poetry + poetry install + - name: Generate docs + run: | + poetry run bbot/scripts/docs.py + - name: Commit changes + uses: EndBug/add-and-commit@v9 + with: + add: '["*.md", "docs/data/chord_graph/*.json"]' + author_name: "BBOT Docs Autopublish" + author_email: info@blacklanternsecurity.com + message: "Refresh module docs" + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} + branch: update-docs + base: dev + title: "Daily Docs Update" + body: "This is an automated pull request to update the documentation." diff --git a/.github/workflows/version_updater.yaml b/.github/workflows/version_updater.yml similarity index 100% rename from .github/workflows/version_updater.yaml rename to .github/workflows/version_updater.yml diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 689cd9f0d..5a02213ce 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -158,7 +158,7 @@ def __init__( Raises: ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False. """ - self.uuid = uuid.uuid4() + self._uuid = uuid.uuid4() self._id = None self._hash = None self._data = None @@ -456,6 +456,13 @@ def id(self): self._id = f"{self.type}:{self.data_hash.hex()}" return self._id + @property + def uuid(self): + """ + A universally unique identifier for the event + """ + return f"{self.type}:{self._uuid}" + @property def data_hash(self): """ @@ -1718,7 +1725,7 @@ def event_from_json(j, siem_friendly=False): event = make_event(**kwargs) event_uuid = j.get("uuid", None) if event_uuid is not None: - event.uuid = uuid.UUID(event_uuid) + event._uuid = uuid.UUID(event_uuid.split(":")[-1]) resolved_hosts = j.get("resolved_hosts", []) event._resolved_hosts = set(resolved_hosts) @@ -1730,7 +1737,8 @@ def event_from_json(j, siem_friendly=False): event._parent_id = parent_id parent_uuid = j.get("parent_uuid", None) if parent_uuid is not None: - event._parent_uuid = uuid.UUID(parent_uuid) + parent_type, parent_uuid = parent_uuid.split(":", 1) + event._parent_uuid = parent_type + ":" + str(uuid.UUID(parent_uuid)) return event except KeyError as e: raise ValidationError(f"Event missing required field: {e}") diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index ed3452c6e..1fd513e5a 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -54,6 +54,9 @@ # uuid regex _uuid_regex = r"[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}" uuid_regex = re.compile(_uuid_regex, re.I) +# event uuid regex +_event_uuid_regex = r"[0-9A-Z_]+:[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}" +event_uuid_regex = re.compile(_event_uuid_regex, re.I) _open_port_regexes = ( _dns_name_regex + r":[0-9]{1,5}", diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index fbee2f915..00ce75ff8 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -4,7 +4,7 @@ from ..bbot_fixtures import * from bbot.scanner import Scanner -from bbot.core.helpers.regexes import uuid_regex +from bbot.core.helpers.regexes import event_uuid_regex @pytest.mark.asyncio @@ -443,11 +443,17 @@ async def test_events(events, helpers): parent_event2 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context") event1 = scan.make_event("evilcorp.com:80", parent=parent_event1, context="test context") + assert hasattr(event1, "_uuid") assert hasattr(event1, "uuid") - assert isinstance(event1.uuid, uuid.UUID) + assert isinstance(event1._uuid, uuid.UUID) + assert isinstance(event1.uuid, str) + assert event1.uuid == f"{event1.type}:{event1._uuid}" event2 = scan.make_event("evilcorp.com:80", parent=parent_event2, context="test context") + assert hasattr(event2, "_uuid") assert hasattr(event2, "uuid") - assert isinstance(event2.uuid, uuid.UUID) + assert isinstance(event2._uuid, uuid.UUID) + assert isinstance(event2.uuid, str) + assert event2.uuid == f"{event2.type}:{event2._uuid}" # ids should match because the event type + data is the same assert event1.id == event2.id # but uuids should be unique! @@ -470,7 +476,7 @@ async def test_events(events, helpers): assert db_event.discovery_context == "test context" assert db_event.discovery_path == ["test context"] assert len(db_event.parent_chain) == 1 - assert all([uuid_regex.match(u) for u in db_event.parent_chain]) + assert all([event_uuid_regex.match(u) for u in db_event.parent_chain]) assert db_event.parent_chain[0] == str(db_event.uuid) assert db_event.parent.uuid == scan.root_event.uuid assert db_event.parent_uuid == scan.root_event.uuid @@ -490,7 +496,7 @@ async def test_events(events, helpers): assert json_event["parent_chain"] == db_event.parent_chain assert json_event["parent_chain"][0] == str(db_event.uuid) reconstituted_event = event_from_json(json_event) - assert isinstance(reconstituted_event.uuid, uuid.UUID) + assert isinstance(reconstituted_event._uuid, uuid.UUID) assert str(reconstituted_event.uuid) == json_event["uuid"] assert str(reconstituted_event.parent_uuid) == json_event["parent_uuid"] assert reconstituted_event.uuid == db_event.uuid diff --git a/docs/scanning/events.md b/docs/scanning/events.md index ed5a7cdb9..9b0098808 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -8,50 +8,91 @@ An Event is a piece of data discovered by BBOT. Examples include `IP_ADDRESS`, ` event type event data source module tags ``` -In addition to the obvious data (e.g. `www.evilcorp.com`), an event also contains other useful information such as: - -- a `.discovery_path` showing exactly how the event was discovered, starting from the first scan target -- a `.timestamp` of when the data was discovered -- the `.module` that discovered it -- the `.parent` event that led to its discovery -- its `.scope_distance` (how many hops it is from the main scope, 0 == in-scope) -- a list of `.tags` that describe the data (`mx-record`, `http-title`, etc.) +## Event Attributes + +Each BBOT event has the following attributes. Not all of these attributes are visible in the terminal output. However, they are always saved in `output.json` in the scan output folder. If you want to see them on the terminal, you can use `--json`. + +- `.type`: the event type (e.g. `DNS_NAME`, `IP_ADDRESS`, `OPEN_TCP_PORT`, etc.) +- `.id`: an identifier representing the event type + a SHA1 hash of its data (note: multiple events can have the same `.id`) +- `.uuid`: a universally unique identifier for the event (e.g. `DNS_NAME:6c96d512-090a-47f0-82e4-6860e46aac13`) +- `.scope_description`: describes the scope of the event (e.g. `in-scope`, `affiliate`, `distance-2`) +- `.data`: the actual discovered data (for some events like `DNS_NAME` or `IP_ADDRESS`, this is a string. For other more complex events like `HTTP_RESPONSE`, it's a dictionary) +- `.host`: the hostname or IP address (e.g. `evilcorp.com` or `1.2.3.4`) +- `.port`: the port number (e.g. `80`, `443`) +- `.netloc`: the network location, including both the hostname and port (e.g. `www.evilcorp.com:443`) +- `.resolved_hosts`: a list of all resolved hosts for the event (`A`, `AAAA`, and `CNAME` records) +- `.dns_children`: a dictionary of all DNS records for the event (typically only present on `DNS_NAME`) +- `.web_spider_distance`: a count of how many URL links have been followed in a row to get to this event +- `.scope_distance`: a count of how many hops it is from the main scope (0 == in-scope) +- `.scan`: the ID of the scan that produced the event +- `.timestamp`: the date/time when the event was discovered +- `.parent`: the ID of the parent event that led to the discovery of this event +- `.parent_uuid`: the universally unique identifier for the parent event +- `.tags`: a list of tags describing the event (e.g. `mx-record`, `http-title`, etc.) +- `.module`: the module that discovered the event +- `.module_sequence`: the recent sequence of modules that were executed to discover the event (including omitted events) +- `.discovery_context`: a description of the context in which the event was discovered +- `.discovery_path`: a list of every discovery context leading to this event +- `.parent_chain`: a list of every event UUID leading to the discovery of this event (corresponds exactly to `.discovery_path`) These attributes allow us to construct a visual graph of events (e.g. in [Neo4j](../output#neo4j)) and query/filter/grep them more easily. Here is what a typical event looks like in JSON format: ```json { "type": "DNS_NAME", - "id": "DNS_NAME:879e47564ff0ed7711b707d3dbecb706ad6af1a3", + "id": "DNS_NAME:33bc005c2bdfea4d73e07db733bd11861cf6520e", + "uuid": "DNS_NAME:6c96d512-090a-47f0-82e4-6860e46aac13", "scope_description": "in-scope", - "data": "www.blacklanternsecurity.com", - "host": "www.blacklanternsecurity.com", + "data": "link.tesla.com", + "host": "link.tesla.com", "resolved_hosts": [ - "185.199.108.153", - "2606:50c0:8003::153", - "blacklanternsecurity.github.io" + "184.31.52.65", + "2600:1402:b800:d82::700", + "2600:1402:b800:d87::700", + "link.tesla.com.edgekey.net" ], - "dns_children": {}, + "dns_children": { + "A": [ + "184.31.52.65" + ], + "AAAA": [ + "2600:1402:b800:d82::700", + "2600:1402:b800:d87::700" + ], + "CNAME": [ + "link.tesla.com.edgekey.net" + ] + }, "web_spider_distance": 0, "scope_distance": 0, - "scan": "SCAN:477d1e6b94be928bf85c554b0845985189cfc81d", - "timestamp": "2024-08-17T03:49:47.906017+00:00", - "parent": "DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", + "scan": "SCAN:b6ef48bc036bc8d001595ae5061846a7e6beadb6", + "timestamp": "2024-10-18T15:40:13.716880+00:00", + "parent": "DNS_NAME:94c92b7eaed431b37ae2a757fec4e678cc3bd213", + "parent_uuid": "DNS_NAME:c737dffa-d4f0-4b6e-a72d-cc8c05bd892e", "tags": [ - "cdn-github", "subdomain", - "in-scope" + "a-record", + "cdn-akamai", + "in-scope", + "cname-record", + "aaaa-record" ], - "module": "otx", - "module_sequence": "otx", - "discovery_context": "otx searched otx API for \"blacklanternsecurity.com\" and found DNS_NAME: www.blacklanternsecurity.com", + "module": "speculate", + "module_sequence": "speculate->speculate", + "discovery_context": "speculated parent DNS_NAME: link.tesla.com", "discovery_path": [ - "Scan demonic_jimmy seeded with DNS_NAME: blacklanternsecurity.com", - "otx searched otx API for \"blacklanternsecurity.com\" and found DNS_NAME: www.blacklanternsecurity.com" + "Scan insidious_frederick seeded with DNS_NAME: tesla.com", + "TXT record for tesla.com contains IP_ADDRESS: 149.72.247.52", + "PTR record for 149.72.247.52 contains DNS_NAME: o1.ptr2410.link.tesla.com", + "speculated parent DNS_NAME: ptr2410.link.tesla.com", + "speculated parent DNS_NAME: link.tesla.com" ], "parent_chain": [ - "DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc", - "DNS_NAME:879e47564ff0ed7711b707d3dbecb706ad6af1a3" + "DNS_NAME:34c657a3-0bfa-457e-9e6e-0f22f04b8da5", + "IP_ADDRESS:efc0fb3b-1b42-44da-916e-83db2360e10e", + "DNS_NAME:c737dffa-d4f0-4b6e-a72d-cc8c05bd892e", + "DNS_NAME_UNRESOLVED:722a3473-30c6-40f1-90aa-908d47105d5a", + "DNS_NAME:6c96d512-090a-47f0-82e4-6860e46aac13" ] } ``` diff --git a/poetry.lock b/poetry.lock index d084aded1..6e6dd87f1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -522,38 +522,38 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "43.0.1" +version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, - {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, - {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, - {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, - {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, - {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, - {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, - {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, - {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, - {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, - {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, + {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6"}, + {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd"}, + {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73"}, + {file = "cryptography-43.0.3-cp37-abi3-win32.whl", hash = "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2"}, + {file = "cryptography-43.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd"}, + {file = "cryptography-43.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7"}, + {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16"}, + {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73"}, + {file = "cryptography-43.0.3-cp39-abi3-win32.whl", hash = "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995"}, + {file = "cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83"}, + {file = "cryptography-43.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa"}, + {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, + {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, ] [package.dependencies] @@ -566,7 +566,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.3)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -708,13 +708,13 @@ dev = ["flake8", "markdown", "twine", "wheel"] [[package]] name = "griffe" -version = "1.4.1" +version = "1.5.1" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." optional = false python-versions = ">=3.9" files = [ - {file = "griffe-1.4.1-py3-none-any.whl", hash = "sha256:84295ee0b27743bd880aea75632830ef02ded65d16124025e4c263bb826ab645"}, - {file = "griffe-1.4.1.tar.gz", hash = "sha256:911a201b01dc92e08c0e84c38a301e9da5ec067f00e7d9f2e39bc24dbfa3c176"}, + {file = "griffe-1.5.1-py3-none-any.whl", hash = "sha256:ad6a7980f8c424c9102160aafa3bcdf799df0e75f7829d75af9ee5aef656f860"}, + {file = "griffe-1.5.1.tar.gz", hash = "sha256:72964f93e08c553257706d6cd2c42d1c172213feb48b2be386f243380b405d4b"}, ] [package.dependencies] @@ -1081,72 +1081,72 @@ testing = ["coverage", "pyyaml"] [[package]] name = "markupsafe" -version = "3.0.1" +version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" files = [ - {file = "MarkupSafe-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:db842712984e91707437461930e6011e60b39136c7331e971952bb30465bc1a1"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ffb4a8e7d46ed96ae48805746755fadd0909fea2306f93d5d8233ba23dda12a"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67c519635a4f64e495c50e3107d9b4075aec33634272b5db1cde839e07367589"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48488d999ed50ba8d38c581d67e496f955821dc183883550a6fbc7f1aefdc170"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f31ae06f1328595d762c9a2bf29dafd8621c7d3adc130cbb46278079758779ca"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80fcbf3add8790caddfab6764bde258b5d09aefbe9169c183f88a7410f0f6dea"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3341c043c37d78cc5ae6e3e305e988532b072329639007fd408a476642a89fd6"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cb53e2a99df28eee3b5f4fea166020d3ef9116fdc5764bc5117486e6d1211b25"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-win32.whl", hash = "sha256:db15ce28e1e127a0013dfb8ac243a8e392db8c61eae113337536edb28bdc1f97"}, - {file = "MarkupSafe-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:4ffaaac913c3f7345579db4f33b0020db693f302ca5137f106060316761beea9"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-win32.whl", hash = "sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8ae369e84466aa70f3154ee23c1451fda10a8ee1b63923ce76667e3077f2b0c4"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40f1e10d51c92859765522cbd79c5c8989f40f0419614bcdc5015e7b6bf97fc5"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a4cb365cb49b750bdb60b846b0c0bc49ed62e59a76635095a179d440540c346"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee3941769bd2522fe39222206f6dd97ae83c442a94c90f2b7a25d847d40f4729"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62fada2c942702ef8952754abfc1a9f7658a4d5460fabe95ac7ec2cbe0d02abc"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c2d64fdba74ad16138300815cfdc6ab2f4647e23ced81f59e940d7d4a1469d9"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fb532dd9900381d2e8f48172ddc5a59db4c445a11b9fab40b3b786da40d3b56b"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0f84af7e813784feb4d5e4ff7db633aba6c8ca64a833f61d8e4eade234ef0c38"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-win32.whl", hash = "sha256:cbf445eb5628981a80f54087f9acdbf84f9b7d862756110d172993b9a5ae81aa"}, - {file = "MarkupSafe-3.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:a10860e00ded1dd0a65b83e717af28845bb7bd16d8ace40fe5531491de76b79f"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e81c52638315ff4ac1b533d427f50bc0afc746deb949210bc85f05d4f15fd772"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:312387403cd40699ab91d50735ea7a507b788091c416dd007eac54434aee51da"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ae99f31f47d849758a687102afdd05bd3d3ff7dbab0a8f1587981b58a76152a"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c97ff7fedf56d86bae92fa0a646ce1a0ec7509a7578e1ed238731ba13aabcd1c"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7420ceda262dbb4b8d839a4ec63d61c261e4e77677ed7c66c99f4e7cb5030dd"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45d42d132cff577c92bfba536aefcfea7e26efb975bd455db4e6602f5c9f45e7"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4c8817557d0de9349109acb38b9dd570b03cc5014e8aabf1cbddc6e81005becd"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a54c43d3ec4cf2a39f4387ad044221c66a376e58c0d0e971d47c475ba79c6b5"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-win32.whl", hash = "sha256:c91b394f7601438ff79a4b93d16be92f216adb57d813a78be4446fe0f6bc2d8c"}, - {file = "MarkupSafe-3.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:fe32482b37b4b00c7a52a07211b479653b7fe4f22b2e481b9a9b099d8a430f2f"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:17b2aea42a7280db02ac644db1d634ad47dcc96faf38ab304fe26ba2680d359a"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:852dc840f6d7c985603e60b5deaae1d89c56cb038b577f6b5b8c808c97580f1d"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0778de17cff1acaeccc3ff30cd99a3fd5c50fc58ad3d6c0e0c4c58092b859396"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:800100d45176652ded796134277ecb13640c1a537cad3b8b53da45aa96330453"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d06b24c686a34c86c8c1fba923181eae6b10565e4d80bdd7bc1c8e2f11247aa4"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:33d1c36b90e570ba7785dacd1faaf091203d9942bc036118fab8110a401eb1a8"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:beeebf760a9c1f4c07ef6a53465e8cfa776ea6a2021eda0d0417ec41043fe984"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:bbde71a705f8e9e4c3e9e33db69341d040c827c7afa6789b14c6e16776074f5a"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-win32.whl", hash = "sha256:82b5dba6eb1bcc29cc305a18a3c5365d2af06ee71b123216416f7e20d2a84e5b"}, - {file = "MarkupSafe-3.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:730d86af59e0e43ce277bb83970530dd223bf7f2a838e086b50affa6ec5f9295"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4935dd7883f1d50e2ffecca0aa33dc1946a94c8f3fdafb8df5c330e48f71b132"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e9393357f19954248b00bed7c56f29a25c930593a77630c719653d51e7669c2a"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40621d60d0e58aa573b68ac5e2d6b20d44392878e0bfc159012a5787c4e35bc8"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f94190df587738280d544971500b9cafc9b950d32efcb1fba9ac10d84e6aa4e6"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6a387d61fe41cdf7ea95b38e9af11cfb1a63499af2759444b99185c4ab33f5b"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8ad4ad1429cd4f315f32ef263c1342166695fad76c100c5d979c45d5570ed58b"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e24bfe89c6ac4c31792793ad9f861b8f6dc4546ac6dc8f1c9083c7c4f2b335cd"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2a4b34a8d14649315c4bc26bbfa352663eb51d146e35eef231dd739d54a5430a"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-win32.whl", hash = "sha256:242d6860f1fd9191aef5fae22b51c5c19767f93fb9ead4d21924e0bcb17619d8"}, - {file = "MarkupSafe-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:93e8248d650e7e9d49e8251f883eed60ecbc0e8ffd6349e18550925e31bd029b"}, - {file = "markupsafe-3.0.1.tar.gz", hash = "sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] [[package]] @@ -2773,13 +2773,13 @@ test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] [[package]] name = "virtualenv" -version = "20.26.6" +version = "20.27.0" description = "Virtual Python Environment builder" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "virtualenv-20.26.6-py3-none-any.whl", hash = "sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2"}, - {file = "virtualenv-20.26.6.tar.gz", hash = "sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48"}, + {file = "virtualenv-20.27.0-py3-none-any.whl", hash = "sha256:44a72c29cceb0ee08f300b314848c86e57bf8d1f13107a5e671fb9274138d655"}, + {file = "virtualenv-20.27.0.tar.gz", hash = "sha256:2ca56a68ed615b8fe4326d11a0dca5dfbe8fd68510fb6c6349163bed3c15f2b2"}, ] [package.dependencies] @@ -3077,4 +3077,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "c25df7cf571a94b66f847340ccfcf4f06131e97df02562f7f33ba8346a28d4a1" +content-hash = "056fa05ead5abab1767c7c410539a4536659d1b6420bd13375aab965f98e326e" diff --git a/pyproject.toml b/pyproject.toml index 666bf2982..9158de2cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ pytest-timeout = "^2.3.1" pytest-httpx = "^0.30.0" pytest-httpserver = "^1.0.11" pytest = "^8.3.1" -pytest-asyncio = "=0.23.8" +pytest-asyncio = "0.23.8" [tool.poetry.group.docs.dependencies] mkdocs = "^1.5.2" From 29206f0a6e00225e7be38fecbd00f1bbfc6a1446 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 12:03:22 -0400 Subject: [PATCH 254/254] evilcorp --- docs/scanning/events.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/scanning/events.md b/docs/scanning/events.md index 9b0098808..06fefd4e1 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -43,13 +43,13 @@ These attributes allow us to construct a visual graph of events (e.g. in [Neo4j] "id": "DNS_NAME:33bc005c2bdfea4d73e07db733bd11861cf6520e", "uuid": "DNS_NAME:6c96d512-090a-47f0-82e4-6860e46aac13", "scope_description": "in-scope", - "data": "link.tesla.com", - "host": "link.tesla.com", + "data": "link.evilcorp.com", + "host": "link.evilcorp.com", "resolved_hosts": [ "184.31.52.65", "2600:1402:b800:d82::700", "2600:1402:b800:d87::700", - "link.tesla.com.edgekey.net" + "link.evilcorp.com.edgekey.net" ], "dns_children": { "A": [ @@ -60,7 +60,7 @@ These attributes allow us to construct a visual graph of events (e.g. in [Neo4j] "2600:1402:b800:d87::700" ], "CNAME": [ - "link.tesla.com.edgekey.net" + "link.evilcorp.com.edgekey.net" ] }, "web_spider_distance": 0, @@ -79,13 +79,13 @@ These attributes allow us to construct a visual graph of events (e.g. in [Neo4j] ], "module": "speculate", "module_sequence": "speculate->speculate", - "discovery_context": "speculated parent DNS_NAME: link.tesla.com", + "discovery_context": "speculated parent DNS_NAME: link.evilcorp.com", "discovery_path": [ - "Scan insidious_frederick seeded with DNS_NAME: tesla.com", - "TXT record for tesla.com contains IP_ADDRESS: 149.72.247.52", - "PTR record for 149.72.247.52 contains DNS_NAME: o1.ptr2410.link.tesla.com", - "speculated parent DNS_NAME: ptr2410.link.tesla.com", - "speculated parent DNS_NAME: link.tesla.com" + "Scan insidious_frederick seeded with DNS_NAME: evilcorp.com", + "TXT record for evilcorp.com contains IP_ADDRESS: 149.72.247.52", + "PTR record for 149.72.247.52 contains DNS_NAME: o1.ptr2410.link.evilcorp.com", + "speculated parent DNS_NAME: ptr2410.link.evilcorp.com", + "speculated parent DNS_NAME: link.evilcorp.com" ], "parent_chain": [ "DNS_NAME:34c657a3-0bfa-457e-9e6e-0f22f04b8da5",