From 66b7e94258cb8020ae613ac397819274d62e3b0b Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 10 Sep 2024 16:24:16 -0400 Subject: [PATCH 1/4] add scan finish event --- bbot/scanner/scanner.py | 97 +++++++++++++------ bbot/test/test_step_1/test_dns.py | 16 +-- bbot/test/test_step_1/test_events.py | 2 +- .../test_step_1/test_manager_deduplication.py | 2 +- .../test_manager_scope_accuracy.py | 40 ++++---- bbot/test/test_step_1/test_modules_basic.py | 4 +- bbot/test/test_step_1/test_scope.py | 7 +- bbot/test/test_step_1/test_target.py | 6 +- .../module_tests/test_module_credshed.py | 2 +- .../module_tests/test_module_dehashed.py | 2 +- .../module_tests/test_module_dnsbrute.py | 2 +- .../module_tests/test_module_dnscommonsrv.py | 2 +- .../module_tests/test_module_github_org.py | 8 +- .../test_module_github_workflows.py | 2 +- .../module_tests/test_module_json.py | 20 ++-- .../module_tests/test_module_sslcert.py | 2 +- .../module_tests/test_module_stdout.py | 10 +- .../test_template_subdomain_enum.py | 4 +- 18 files changed, 136 insertions(+), 92 deletions(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index fc62fb4f1..f31ae5b0f 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -116,6 +116,13 @@ def __init__( **kwargs (list[str], optional): Additional keyword arguments (passed through to `Preset`). """ self._root_event = None + self.start_time = None + self.end_time = None + self.duration = None + self.duration_human = None + self.duration_seconds = None + + self._success = False if scan_id is not None: self.id = str(id) @@ -306,13 +313,13 @@ async def async_start_without_generator(self): async def async_start(self): """ """ - failed = True - scan_start_time = datetime.now() + self.start_time = datetime.now() + self.root_event.data["started_at"] = self.start_time.isoformat() try: await self._prep() self._start_log_handlers() - self.trace(f'Ran BBOT {__version__} at {scan_start_time}, command: {" ".join(sys.argv)}') + self.trace(f'Ran BBOT {__version__} at {self.start_time}, command: {" ".join(sys.argv)}') self.trace(f"Target: {self.preset.target.json}") self.trace(f"Preset: {self.preset.to_dict(redact_secrets=True)}") @@ -363,16 +370,19 @@ async def async_start(self): if self._finished_init and self.modules_finished: new_activity = await self.finish() if not new_activity: + self._success = True + await self._mark_finished() + yield self.root_event break await asyncio.sleep(0.1) - failed = False + self._success = True except BaseException as e: if self.helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)): self.stop() - failed = False + self._success = True else: try: raise @@ -396,24 +406,46 @@ async def async_start(self): await self._report() await self._cleanup() - log_fn = self.hugesuccess - if self.status == "ABORTING": - self.status = "ABORTED" - log_fn = self.hugewarning - elif failed: - self.status = "FAILED" - log_fn = self.critical - else: - self.status = "FINISHED" - - scan_run_time = datetime.now() - scan_start_time - scan_run_time = self.helpers.human_timedelta(scan_run_time) - log_fn(f"Scan {self.name} completed in {scan_run_time} with status {self.status}") - await self.dispatcher.on_finish(self) self._stop_log_handlers() + async def _mark_finished(self): + log_fn = self.hugesuccess + if self.status == "ABORTING": + status = "ABORTED" + log_fn = self.hugewarning + elif not self._success: + status = "FAILED" + log_fn = self.critical + else: + status = "FINISHED" + + self.end_time = datetime.now() + self.duration = self.end_time - self.start_time + self.duration_seconds = self.duration.total_seconds() + self.duration_human = self.helpers.human_timedelta(self.duration) + + status_message = f"Scan {self.name} completed in {self.duration_human} with status {status}" + + scan_finish_event = self.make_root_event(status_message) + scan_finish_event.data["status"] = status + + # queue final scan event with output modules + output_modules = [m for m in self.modules.values() if m._type == "output" and m.name != "python"] + for m in output_modules: + await m.queue_event(scan_finish_event) + # wait until output modules are flushed + while 1: + modules_finished = all([m.finished for m in output_modules]) + self.verbose(modules_finished) + if modules_finished: + break + await asyncio.sleep(0.05) + + self.status = status + log_fn(status_message) + def _start_modules(self): self.verbose(f"Starting module worker loops") for module in self.modules.values(): @@ -727,8 +759,8 @@ async def finish(self): await module.queue_event(finished_event) self.verbose("Completed finish()") return True - # Return False if no new events were generated since last time self.verbose("Completed final finish()") + # Return False if no new events were generated since last time return False def _drain_queues(self): @@ -948,15 +980,18 @@ def root_event(self): ``` """ if self._root_event is None: - root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True) - root_event._id = self.id - root_event.scope_distance = 0 - root_event.parent = root_event - root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") - root_event.discovery_context = f"Scan {self.name} started at {root_event.timestamp}" - self._root_event = root_event + self._root_event = self.make_root_event(f"Scan {self.name} started at {self.start_time}") + self._root_event.data["status"] = self.status return self._root_event + def make_root_event(self, context): + root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True, context=context) + root_event._id = self.id + root_event.scope_distance = 0 + root_event.parent = root_event + root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") + return root_event + @property def dns_strings(self): """ @@ -1030,6 +1065,14 @@ def json(self): j.update({i: v}) j["target"] = self.preset.target.json j["preset"] = self.preset.to_dict(redact_secrets=True) + if self.start_time is not None: + j["started_at"] = self.start_time.isoformat() + if self.end_time is not None: + j["finished_at"] = self.end_time.isoformat() + if self.duration is not None: + j["duration_seconds"] = self.duration_seconds + if self.duration_human is not None: + j["duration"] = self.duration_human return j def debug(self, *args, trace=False, **kwargs): diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index f4bfb218a..4829125a4 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -263,7 +263,7 @@ def custom_lookup(query, rdtype): await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) events = [e async for e in scan.async_start()] - assert len(events) == 11 + assert len(events) == 12 assert len([e for e in events if e.type == "DNS_NAME"]) == 5 assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ @@ -320,7 +320,7 @@ def custom_lookup(query, rdtype): await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) events = [e async for e in scan.async_start()] - assert len(events) == 11 + assert len(events) == 12 assert len([e for e in events if e.type == "DNS_NAME"]) == 5 assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ @@ -418,7 +418,7 @@ def custom_lookup(query, rdtype): events = [e async for e in scan.async_start()] - assert len(events) == 10 + assert len(events) == 11 assert len([e for e in events if e.type == "DNS_NAME"]) == 5 assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [ @@ -546,8 +546,8 @@ def custom_lookup(query, rdtype): ) await scan2.ingress_module.queue_event(other_event, {}) events = [e async for e in scan2.async_start()] - assert len(events) == 3 - assert 1 == len([e for e in events if e.type == "SCAN"]) + assert len(events) == 4 + assert 2 == len([e for e in events if e.type == "SCAN"]) unmodified_wildcard_events = [ e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io" ] @@ -592,8 +592,8 @@ def custom_lookup(query, rdtype): ) await scan2.ingress_module.queue_event(other_event, {}) events = [e async for e in scan2.async_start()] - assert len(events) == 3 - assert 1 == len([e for e in events if e.type == "SCAN"]) + assert len(events) == 4 + assert 2 == len([e for e in events if e.type == "SCAN"]) unmodified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and "_wildcard" not in e.data] assert len(unmodified_wildcard_events) == 2 assert 1 == len( @@ -729,7 +729,7 @@ async def test_dns_graph_structure(bbot_scanner): } ) events = [e async for e in scan.async_start()] - assert len(events) == 5 + assert len(events) == 6 non_scan_events = [e for e in events if e.type != "SCAN"] assert sorted([e.type for e in non_scan_events]) == ["DNS_NAME", "DNS_NAME", "DNS_NAME", "URL_UNVERIFIED"] events_by_data = {e.data: e for e in non_scan_events} diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index cf5aebb42..f7ab9db2b 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -656,7 +656,7 @@ async def handle_event(self, event): ) events = [e async for e in scan.async_start()] - assert len(events) == 6 + assert len(events) == 7 assert 1 == len( [ diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index 06fc026f7..65fbaeb17 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -91,7 +91,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - assert len(events) == 21 + assert len(events) == 22 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"]) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index ef254f38c..f860f1746 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -138,7 +138,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - assert len(events) == 2 + assert len(events) == 3 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) @@ -153,7 +153,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 0 == len([e for e in _all_events if e.type == "DNS_NAME" and e.data == "www.test.notreal"]) assert 0 == len([e for e in _all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"]) - assert len(graph_output_events) == 2 + assert len(graph_output_events) == 3 assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) @@ -167,7 +167,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - assert len(events) == 3 + assert len(events) == 4 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"]) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) @@ -195,7 +195,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 5 + assert len(_graph_output_events) == 6 assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == True and e.scope_distance == 1]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 1]) @@ -211,7 +211,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) _dns_mock=dns_mock_chain, ) - assert len(events) == 6 + assert len(events) == 7 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == False and e.scope_distance == 1]) @@ -239,7 +239,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == False and e.scope_distance == 1]) @@ -282,7 +282,7 @@ def custom_setup(scan): _dns_mock=dns_mock_chain, ) - assert len(events) == 4 + assert len(events) == 5 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"]) @@ -304,7 +304,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e.internal == False and e.scope_distance == 3]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1]) assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 2]) @@ -326,7 +326,7 @@ def custom_setup(scan): _dns_mock={}, ) - assert len(events) == 6 + assert len(events) == 7 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -366,7 +366,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == True and e.scope_distance == 1]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -393,7 +393,7 @@ def custom_setup(scan): }, ) - assert len(events) == 7 + assert len(events) == 8 # 2024-08-01 # Removed OPEN_TCP_PORT("127.0.0.77:8888") # before, this event was speculated off the URL_UNVERIFIED, and that's what was used by httpx to generate the URL. it was graph-important. @@ -450,7 +450,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 7 + assert len(_graph_output_events) == 8 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -481,7 +481,7 @@ def custom_setup(scan): }, ) - assert len(events) == 7 + assert len(events) == 8 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -541,7 +541,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.99:8888/" and e.internal == True and e.scope_distance == 3]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 7 + assert len(_graph_output_events) == 8 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -574,7 +574,7 @@ def custom_setup(scan): }, ) - assert len(events) == 9 + assert len(events) == 10 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0]) @@ -659,7 +659,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888" and e.internal == True and e.scope_distance == 1]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 9 + assert len(_graph_output_events) == 10 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0]) @@ -695,7 +695,7 @@ def custom_setup(scan): _dns_mock={"www.bbottest.notreal": {"A": ["127.0.1.0"]}, "test.notreal": {"A": ["127.0.0.1"]}}, ) - assert len(events) == 6 + assert len(events) == 7 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) @@ -731,7 +731,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 6 + assert len(_graph_output_events) == 7 assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0]) @@ -752,7 +752,7 @@ def custom_setup(scan): _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}}, ) - assert len(events) == 3 + assert len(events) == 4 assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"]) @@ -783,7 +783,7 @@ def custom_setup(scan): assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"]) for _graph_output_events in (graph_output_events, graph_output_batch_events): - assert len(_graph_output_events) == 5 + assert len(_graph_output_events) == 6 assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"]) assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2]) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 10cfa5bb6..9fbaed085 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -380,8 +380,8 @@ async def handle_event(self, event): scan.modules["dummy"] = dummy(scan) events = [e async for e in scan.async_start()] - assert len(events) == 8 - assert 1 == len([e for e in events if e.type == "SCAN"]) + assert len(events) == 9 + assert 2 == len([e for e in events if e.type == "SCAN"]) assert 3 == len([e for e in events if e.type == "DNS_NAME"]) # one from target and one from speculate assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com"]) diff --git a/bbot/test/test_step_1/test_scope.py b/bbot/test/test_step_1/test_scope.py index 7435b82af..ac2d8c042 100644 --- a/bbot/test/test_step_1/test_scope.py +++ b/bbot/test/test_step_1/test_scope.py @@ -12,7 +12,8 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert len(events) == 6 + assert len(events) == 7 + assert 2 == len([e for e in events if e.type == "SCAN"]) assert 1 == len( [ e @@ -62,7 +63,7 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - assert len(events) == 1 + assert len(events) == 2 assert not any(e.type == "URL" for e in events) assert not any(str(e.host) == "127.0.0.1" for e in events) @@ -72,7 +73,7 @@ class TestScopeWhitelist(TestScopeBlacklist): whitelist = ["255.255.255.255"] def check(self, module_test, events): - assert len(events) == 3 + assert len(events) == 4 assert not any(e.type == "URL" for e in events) assert 1 == len( [ diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index efdf089d3..5b974bd45 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -182,14 +182,14 @@ async def test_target(bbot_scanner): for org_target in ("ORG:evilcorp", "ORG_STUB:evilcorp"): scan = bbot_scanner(org_target) events = [e async for e in scan.async_start()] - assert len(events) == 2 + assert len(events) == 3 assert set([e.type for e in events]) == {"SCAN", "ORG_STUB"} # test username as target for user_target in ("USER:vancerefrigeration", "USERNAME:vancerefrigeration"): scan = bbot_scanner(user_target) events = [e async for e in scan.async_start()] - assert len(events) == 2 + assert len(events) == 3 assert set([e.type for e in events]) == {"SCAN", "USERNAME"} # verify hash values @@ -225,7 +225,7 @@ async def test_target(bbot_scanner): ) events = [e async for e in scan.async_start()] scan_events = [e for e in events if e.type == "SCAN"] - assert len(scan_events) == 1 + assert len(scan_events) == 2 target_dict = scan_events[0].data["target"] assert target_dict["strict_scope"] == False assert target_dict["hash"] == b"\x0b\x908\xe3\xef\n=\x13d\xdf\x00;\xack\x0c\xbc\xd2\xcc'\xba".hex() diff --git a/bbot/test/test_step_2/module_tests/test_module_credshed.py b/bbot/test/test_step_2/module_tests/test_module_credshed.py index 4b9566077..44b9133c9 100644 --- a/bbot/test/test_step_2/module_tests/test_module_credshed.py +++ b/bbot/test/test_step_2/module_tests/test_module_credshed.py @@ -69,7 +69,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 10 + assert len(events) == 11 assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "bob@blacklanternsecurity.com"]) assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "judy@blacklanternsecurity.com"]) assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "tim@blacklanternsecurity.com"]) diff --git a/bbot/test/test_step_2/module_tests/test_module_dehashed.py b/bbot/test/test_step_2/module_tests/test_module_dehashed.py index e8c90273f..0ac91c3b8 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dehashed.py +++ b/bbot/test/test_step_2/module_tests/test_module_dehashed.py @@ -56,7 +56,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 11 + assert len(events) == 12 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"]) assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity"]) assert 1 == len( diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py index 34702b2af..12427b050 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py @@ -92,7 +92,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs): assert reason == f'"ptr1234.blacklanternsecurity.com" looks like an autogenerated PTR' def check(self, module_test, events): - assert len(events) == 3 + assert len(events) == 4 assert 1 == len( [e for e in events if e.data == "asdf.blacklanternsecurity.com" and str(e.module) == "dnsbrute"] ) diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py index 947574dfd..6c5023db1 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py @@ -56,7 +56,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs): ) def check(self, module_test, events): - assert len(events) == 19 + assert len(events) == 20 assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"]) assert 1 == len( [ diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py index a4313d182..039c6125b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_github_org.py +++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py @@ -284,7 +284,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 6 + assert len(events) == 7 assert 1 == len( [ e @@ -335,7 +335,7 @@ class TestGithub_Org_No_Members(TestGithub_Org): config_overrides = {"modules": {"github_org": {"include_members": False}}} def check(self, module_test, events): - assert len(events) == 5 + assert len(events) == 6 assert 1 == len( [ e @@ -363,7 +363,7 @@ class TestGithub_Org_MemberRepos(TestGithub_Org): config_overrides = {"modules": {"github_org": {"include_member_repos": True}}} def check(self, module_test, events): - assert len(events) == 7 + assert len(events) == 8 assert 1 == len( [ e @@ -381,7 +381,7 @@ class TestGithub_Org_Custom_Target(TestGithub_Org): config_overrides = {"scope": {"report_distance": 10}, "omit_event_types": [], "speculate": True} def check(self, module_test, events): - assert len(events) == 7 + assert len(events) == 8 assert 1 == len( [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 0] ) diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py index f3c4a2cf5..621df5871 100644 --- a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py +++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py @@ -477,7 +477,7 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 8 + assert len(events) == 9 assert 1 == len( [ e diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index ad3417539..dc5ebd842 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -19,17 +19,17 @@ def check(self, module_test, events): json_events = [json.loads(line) for line in lines] scan_json = [e for e in json_events if e["type"] == "SCAN"] dns_json = [e for e in json_events if e["type"] == "DNS_NAME"] - assert len(scan_json) == 1 + assert len(scan_json) == 2 assert len(dns_json) == 1 - scan_json = scan_json[0] dns_json = dns_json[0] - assert scan_json["data"]["name"] == module_test.scan.name - assert scan_json["data"]["id"] == module_test.scan.id - assert scan_json["id"] == module_test.scan.id - assert scan_json["uuid"] == str(module_test.scan.root_event.uuid) - assert scan_json["parent_uuid"] == str(module_test.scan.root_event.uuid) - assert scan_json["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] - assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] + for scan in scan_json: + assert scan["data"]["name"] == module_test.scan.name + assert scan["data"]["id"] == module_test.scan.id + assert scan["id"] == module_test.scan.id + assert scan["uuid"] == str(module_test.scan.root_event.uuid) + assert scan["parent_uuid"] == str(module_test.scan.root_event.uuid) + assert scan["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] + assert scan["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["id"] == str(dns_event.id) assert dns_json["uuid"] == str(dns_event.uuid) @@ -39,7 +39,7 @@ def check(self, module_test, events): assert dns_json["parent_chain"] == [dns_json["uuid"]] # event objects reconstructed from json - scan_reconstructed = event_from_json(scan_json) + scan_reconstructed = event_from_json(scan_json[0]) dns_reconstructed = event_from_json(dns_json) assert scan_reconstructed.data["name"] == module_test.scan.name assert scan_reconstructed.data["id"] == module_test.scan.id diff --git a/bbot/test/test_step_2/module_tests/test_module_sslcert.py b/bbot/test/test_step_2/module_tests/test_module_sslcert.py index f8c4948fd..a81482ff5 100644 --- a/bbot/test/test_step_2/module_tests/test_module_sslcert.py +++ b/bbot/test/test_step_2/module_tests/test_module_sslcert.py @@ -6,7 +6,7 @@ class TestSSLCert(ModuleTestBase): config_overrides = {"scope": {"report_distance": 1}} def check(self, module_test, events): - assert len(events) == 6 + assert len(events) == 7 assert 1 == len( [ e diff --git a/bbot/test/test_step_2/module_tests/test_module_stdout.py b/bbot/test/test_step_2/module_tests/test_module_stdout.py index 2c9eaf9bd..7abfc1b79 100644 --- a/bbot/test/test_step_2/module_tests/test_module_stdout.py +++ b/bbot/test/test_step_2/module_tests/test_module_stdout.py @@ -17,7 +17,7 @@ class TestStdoutEventTypes(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() - assert len(out.splitlines()) == 1 + assert len(out.splitlines()) == 2 assert out.startswith("[DNS_NAME] \tblacklanternsecurity.com\tTARGET") @@ -41,7 +41,7 @@ class TestStdoutJSON(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 2 + assert len(lines) == 3 for i, line in enumerate(lines): event = json.loads(line) if i == 0: @@ -56,7 +56,7 @@ class TestStdoutJSONFields(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 2 + assert len(lines) == 3 for line in lines: event = json.loads(line) assert set(event) == {"data", "module_sequence"} @@ -79,7 +79,7 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 3 + assert len(lines) == 4 assert out.count("[IP_ADDRESS] \t127.0.0.2") == 2 @@ -97,5 +97,5 @@ class TestStdoutNoDupes(TestStdoutDupes): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 2 + assert len(lines) == 3 assert out.count("[IP_ADDRESS] \t127.0.0.2") == 1 diff --git a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py index c0bcb25a5..bfa186707 100644 --- a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py +++ b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py @@ -120,7 +120,7 @@ async def mock_query(query): def check(self, module_test, events): assert self.queries == ["walmart.cn"] - assert len(events) == 6 + assert len(events) == 7 assert 2 == len( [ e @@ -185,7 +185,7 @@ def custom_lookup(query, rdtype): def check(self, module_test, events): # no subdomain enum should happen on this domain! assert self.queries == [] - assert len(events) == 6 + assert len(events) == 7 assert 2 == len( [e for e in events if e.type == "IP_ADDRESS" and str(e.module) == "A" and e.scope_distance == 1] ) From 5644bfade1b5e67c0a8711d879d33ed2674b0d2c Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 20 Sep 2024 14:29:39 -0400 Subject: [PATCH 2/4] fix dnsbrute tests --- .../test_step_2/module_tests/test_module_dnsbrute_mutations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py index 0a7627f25..0c9b6baaa 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py @@ -47,7 +47,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs): ) def check(self, module_test, events): - assert len(events) == 9 + assert len(events) == 10 assert 1 == len( [ e From 126d3173879c7e574754c40b9ac2debb3008cccf Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 20 Sep 2024 22:49:45 -0400 Subject: [PATCH 3/4] work on tests --- .../test_step_2/module_tests/test_module_json.py | 16 ++++++++-------- .../module_tests/test_module_postman.py | 2 +- .../module_tests/test_module_stdout.py | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index dc5ebd842..27ed5a55e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -22,14 +22,14 @@ def check(self, module_test, events): assert len(scan_json) == 2 assert len(dns_json) == 1 dns_json = dns_json[0] - for scan in scan_json: - assert scan["data"]["name"] == module_test.scan.name - assert scan["data"]["id"] == module_test.scan.id - assert scan["id"] == module_test.scan.id - assert scan["uuid"] == str(module_test.scan.root_event.uuid) - assert scan["parent_uuid"] == str(module_test.scan.root_event.uuid) - assert scan["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] - assert scan["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] + scan = scan_json[0] + assert scan["data"]["name"] == module_test.scan.name + assert scan["data"]["id"] == module_test.scan.id + assert scan["id"] == module_test.scan.id + assert scan["uuid"] == str(module_test.scan.root_event.uuid) + assert scan["parent_uuid"] == str(module_test.scan.root_event.uuid) + assert scan["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] + assert scan["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["id"] == str(dns_event.id) assert dns_json["uuid"] == str(dns_event.uuid) diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py index b6f807a42..1066c5d73 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman.py @@ -90,7 +90,7 @@ async def setup_after_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 4 + assert len(events) == 5 assert 1 == len( [ e diff --git a/bbot/test/test_step_2/module_tests/test_module_stdout.py b/bbot/test/test_step_2/module_tests/test_module_stdout.py index 7abfc1b79..f21845ded 100644 --- a/bbot/test/test_step_2/module_tests/test_module_stdout.py +++ b/bbot/test/test_step_2/module_tests/test_module_stdout.py @@ -17,7 +17,7 @@ class TestStdoutEventTypes(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() - assert len(out.splitlines()) == 2 + assert len(out.splitlines()) == 1 assert out.startswith("[DNS_NAME] \tblacklanternsecurity.com\tTARGET") @@ -79,7 +79,7 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 4 + assert len(lines) == 3 assert out.count("[IP_ADDRESS] \t127.0.0.2") == 2 @@ -97,5 +97,5 @@ class TestStdoutNoDupes(TestStdoutDupes): def check(self, module_test, events): out, err = module_test.capsys.readouterr() lines = out.splitlines() - assert len(lines) == 3 + assert len(lines) == 2 assert out.count("[IP_ADDRESS] \t127.0.0.2") == 1 From c58cf71afb15046f84f7576600e3af12bff01289 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 21 Sep 2024 10:35:05 -0400 Subject: [PATCH 4/4] more work on tests --- bbot/test/test_step_2/module_tests/test_module_stdout.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_stdout.py b/bbot/test/test_step_2/module_tests/test_module_stdout.py index f21845ded..27d8a3059 100644 --- a/bbot/test/test_step_2/module_tests/test_module_stdout.py +++ b/bbot/test/test_step_2/module_tests/test_module_stdout.py @@ -46,8 +46,10 @@ def check(self, module_test, events): event = json.loads(line) if i == 0: assert event["type"] == "SCAN" - elif i == 2: + elif i == 1: assert event["type"] == "DNS_NAME" and event["data"] == "blacklanternsecurity.com" + if i == 2: + assert event["type"] == "SCAN" class TestStdoutJSONFields(TestStdout):