Fix yara rule hostname generation bug

bbot/modules/internal/excavate.py

@@ -805,9 +805,9 @@ async def setup(self):
             if Path(self.custom_yara_rules).is_file():
                 with open(self.custom_yara_rules) as f:
                     rules_content = f.read()
-                self.debug(f"Successfully loaded secrets file [{self.custom_yara_rules}]")
+                self.debug(f"Successfully loaded custom yara rules file [{self.custom_yara_rules}]")
             else:
-                self.debug(f"Custom secrets is NOT a file. Will attempt to treat it as rule content")
+                self.debug(f"Custom yara rules file is NOT a file. Will attempt to treat it as rule content")
                 rules_content = self.custom_yara_rules
 
             self.debug(f"Final combined yara rule contents: {rules_content}")
@@ -816,13 +816,11 @@ async def setup(self):
                 try:
                     yara.compile(source=rule_content)
                 except yara.SyntaxError as e:
-                    self.hugewarning(f"Custom Yara rule failed to compile: {e}")
-                    return False
+                    return False, f"Custom Yara rule failed to compile: {e}"
 
                 rule_match = await self.helpers.re.search(self.yara_rule_name_regex, rule_content)
                 if not rule_match:
-                    self.hugewarning(f"Custom Yara formatted incorrectly: could not find rule name")
-                    return False
+                    return False, f"Custom Yara formatted incorrectly: could not find rule name"
 
                 rule_name = rule_match.groups(1)[0]
                 c = CustomExtractor(self)
@@ -838,9 +836,8 @@ async def setup(self):
         try:
             self.yara_rules = yara.compile(source=yara_rules_combined)
         except yara.SyntaxError as e:
-            self.hugewarning(f"Yara Rules failed to compile with error: [{e}]")
             self.debug(yara_rules_combined)
-            return False
+            return False, f"Yara Rules failed to compile with error: [{e}]"
 
         # pre-load valid URL schemes
         valid_schemes_filename = self.helpers.wordlist_dir / "valid_url_schemes.txt"

bbot/scanner/scanner.py

@@ -971,6 +971,7 @@ def dns_strings(self):
             dns_strings = []
             for t in dns_targets:
                 if not any(x in dns_targets_set for x in self.helpers.domain_parents(t, include_self=True)):
+                    dns_targets_set.add(t)
                     dns_strings.append(t)
             self._dns_strings = dns_strings
         return self._dns_strings

bbot/test/test_step_1/test_scan.py

@@ -78,6 +78,9 @@ async def test_scan(
     for scan in (scan0, scan1, scan2, scan4, scan5):
         await scan._cleanup()
 
+    scan6 = bbot_scanner("a.foobar.io", "b.foobar.io", "c.foobar.io", "foobar.io")
+    assert len(scan6.dns_strings) == 1
+
 
 @pytest.mark.asyncio
 async def test_url_extension_handling(bbot_scanner):