Filter out already processed events from trufflehog

bbot/modules/trufflehog.py

@@ -1,4 +1,5 @@
 import json
+from pathlib import Path
 from bbot.modules.base import BaseModule
 
 
@@ -59,6 +60,7 @@ async def setup(self):
             if not self.github_token:
                 self.deleted_forks = False
                 return None, "A github api_key must be provided to the github modules for deleted forks to be scanned"
+        self.processed = set()
         return True
 
     async def filter_event(self, event):
@@ -70,6 +72,13 @@ async def filter_event(self, event):
                     return False, "Module only accepts github CODE_REPOSITORY events"
             else:
                 return False, "Deleted forks is not enabled"
+        else:
+            path = event.data["path"]
+            for processed in self.processed:
+                processed_path = Path(processed)
+                new_path = Path(path)
+                if new_path.is_relative_to(processed_path):
+                    return False, "Parent folder has already been processed"
         return True
 
     async def handle_event(self, event):
@@ -80,6 +89,7 @@ async def handle_event(self, event):
                 module = "github-experimental"
         else:
             path = event.data["path"]
+            self.processed.add(path)
             if "git" in event.tags:
                 module = "git"
             elif "docker" in event.tags:

bbot/test/test_step_2/module_tests/test_module_trufflehog.py

@@ -13,6 +13,7 @@ class TestTrufflehog(ModuleTestBase):
         "github_org",
         "speculate",
         "git_clone",
+        "unstructured",
         "github_workflows",
         "dockerhub",
         "docker_pull",
@@ -854,6 +855,7 @@ def check(self, module_test, events):
             and "Raw result: [https://admin:admin@the-internet.herokuapp.com]" in e.data["description"]
             and "RawV2 result: [https://admin:admin@the-internet.herokuapp.com/basic_auth]" in e.data["description"]
         ]
+        # Trufflehog should find 3 verifiable secrets, 1 from the github, 1 from the workflow log and 1 from the docker image. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder
         assert 3 == len(vuln_events), "Failed to find secret in events"
         github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].parent
         folder = Path(github_repo_event.data["path"])
@@ -901,6 +903,7 @@ def check(self, module_test, events):
             and "Potential Secret Found." in e.data["description"]
             and "Raw result: [https://admin:admin@internal.host.com]" in e.data["description"]
         ]
+        # Trufflehog should find 3 unverifiable secrets, 1 from the github, 1 from the workflow log and 1 from the docker image. Unstructured will extract the text file but trufflehog should reject it as its already scanned the containing folder
         assert 3 == len(finding_events), "Failed to find secret in events"
         github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].parent
         folder = Path(github_repo_event.data["path"])