Merge pull request #1724 from blacklanternsecurity/dev

Dev --> Stable (2.1.0)

.github/workflows/docs_updater.yml

@@ -0,0 +1,40 @@
+name: Daily Docs Update
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs daily at midnight UTC
+
+jobs:
+  update_docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
+          ref: dev  # Checkout the dev branch
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install
+      - name: Generate docs
+        run: |
+          poetry run bbot/scripts/docs.py
+      - name: Commit changes
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: '["*.md", "docs/data/chord_graph/*.json"]'
+          author_name: "BBOT Docs Autopublish"
+          author_email: info@blacklanternsecurity.com
+          message: "Refresh module docs"
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
+          branch: update-docs
+          base: dev
+          title: "Daily Docs Update"
+          body: "This is an automated pull request to update the documentation."

.github/workflows/tests.yml

@@ -58,7 +58,7 @@ jobs:
   update_docs:
     needs: test
     runs-on: ubuntu-latest
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
+    if: github.event_name == 'push' && (github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/stable')
     steps:
       - uses: actions/checkout@v3
         with:
@@ -83,11 +83,13 @@ jobs:
           author_email: info@blacklanternsecurity.com
           message: "Refresh module docs"
   publish_docs:
-    needs: update_docs
+    needs: test
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev')
     steps:
       - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - uses: actions/setup-python@v4
         with:
           python-version: "3.x"
@@ -125,7 +127,7 @@ jobs:
           git switch gh-pages
           git push
   publish_code:
-    needs: update_docs
+    needs: test
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     continue-on-error: true

.github/workflows/version_updater.yaml → .github/workflows/version_updater.yml

@@ -9,10 +9,11 @@ jobs:
   update-nuclei-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: dev
           fetch-depth: 0
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -54,16 +55,17 @@ jobs:
             # Release notes:
             ${{ env.release_notes }}
           branch: "update-nuclei"
-          committer: GitHub <noreply@github.com>
-          author: GitHub <noreply@github.com>
+          committer: blsaccess <info@blacklanternsecurity.com>
+          author: blsaccess <info@blacklanternsecurity.com>
           assignees: "TheTechromancer"
   update-trufflehog-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: dev
           fetch-depth: 0
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -105,6 +107,6 @@ jobs:
             # Release notes:
             ${{ env.release_notes }}
           branch: "update-trufflehog"
-          committer: GitHub <noreply@github.com>
-          author: GitHub <noreply@github.com>
+          committer: blsaccess <info@blacklanternsecurity.com>
+          author: blsaccess <info@blacklanternsecurity.com>
           assignees: "TheTechromancer"

README.md

@@ -303,13 +303,17 @@ For more information, see [Targets](https://www.blacklanternsecurity.com/bbot/St
 
 Similar to Amass or Subfinder, BBOT supports API keys for various third-party services such as SecurityTrails, etc.
 
-The standard way to do this is to enter your API keys in **`~/.config/bbot/bbot.yml`**:
+The standard way to do this is to enter your API keys in **`~/.config/bbot/bbot.yml`**. Note that multiple API keys are allowed:
 ```yaml
 modules:
   shodan_dns:
     api_key: 4f41243847da693a4f356c0486114bc6
   c99:
-    api_key: 21a270d5f59c9b05813a72bb41707266
+    # multiple API keys
+    api_key:
+      - 21a270d5f59c9b05813a72bb41707266
+      - ea8f243d9885cf8ce9876a580224fd3c
+      - 5bc6ed268ab6488270e496d3183a1a27
   virustotal:
     api_key: dd5f0eee2e4a99b71a939bded450b246
   securitytrails:

bbot/core/core.py

@@ -34,8 +34,6 @@ def __init__(self):
         self._logger = None
         self._files_config = None
 
-        self.bbot_sudo_pass = None
-
         self._config = None
         self._custom_config = None
 

bbot/core/engine.py

@@ -643,6 +643,7 @@ async def finished_tasks(self, tasks, timeout=None):
                     self.log.warning(f"{self.name}: Timeout after {timeout:,} seconds in finished_tasks({tasks})")
                     for task in tasks:
                         task.cancel()
+                        self._await_cancelled_task(task)
                 else:
                     if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):
                         self.log.error(f"{self.name}: Unhandled exception in finished_tasks({tasks}): {e}")
@@ -664,9 +665,9 @@ async def cancel_task(self, client_id):
                 child_task.cancel()
 
         for task in [parent_task] + list(child_tasks):
-            await self._cancel_task(task)
+            await self._await_cancelled_task(task)
 
-    async def _cancel_task(self, task):
+    async def _await_cancelled_task(self, task):
         try:
             await asyncio.wait_for(task, timeout=10)
         except (TimeoutError, asyncio.exceptions.TimeoutError):
@@ -683,4 +684,4 @@ async def cancel_all_tasks(self):
             await self.cancel_task(client_id)
         for client_id, tasks in self.child_tasks.items():
             for task in tasks:
-                await self._cancel_task(task)
+                await self._await_cancelled_task(task)