From 16a37a0de3f3c0d9aa5a9d52021cfa89f68328c9 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Thu, 2 May 2024 07:56:44 -0400 Subject: [PATCH 01/12] fix: load external tasks from template dir --- secator/output_types/vulnerability.py | 6 --- secator/tasks/__init__.py | 10 ++--- secator/utils.py | 42 ++++++++++++++------- secator/utils_test.py | 13 +++++++ tests/fixtures/ls.py | 36 ++++++++++++++++++ tests/unit/test_offline.py | 11 ++---- tests/unit/test_template.py | 53 +++++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 33 deletions(-) create mode 100644 tests/fixtures/ls.py create mode 100644 tests/unit/test_template.py diff --git a/secator/output_types/vulnerability.py b/secator/output_types/vulnerability.py index 05c06da1..74c89344 100644 --- a/secator/output_types/vulnerability.py +++ b/secator/output_types/vulnerability.py @@ -89,11 +89,5 @@ def __repr__(self): s = f'[dim]{s}[/]' return rich_to_ansi(s) - # def __gt__(self, other): - # # favor httpx over other url info tools - # if self._source == 'httpx' and other._source != 'httpx': - # return True - # return super().__gt__(other) - def __str__(self): return self.matched_at + ' -> ' + self.name diff --git a/secator/tasks/__init__.py b/secator/tasks/__init__.py index 2606bc2e..867bbdf3 100644 --- a/secator/tasks/__init__.py +++ b/secator/tasks/__init__.py @@ -1,10 +1,8 @@ -from secator.utils import discover_internal_tasks, discover_external_tasks -INTERNAL_TASKS = discover_internal_tasks() -EXTERNAL_TASKS = discover_external_tasks() -ALL_TASKS = INTERNAL_TASKS + EXTERNAL_TASKS +from secator.utils import discover_tasks +TASKS = discover_tasks() __all__ = [ cls.__name__ - for cls in ALL_TASKS + for cls in TASKS ] -for cls in INTERNAL_TASKS: +for cls in TASKS: exec(f'from .{cls.__name__} import {cls.__name__}') diff --git a/secator/utils.py b/secator/utils.py index b12a1c9c..abb9df12 100644 --- a/secator/utils.py +++ b/secator/utils.py @@ -1,4 +1,5 @@ import inspect +import importlib import itertools import logging import operator @@ -8,7 +9,7 @@ import sys import warnings from datetime import datetime -from importlib import import_module + from inspect import isclass from pathlib import Path from pkgutil import iter_modules @@ -138,7 +139,7 @@ def discover_internal_tasks(): if module_name.startswith('_'): continue try: - module = import_module(f'secator.tasks.{module_name}') + module = importlib.import_module(f'secator.tasks.{module_name}') except ImportError as e: console.print(f'[bold red]Could not import secator.tasks.{module_name}:[/]') console.print(f'\t[bold red]{type(e).__name__}[/]: {str(e)}') @@ -160,17 +161,32 @@ def discover_internal_tasks(): def discover_external_tasks(): """Find external secator tasks.""" - if not os.path.exists('config.secator'): - return [] - with open('config.secator', 'r') as f: - classes = f.read().splitlines() output = [] - for cls_path in classes: - cls = import_dynamic(cls_path, cls_root='Command') - if not cls: - continue - # logger.warning(f'Added external tool {cls_path}') - output.append(cls) + sys.dont_write_bytecode = True + for path in CONFIG.dirs.templates.glob('**/*.py'): + try: + task_name = path.stem + module_name = f'secator.tasks.{task_name}' + + # console.print(f'Importing module {module_name} from {path}') + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + # console.print(f'Adding module "{module_name}" to sys path') + sys.modules[module_name] = module + + # console.print(f'Executing module "{module}"') + spec.loader.exec_module(module) + + # console.print(f'Checking that {module} contains task {task_name}') + if not hasattr(module, task_name): + console.print(f'[bold orange1]Could not load external task "{task_name}" from module {path.name}[/] ({path})') + continue + cls = getattr(module, task_name) + console.print(f'[bold green]Successfully loaded external task "{task_name}"[/] ({path})') + output.append(cls) + except Exception as e: + console.print(f'[bold red]Could not load external module {path.name}. Reason: {str(e)}.[/] ({path})') + sys.dont_write_bytecode = False return output @@ -194,7 +210,7 @@ def import_dynamic(cls_path, cls_root='Command'): """ try: package, name = cls_path.rsplit(".", maxsplit=1) - cls = getattr(import_module(package), name) + cls = getattr(importlib.import_module(package), name) root_cls = inspect.getmro(cls)[-2] if root_cls.__name__ == cls_root: return cls diff --git a/secator/utils_test.py b/secator/utils_test.py index 753e15d2..27c0ef2c 100644 --- a/secator/utils_test.py +++ b/secator/utils_test.py @@ -1,6 +1,7 @@ import contextlib import json import os +import sys import unittest.mock from fp.fp import FreeProxy @@ -182,3 +183,15 @@ def _test_task_output( raise console.print('[bold green] ok[/]') + + +def clear_modules(): + """Clear all secator modules imports. + See https://stackoverflow.com/questions/7460363/re-import-module-under-test-to-lose-context for context. + """ + keys_to_delete = [] + for k, _ in sys.modules.items(): + if k.startswith('secator'): + keys_to_delete.append(k) + for k in keys_to_delete: + del sys.modules[k] diff --git a/tests/fixtures/ls.py b/tests/fixtures/ls.py new file mode 100644 index 00000000..0a5fbcce --- /dev/null +++ b/tests/fixtures/ls.py @@ -0,0 +1,36 @@ +from secator.runners import Command +from secator.decorators import task +from secator.output_types import Vulnerability + + +@task() +class ls(Command): + cmd = 'ls -al' + output_types = [Vulnerability] + output_map = { + Vulnerability: {} + } + + @staticmethod + def item_loader(self, line): + fields = ['permissions', 'link_count', 'owner', 'group', 'size', 'month', 'day', 'hour', 'path'] + result = [c for c in line.split(' ') if c] + if len(result) != len(fields): + return None + data = {} + for ix, value in enumerate(result): + data[fields[ix]] = value + + # Output vulnerabilities + permissions = data['permissions'] + path = data['path'] + full_path = f'{self.input}/{path}' + if permissions[-2] == 'w': # found a vulnerability ! + yield Vulnerability( + name='World-writeable path', + severity='high', + confidence='high', + provider='ls', + matched_at=full_path, + extra_data={k: v for k, v in data.items() if k != 'path'} + ) diff --git a/tests/unit/test_offline.py b/tests/unit/test_offline.py index e77103cf..9377bb02 100644 --- a/tests/unit/test_offline.py +++ b/tests/unit/test_offline.py @@ -1,17 +1,12 @@ import os -import sys import unittest +from secator.utils_test import clear_modules + class TestOffline(unittest.TestCase): def setUp(self): - try: - # This allows to drop the secator module loaded from other tests in order to reload the config with modified - # environment variables. - # See https://stackoverflow.com/questions/7460363/re-import-module-under-test-to-lose-context for context. - del sys.modules['secator'] - except KeyError: - pass + clear_modules() os.environ['SECATOR_OFFLINE_MODE'] = '1' def test_offline_cve_lookup(self): diff --git a/tests/unit/test_template.py b/tests/unit/test_template.py new file mode 100644 index 00000000..575254a4 --- /dev/null +++ b/tests/unit/test_template.py @@ -0,0 +1,53 @@ +import unittest +from secator.config import CONFIG +from secator.output_types import Vulnerability +from secator.utils_test import FIXTURES_DIR, clear_modules +import os + +import shutil + + +class TestTemplate(unittest.TestCase): + def setUp(self): + self.template_dir = CONFIG.dirs.templates + self.custom_task_path = self.template_dir / 'ls.py' + self.writeable_file = self.template_dir / 'test.txt' + self.custom_workflow_path = self.template_dir / 'ls.yml' + shutil.copy(f'{FIXTURES_DIR}/ls.py', self.custom_task_path) + shutil.copy(f'{FIXTURES_DIR}/ls.yml', self.custom_workflow_path) + self.writeable_file.touch() + os.chmod(self.writeable_file, 0o007) + self.expected_vuln = Vulnerability( + name='World-writeable path', + severity='high', + confidence='high', + provider='ls', + matched_at=f'/home/osboxes/.secator/templates/test.txt', + _source='ls', + ) + clear_modules() + self.maxDiff = None + + def tearDown(self): + self.custom_task_path.unlink() + self.custom_workflow_path.unlink() + self.writeable_file.unlink() + + def test_external_task(self): + from secator.tasks import ls + results = ls(str(self.template_dir)).run() + self.assertEqual(len(results), 1) + self.assertTrue(self.expected_vuln == Vulnerability.load(results[0].toDict())) + + def test_external_workflow(self): + from secator.cli import ALL_WORKFLOWS + from secator.runners import Workflow + ls_workflow = None + for w in ALL_WORKFLOWS: + if w.name == 'ls': + ls_workflow = w + self.assertIsNotNone(ls_workflow) + results = Workflow(ls_workflow, targets=[str(self.template_dir)]).run() + self.assertEqual(len(results), 2) + self.assertTrue(self.expected_vuln == Vulnerability.load(results[1].toDict())) + From 7e9fd4557fc9fec8f649ce63dea5483870616ef1 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Thu, 2 May 2024 07:58:03 -0400 Subject: [PATCH 02/12] add ls.yml --- tests/fixtures/ls.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/fixtures/ls.yml diff --git a/tests/fixtures/ls.yml b/tests/fixtures/ls.yml new file mode 100644 index 00000000..b515f50d --- /dev/null +++ b/tests/fixtures/ls.yml @@ -0,0 +1,4 @@ +type: workflow +name: ls +tasks: + ls: \ No newline at end of file From deb6b426080585e67c23d3e97b92979ee72dd603 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Thu, 2 May 2024 09:23:29 -0400 Subject: [PATCH 03/12] update --- tests/unit/test_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_template.py b/tests/unit/test_template.py index 575254a4..db1deb98 100644 --- a/tests/unit/test_template.py +++ b/tests/unit/test_template.py @@ -22,7 +22,7 @@ def setUp(self): severity='high', confidence='high', provider='ls', - matched_at=f'/home/osboxes/.secator/templates/test.txt', + matched_at=f'{str(self.writeable_file)}', _source='ls', ) clear_modules() From 0375f5e9bc5aceac15ab03af73e7a206e820b5a1 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Thu, 2 May 2024 11:03:36 -0400 Subject: [PATCH 04/12] update --- secator/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/secator/utils.py b/secator/utils.py index abb9df12..1fed81f5 100644 --- a/secator/utils.py +++ b/secator/utils.py @@ -162,7 +162,7 @@ def discover_internal_tasks(): def discover_external_tasks(): """Find external secator tasks.""" output = [] - sys.dont_write_bytecode = True + # sys.dont_write_bytecode = True for path in CONFIG.dirs.templates.glob('**/*.py'): try: task_name = path.stem @@ -186,7 +186,7 @@ def discover_external_tasks(): output.append(cls) except Exception as e: console.print(f'[bold red]Could not load external module {path.name}. Reason: {str(e)}.[/] ({path})') - sys.dont_write_bytecode = False + # sys.dont_write_bytecode = False return output From 2e0de76f991f32ac6f46253b7aa54b807d5cc8a4 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Thu, 2 May 2024 11:07:36 -0400 Subject: [PATCH 05/12] update --- secator/cli.py | 2 +- secator/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/secator/cli.py b/secator/cli.py index 2939c9a8..db412da0 100644 --- a/secator/cli.py +++ b/secator/cli.py @@ -1052,5 +1052,5 @@ def integration(tasks, workflows, scans, test, debug): @test.command() def coverage(): """Run coverage report.""" - cmd = f'{sys.executable} -m coverage report -m --omit=*/site-packages/*,*/tests/*' + cmd = f'{sys.executable} -m coverage report -m --omit=*/site-packages/*,*/tests/*,*/templates/*' run_test(cmd, 'coverage') diff --git a/secator/utils.py b/secator/utils.py index 1fed81f5..abb9df12 100644 --- a/secator/utils.py +++ b/secator/utils.py @@ -162,7 +162,7 @@ def discover_internal_tasks(): def discover_external_tasks(): """Find external secator tasks.""" output = [] - # sys.dont_write_bytecode = True + sys.dont_write_bytecode = True for path in CONFIG.dirs.templates.glob('**/*.py'): try: task_name = path.stem @@ -186,7 +186,7 @@ def discover_external_tasks(): output.append(cls) except Exception as e: console.print(f'[bold red]Could not load external module {path.name}. Reason: {str(e)}.[/] ({path})') - # sys.dont_write_bytecode = False + sys.dont_write_bytecode = False return output From 90a09a929dfd24ff5e4c0ae94fe702a50558b4cb Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Thu, 2 May 2024 13:38:42 -0400 Subject: [PATCH 06/12] feat(tasks): bbot integration --- secator/runners/_base.py | 10 +++ secator/tasks/bbot.py | 176 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 secator/tasks/bbot.py diff --git a/secator/runners/_base.py b/secator/runners/_base.py index 7adb6767..c81ac9f3 100644 --- a/secator/runners/_base.py +++ b/secator/runners/_base.py @@ -741,6 +741,16 @@ def _convert_item_schema(self, item): # output type based on the schema new_item = None output_types = getattr(self, 'output_types', []) + output_discriminator = getattr(self, 'output_discriminator', None) + if output_discriminator: + result = output_discriminator(item) + if result: + debug(f'Discriminated output type {result.__name__}', sub='klass.load', level=5) + output_types = [result] + else: + new_item = DotMap(item) + new_item._type = 'unknown' + return new_item debug(f'Input item: {item}', sub='klass.load', level=5) debug(f'Output types to try: {[o.__name__ for o in output_types]}', sub='klass.load', level=5) for klass in output_types: diff --git a/secator/tasks/bbot.py b/secator/tasks/bbot.py new file mode 100644 index 00000000..549cfaea --- /dev/null +++ b/secator/tasks/bbot.py @@ -0,0 +1,176 @@ +from secator.decorators import task +from secator.runners import Command +from secator.output_types import Vulnerability, Port, Url, Record, Ip, Tag + + +BBOT_MODULES = [ + "affiliates", + "ajaxpro", + "anubisdb", + "asn", + "azure_realm", + "azure_tenant", + "badsecrets", + "bevigil", + "binaryedge", + "bucket_amazon", + "bucket_azure", + "bucket_digitalocean", + "bucket_file_enum", + "bucket_firebase", + "bucket_google", + "builtwith", + "bypass403", + "c99", + "censys", + "certspotter", + "chaos", + "columbus", + "credshed", + "crobat", + "crt", + "dastardly", + "dehashed", + "digitorus", + "dnscommonsrv", + "dnsdumpster", + "dnszonetransfer", + "emailformat", + "ffuf", + "ffuf_shortnames", + "filedownload", + "fingerprintx", + "fullhunt", + "generic_ssrf", + "git", + "github_codesearch", + "github_org", + "gowitness", + "hackertarget", + "host_header", + "httpx", + "hunt", + "hunterio", + "iis_shortnames", + "internetdb", + "ip2location", + "ipneighbor", + "ipstack", + "leakix", + "masscan", + "massdns", + "myssl", + "newsletters", + "nmap", + "nsec", + "ntlm", + "nuclei", + "oauth", + "otx", + "paramminer_cookies", + "paramminer_getparams", + "paramminer_headers", + "passivetotal", + "pgp", + "postman", + "rapiddns", + "riddler", + "robots", + "secretsdb", + "securitytrails", + "shodan_dns", + "sitedossier", + "skymem", + "smuggler", + "social", + "sslcert", + "subdomain_hijack", + "subdomaincenter", + "sublist3r", + "telerik", + "threatminer", + "url_manipulation", + "urlscan", + "vhost", + "viewdns", + "virustotal", + "wafw00f", + "wappalyzer", + "wayback", + "zoomeye" +] +BBOT_MODULES_STR = ' '.join(BBOT_MODULES) + +def output_discriminator(self, item): + map_types = { + 'IP_ADDRESS': Ip, + 'PROTOCOL': Port, + 'OPEN_TCP_PORT': Port, + 'URL': Url, + 'Technology': Tag + } + type_ = item['type'] + if not type_ in map_types: + print(f'Found unsupported type {type_}') + return None + return map_types[item['type']] + + +@task() +class bbot(Command): + cmd = f'bbot -y -m {BBOT_MODULES_STR} --allow-deadly' + json_flag = '-om json' + input_flag = '-t' + file_flag = None + output_types = [Vulnerability, Port, Url, Record, Ip] + output_discriminator = output_discriminator + output_map = { + Ip: { + 'ip': lambda x: x['data'], + 'host': lambda x: x['data'], + 'alive': lambda x: True, + }, + Tag: { + 'name': lambda x: x['data']['technology'], + 'match': lambda x: x['data']['url'], + 'extra_data': { + 'host': lambda x: x['data']['host'] + }, + '_source': lambda x: 'bbot-' + x['module'] + }, + Url: { + 'url': 'data', + 'host': lambda x: x['resolved_hosts'][0], + 'status_code': lambda x: int([c.split('-')[-1] for c in x['tags'] if 'status-' in c][0]), + 'title': lambda x: [' '.join(c.split('-')[2:]) for c in x['tags'] if 'http-title-' in c][0], + '_source': lambda x: 'bbot-' + x['module'] + }, + Port: { + 'port': lambda x: x['data']['port'] if 'port' in x['data'] else x['data'].split(':')[-1], + 'ip': lambda x: x['resolved_hosts'][0], + 'state': lambda x: 'OPEN', + 'service_name': lambda x: x['data']['protocol'] if 'protocol' in x['data'] else '', + 'cpes': lambda x: [], + 'host': lambda x: x['resolved_hosts'][0], + 'extra_data': lambda x: {}, + } + } + + # @staticmethod + # def item_loader(self, line): + # import json + # item = json.loads(line) + # print(item) + # return item + +# ASN +# AZURE_TENANT +# DNS_NAME +# FINDING +# FINDING +# IP_ADDRESS +# OPEN_TCP_PORT +# ORG_STUB +# SCAN +# TECHNOLOGY +# URL From 1035d9c2ec62b25dee880c2c99a737125032f0a2 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Sun, 29 Sep 2024 11:41:51 -0400 Subject: [PATCH 07/12] update --- secator/output_types/_base.py | 8 +++- secator/runners/_base.py | 4 +- secator/tasks/bbot.py | 81 +++++++++++++++++++++++------------ 3 files changed, 62 insertions(+), 31 deletions(-) diff --git a/secator/output_types/_base.py b/secator/output_types/_base.py index 97186da7..1da05043 100644 --- a/secator/output_types/_base.py +++ b/secator/output_types/_base.py @@ -1,6 +1,7 @@ import logging import re from dataclasses import _MISSING_TYPE, dataclass, fields +from secator.rich import console logger = logging.getLogger(__name__) @@ -66,7 +67,12 @@ def load(cls, item, output_map={}): if key in output_map: mapped_key = output_map[key] if callable(mapped_key): - mapped_val = mapped_key(item) + try: + mapped_val = mapped_key(item) + except Exception as e: + pass + # console.print_exception() + # raise TypeError(f'Fail to map value for "{key}".') else: mapped_val = item.get(mapped_key) new_item[key] = mapped_val diff --git a/secator/runners/_base.py b/secator/runners/_base.py index 9412fd37..4f3a6f6a 100644 --- a/secator/runners/_base.py +++ b/secator/runners/_base.py @@ -767,8 +767,8 @@ def _convert_item_schema(self, item): f'[dim red]Failed loading item as {klass.__name__}: {type(e).__name__}: {str(e)}.[/] [dim green]Continuing.[/]', sub='klass.load', level=5) - if DEBUG == 6: - console.print_exception(show_locals=False) + # if DEBUG == 6: + console.print_exception(show_locals=False) continue # No output type was found, so make no conversion diff --git a/secator/tasks/bbot.py b/secator/tasks/bbot.py index 549cfaea..bcde49fe 100644 --- a/secator/tasks/bbot.py +++ b/secator/tasks/bbot.py @@ -5,7 +5,7 @@ BBOT_MODULES = [ "affiliates", - "ajaxpro", + # "ajaxpro", "anubisdb", "asn", "azure_realm", @@ -13,24 +13,24 @@ "badsecrets", "bevigil", "binaryedge", - "bucket_amazon", + "bucket_aws", "bucket_azure", "bucket_digitalocean", - "bucket_file_enum", + # "bucket_file_enum", "bucket_firebase", - "bucket_google", + "bucket_gcp", "builtwith", "bypass403", "c99", "censys", "certspotter", - "chaos", + # "chaos", "columbus", - "credshed", + # "credshed", "crobat", "crt", - "dastardly", - "dehashed", + # "dastardly", + # "dehashed", "digitorus", "dnscommonsrv", "dnsdumpster", @@ -38,13 +38,14 @@ "emailformat", "ffuf", "ffuf_shortnames", - "filedownload", + # "filedownload", "fingerprintx", "fullhunt", "generic_ssrf", "git", - "github_codesearch", - "github_org", + # "github_codesearch", + "github", + # "github_org", "gowitness", "hackertarget", "host_header", @@ -52,15 +53,15 @@ "hunt", "hunterio", "iis_shortnames", - "internetdb", - "ip2location", + # "internetdb", + # "ip2location", "ipneighbor", "ipstack", "leakix", "masscan", "massdns", "myssl", - "newsletters", + # "newsletters", "nmap", "nsec", "ntlm", @@ -72,7 +73,7 @@ "paramminer_headers", "passivetotal", "pgp", - "postman", + # "postman", "rapiddns", "riddler", "robots", @@ -94,7 +95,7 @@ "vhost", "viewdns", "virustotal", - "wafw00f", + # "wafw00f", "wappalyzer", "wayback", "zoomeye" @@ -107,21 +108,30 @@ def output_discriminator(self, item): 'PROTOCOL': Port, 'OPEN_TCP_PORT': Port, 'URL': Url, - 'Technology': Tag + 'TECHNOLOGY': Tag, + # 'DNS_NAME': Record, + 'VULNERABILITY': Vulnerability, + 'FINDING': Tag } - type_ = item['type'] + type_ = item.get('type') if not type_ in map_types: - print(f'Found unsupported type {type_}') + self._print(f'Found unsupported bbot type: {type_}', 'bold orange3') return None return map_types[item['type']] @task() class bbot(Command): - cmd = f'bbot -y -m {BBOT_MODULES_STR} --allow-deadly' + cmd = f'bbot -y --allow-deadly' json_flag = '-om json' input_flag = '-t' file_flag = None + opts = { + 'modules': {'type': str, 'short': 'm', 'default': ','.join(BBOT_MODULES)} + } + opt_key_map = { + 'modules': 'm' + } output_types = [Vulnerability, Port, Url, Record, Ip] output_discriminator = output_discriminator output_map = { @@ -131,11 +141,9 @@ class bbot(Command): 'alive': lambda x: True, }, Tag: { - 'name': lambda x: x['data']['technology'], + 'name': lambda x: x['data'].get('technology') or x['data']['description'], 'match': lambda x: x['data']['url'], - 'extra_data': { - 'host': lambda x: x['data']['host'] - }, + 'extra_data': lambda x: {'host': x['data']['host']}, '_source': lambda x: 'bbot-' + x['module'] }, Url: { @@ -146,14 +154,31 @@ class bbot(Command): '_source': lambda x: 'bbot-' + x['module'] }, Port: { - 'port': lambda x: x['data']['port'] if 'port' in x['data'] else x['data'].split(':')[-1], - 'ip': lambda x: x['resolved_hosts'][0], + 'port': lambda x: int(x['data']['port']) if 'port' in x['data'] else x['data'].split(':')[-1], + 'ip': lambda x: [_ for _ in x['resolved_hosts'] if not _.startswith('::')][0], 'state': lambda x: 'OPEN', 'service_name': lambda x: x['data']['protocol'] if 'protocol' in x['data'] else '', 'cpes': lambda x: [], - 'host': lambda x: x['resolved_hosts'][0], + 'host': lambda x: x['data']['host'] if isinstance(x['data'], dict) else x['data'], 'extra_data': lambda x: {}, - } + }, + # Vulnerability: { + # 'name': + # 'provider': + # 'id': + # 'matched_at': + # 'ip': + # 'confidence': + # 'severity': + # 'cvss_score': + # 'tags': + # 'extra_data': + # 'description': + # 'references': + # 'reference': + # 'confidence_nb': + # 'severity_nb': + # } } # @staticmethod From cc92a4386027ff6573838c4db1a1eebc836d3e51 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 30 Sep 2024 06:43:47 -0400 Subject: [PATCH 08/12] update --- secator/output_types/_base.py | 2 + secator/serializers/regex.py | 3 + secator/tasks/bbot.py | 135 +++++++++++++++++----------------- 3 files changed, 74 insertions(+), 66 deletions(-) diff --git a/secator/output_types/_base.py b/secator/output_types/_base.py index 1da05043..26cb2de5 100644 --- a/secator/output_types/_base.py +++ b/secator/output_types/_base.py @@ -70,6 +70,8 @@ def load(cls, item, output_map={}): try: mapped_val = mapped_key(item) except Exception as e: + mapped_val = None + console.print(f'Fail to map value for "{key}" on type "{_type}.') pass # console.print_exception() # raise TypeError(f'Fail to map value for "{key}".') diff --git a/secator/serializers/regex.py b/secator/serializers/regex.py index f875d9e0..215a679c 100644 --- a/secator/serializers/regex.py +++ b/secator/serializers/regex.py @@ -12,6 +12,9 @@ def run(self, line): output = {} if not match: return + if not self.fields: + yield match.group(0) + return for field in self.fields: output[field] = match.group(field) yield output diff --git a/secator/tasks/bbot.py b/secator/tasks/bbot.py index bcde49fe..da7aa8bc 100644 --- a/secator/tasks/bbot.py +++ b/secator/tasks/bbot.py @@ -1,5 +1,6 @@ from secator.decorators import task from secator.runners import Command +from secator.serializers import RegexSerializer from secator.output_types import Vulnerability, Port, Url, Record, Ip, Tag @@ -13,12 +14,12 @@ "badsecrets", "bevigil", "binaryedge", - "bucket_aws", + # "bucket_aws", "bucket_azure", "bucket_digitalocean", # "bucket_file_enum", "bucket_firebase", - "bucket_gcp", + "bucket_google", "builtwith", "bypass403", "c99", @@ -27,14 +28,14 @@ # "chaos", "columbus", # "credshed", - "crobat", + # "crobat", "crt", # "dastardly", # "dehashed", "digitorus", "dnscommonsrv", "dnsdumpster", - "dnszonetransfer", + # "dnszonetransfer", "emailformat", "ffuf", "ffuf_shortnames", @@ -44,8 +45,7 @@ "generic_ssrf", "git", # "github_codesearch", - "github", - # "github_org", + "github_org", "gowitness", "hackertarget", "host_header", @@ -58,12 +58,12 @@ "ipneighbor", "ipstack", "leakix", - "masscan", - "massdns", + # "masscan", + # "massdns", "myssl", # "newsletters", - "nmap", - "nsec", + # "nmap", + # "nsec", "ntlm", "nuclei", "oauth", @@ -75,7 +75,7 @@ "pgp", # "postman", "rapiddns", - "riddler", + # "riddler", "robots", "secretsdb", "securitytrails", @@ -85,11 +85,11 @@ "smuggler", "social", "sslcert", - "subdomain_hijack", + # "subdomain_hijack", "subdomaincenter", - "sublist3r", + # "sublist3r", "telerik", - "threatminer", + # "threatminer", "url_manipulation", "urlscan", "vhost", @@ -101,29 +101,34 @@ "zoomeye" ] BBOT_MODULES_STR = ' '.join(BBOT_MODULES) +BBOT_MAP_TYPES = { + 'IP_ADDRESS': Ip, + 'PROTOCOL': Port, + 'OPEN_TCP_PORT': Port, + 'URL': Url, + 'TECHNOLOGY': Tag, + # 'DNS_NAME': Record, + 'VULNERABILITY': Vulnerability, + 'FINDING': Tag +} +NUCLEI_DATA_REGEX = RegexSerializer( + regex=r'template: \[(?P