From 202b1948226c4c2bfcbc76d57e84fe6f06db5ad8 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Oct 2022 15:34:19 -0700 Subject: [PATCH 1/9] feat: yaml support --- pocsuite3/api/__init__.py | 48 +- pocsuite3/lib/core/common.py | 9 +- pocsuite3/lib/core/interpreter.py | 35 +- pocsuite3/lib/core/option.py | 7 +- pocsuite3/lib/core/register.py | 6 + pocsuite3/lib/core/settings.py | 2 - pocsuite3/lib/yaml/__init__.py | 0 pocsuite3/lib/yaml/nuclei/__init__.py | 216 +++++ pocsuite3/lib/yaml/nuclei/model/__init__.py | 37 + .../lib/yaml/nuclei/operators/__init__.py | 32 + .../nuclei/operators/extrators/__init__.py | 150 ++++ .../nuclei/operators/matchers/__init__.py | 154 ++++ .../lib/yaml/nuclei/protocols/__init__.py | 0 .../protocols/common/expressions/__init__.py | 793 ++++++++++++++++++ .../protocols/common/replacer/__init__.py | 24 + .../yaml/nuclei/protocols/http/__init__.py | 273 ++++++ .../lib/yaml/nuclei/templates/__init__.py | 28 + tests/test_nuclei_helper_functions.py | 190 +++++ 18 files changed, 1958 insertions(+), 46 deletions(-) create mode 100644 pocsuite3/lib/yaml/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/model/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/operators/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/protocols/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/templates/__init__.py create mode 100644 tests/test_nuclei_helper_functions.py diff --git a/pocsuite3/api/__init__.py b/pocsuite3/api/__init__.py index 7a39b320..a021137a 100644 --- a/pocsuite3/api/__init__.py +++ b/pocsuite3/api/__init__.py @@ -1,10 +1,25 @@ +import base64 +import binascii +import collections +import json +import os +import re +import socket +import ssl +import struct +import textwrap +import time +import urllib +import zlib + from pocsuite3.lib.controller.controller import start -from pocsuite3.lib.core.common import (encoder_bash_payload, check_port, +from pocsuite3.lib.core.common import (OrderedDict, OrderedSet, check_port, + encoder_bash_payload, encoder_powershell_payload, get_host_ip, - get_host_ipv6, single_time_warn_message) + get_host_ipv6, mosaic, + single_time_warn_message, urlparse) from pocsuite3.lib.core.data import conf, kb, logger, paths from pocsuite3.lib.core.datatype import AttribDict -from pocsuite3.lib.core.common import OrderedSet, OrderedDict, mosaic, urlparse from pocsuite3.lib.core.enums import PLUGIN_TYPE, POC_CATEGORY, VUL_TYPE from pocsuite3.lib.core.interpreter_option import (OptBool, OptDict, OptFloat, OptInteger, OptIP, OptItems, @@ -17,37 +32,24 @@ from pocsuite3.lib.core.settings import DEFAULT_LISTENER_PORT from pocsuite3.lib.request import requests from pocsuite3.lib.utils import (generate_shellcode_list, get_middle_text, - random_str, minimum_version_required) + minimum_version_required, random_str) +from pocsuite3.lib.yaml.nuclei import Nuclei from pocsuite3.modules.censys import Censys from pocsuite3.modules.ceye import CEye from pocsuite3.modules.fofa import Fofa from pocsuite3.modules.httpserver import PHTTPServer -from pocsuite3.modules.listener import (REVERSE_PAYLOAD, BIND_PAYLOAD, bind_shell, - bind_tcp_shell, bind_telnet_shell) -from pocsuite3.modules.quake import Quake from pocsuite3.modules.hunter import Hunter +from pocsuite3.modules.interactsh import Interactsh +from pocsuite3.modules.listener import (BIND_PAYLOAD, REVERSE_PAYLOAD, + bind_shell, bind_tcp_shell, + bind_telnet_shell) +from pocsuite3.modules.quake import Quake from pocsuite3.modules.seebug import Seebug from pocsuite3.modules.shodan import Shodan from pocsuite3.modules.spider import crawl from pocsuite3.modules.zoomeye import ZoomEye -from pocsuite3.modules.interactsh import Interactsh from pocsuite3.shellcodes import OSShellcodes, WebShell -__all__ = ('requests', 'PluginBase', 'register_plugin', 'PLUGIN_TYPE', - 'POCBase', 'Output', 'AttribDict', 'POC_CATEGORY', 'VUL_TYPE', - 'register_poc', 'conf', 'kb', 'logger', 'paths', 'minimum_version_required', - 'DEFAULT_LISTENER_PORT', 'load_file_to_module', 'OrderedDict', 'OrderedSet', - 'load_string_to_module', 'single_time_warn_message', 'CEye', - 'Seebug', 'ZoomEye', 'Shodan', 'Fofa', 'Quake', 'Hunter', 'Censys', - 'PHTTPServer', 'REVERSE_PAYLOAD', 'BIND_PAYLOAD', 'get_listener_ip', 'mosaic', - 'urlparse', 'get_listener_port', 'get_results', 'init_pocsuite', - 'start_pocsuite', 'get_poc_options', 'crawl', 'OSShellcodes', - 'WebShell', 'OptDict', 'OptIP', 'OptPort', 'OptBool', 'OptInteger', - 'OptFloat', 'OptString', 'OptItems', 'get_middle_text', - 'generate_shellcode_list', 'random_str', 'encoder_bash_payload', 'check_port', - 'encoder_powershell_payload', 'get_host_ip', 'get_host_ipv6', 'bind_shell', - 'bind_tcp_shell', 'bind_telnet_shell', 'Interactsh') - def get_listener_ip(): return conf.connect_back_host diff --git a/pocsuite3/lib/core/common.py b/pocsuite3/lib/core/common.py index c9f154c0..b2ea3fec 100644 --- a/pocsuite3/lib/core/common.py +++ b/pocsuite3/lib/core/common.py @@ -39,7 +39,6 @@ from pocsuite3.lib.core.settings import IP_ADDRESS_REGEX from pocsuite3.lib.core.settings import OLD_VERSION_CHARACTER from pocsuite3.lib.core.settings import POCSUITE_VERSION_CHARACTER -from pocsuite3.lib.core.settings import POC_NAME_REGEX from pocsuite3.lib.core.settings import POC_REQUIRES_REGEX from pocsuite3.lib.core.settings import UNICODE_ENCODING from pocsuite3.lib.core.settings import URL_ADDRESS_REGEX @@ -576,7 +575,11 @@ def get_poc_requires(code): def get_poc_name(code): - return extract_regex_result(POC_NAME_REGEX, code) + if re.search(r'register_poc', code): + return extract_regex_result(r"""(?sm)POCBase\):.*?name\s*=\s*['"](?P.*?)['"]""", code) + elif re.search(r'matchers:\s*-', code): + return extract_regex_result(r"""(?sm)\s*name\s*:\s*(?P[^\n]*).*matchers:""", code) + return '' def is_os_64bit(): @@ -897,7 +900,7 @@ def index_modules(modules_directory): modules = [] for root, _, files in os.walk(modules_directory): - files = filter(lambda x: not x.startswith("__") and x.endswith(".py"), files) + files = filter(lambda x: not x.startswith("__") and x.endswith(".py") or x.endswith(".yaml"), files) modules.extend(map(lambda x: os.path.join(root, os.path.splitext(x)[0]), files)) return modules diff --git a/pocsuite3/lib/core/interpreter.py b/pocsuite3/lib/core/interpreter.py index be9497d1..c30d7eee 100644 --- a/pocsuite3/lib/core/interpreter.py +++ b/pocsuite3/lib/core/interpreter.py @@ -1,8 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# @Time : 2018/12/25 上午10:58 -# @Author : chenghs -# @File : interpreter.py import os import re import chardet @@ -325,19 +320,27 @@ def command_use(self, module_path, *args, **kwargs): logger.warning("Index out of range") return module_path = self.last_search[index] - if not module_path.endswith(".py"): - module_path = module_path + ".py" - if not os.path.exists(module_path): - module_path = os.path.join(self.module_parent_directory, module_path) - if not os.path.exists(module_path): - errMsg = "No such file: '{0}'".format(module_path) - logger.error(errMsg) - return + + module_ext = '' + module_path_found = False + for module_ext in ['.py', '.yaml']: + if os.path.exists(module_path + module_ext): + module_path_found = True + break + elif os.path.exists(os.path.join(self.module_parent_directory, module_path + module_ext)): + module_path_found = True + module_path = os.path.join(self.module_parent_directory, module_path + module_ext) + break + + if not module_path_found: + errMsg = "No such file: '{0}'".format(module_path) + logger.error(errMsg) + return + try: load_file_to_module(module_path) self.current_module = kb.current_poc - self.current_module.pocsuite3_module_path = ltrim( - rtrim(module_path, ".py"), self.module_parent_directory) + self.current_module.pocsuite3_module_path = ltrim(rtrim(module_path, module_ext), self.module_parent_directory) except Exception as err: logger.error(str(err)) @@ -457,6 +460,8 @@ def command_list(self, *args, **kwargs): index = 0 for tmp_module in self.main_modules_dirs: found = os.path.join(self.module_parent_directory, tmp_module + ".py") + if not os.path.exists(found): + found = os.path.join(self.module_parent_directory, tmp_module + ".yaml") code = get_file_text(found) name = get_poc_name(code) tb.add_row([str(index), tmp_module, name]) diff --git a/pocsuite3/lib/core/option.py b/pocsuite3/lib/core/option.py index d562b62b..3be818c9 100644 --- a/pocsuite3/lib/core/option.py +++ b/pocsuite3/lib/core/option.py @@ -338,7 +338,7 @@ def _set_pocs_modules(): elif any([poc in exists_poc_with_ext, poc in exists_pocs]): poc_name, poc_ext = os.path.splitext(poc) - if poc_ext in ['.py', '.pyc']: + if poc_ext in ['.py', '.pyc', '.yaml']: file_path = os.path.join(paths.POCSUITE_POCS_PATH, poc) else: file_path = os.path.join(paths.POCSUITE_POCS_PATH, poc + exists_pocs.get(poc)) @@ -346,12 +346,13 @@ def _set_pocs_modules(): elif check_path(poc): for root, _, files in os.walk(poc): - files = filter(lambda x: not x.startswith("__") and x.endswith(".py"), files) + files = filter(lambda x: not x.startswith("__") and x.endswith(".py") or + x.endswith('.yaml'), files) _pocs.extend(map(lambda x: os.path.join(root, x), files)) for p in _pocs: file_content = get_file_text(p) - if not re.search(r'register_poc', file_content): + if not re.search(r'register_poc|matchers:\s+-', file_content): continue if conf.poc_keyword: if not re.search(conf.poc_keyword, file_content, re.I | re.M): diff --git a/pocsuite3/lib/core/register.py b/pocsuite3/lib/core/register.py index f050d632..6ffbaae5 100644 --- a/pocsuite3/lib/core/register.py +++ b/pocsuite3/lib/core/register.py @@ -9,6 +9,7 @@ from pocsuite3.lib.core.data import kb from pocsuite3.lib.core.data import logger from pocsuite3.lib.core.settings import POC_IMPORTDICT +from pocsuite3.lib.yaml.nuclei import Nuclei class PocLoader(Loader): @@ -68,6 +69,11 @@ def check_requires(data): def exec_module(self, module): filename = self.get_filename(self.fullname) poc_code = self.get_data(filename) + + # convert yaml template to pocsuite3 poc script + if filename.endswith('.yaml') and re.search(r'matchers:\s+-', poc_code): + poc_code = str(Nuclei(poc_code)) + self.check_requires(poc_code) obj = compile(poc_code, filename, 'exec', dont_inherit=True, optimize=-1) try: diff --git a/pocsuite3/lib/core/settings.py b/pocsuite3/lib/core/settings.py index df4731fe..71c33090 100644 --- a/pocsuite3/lib/core/settings.py +++ b/pocsuite3/lib/core/settings.py @@ -93,8 +93,6 @@ POC_REQUIRES_REGEX = r"install_requires\s*=\s*\[(?P.*?)\]" -POC_NAME_REGEX = r"""(?sm)POCBase\):.*?name\s*=\s*['"](?P.*?)['"]""" - MAX_NUMBER_OF_THREADS = 200 DEFAULT_LISTENER_PORT = 6666 diff --git a/pocsuite3/lib/yaml/__init__.py b/pocsuite3/lib/yaml/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pocsuite3/lib/yaml/nuclei/__init__.py b/pocsuite3/lib/yaml/nuclei/__init__.py new file mode 100644 index 00000000..ec1f32a2 --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/__init__.py @@ -0,0 +1,216 @@ +import re +from collections import OrderedDict + +import dacite +import yaml +from requests_toolbelt.utils import dump + +from pocsuite3.lib.core.common import urlparse +from pocsuite3.lib.core.log import LOGGER as logger +from pocsuite3.lib.request import requests +from pocsuite3.lib.utils import random_str +from pocsuite3.lib.yaml.nuclei.model import Severify +from pocsuite3.lib.yaml.nuclei.operators import ExtractorType, MatcherType +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import Evaluate +from pocsuite3.lib.yaml.nuclei.protocols.http import (AttackType, HttpExtract, + HttpMatch, HTTPMethod, + HttpRequest, + httpRequestGenerator) +from pocsuite3.lib.yaml.nuclei.templates import Template + + +def hyphen_to_underscore(dictionary): + """ + Takes an Array or dictionary and replace all the hyphen('-') in any of its keys with a underscore('_') + :param dictionary: + :return: the same object with all hyphens replaced by underscore + """ + # By default return the same object + final_dict = dictionary + + # for Array perform this method on every object + if isinstance(dictionary, list): + final_dict = [] + for item in dictionary: + final_dict.append(hyphen_to_underscore(item)) + + # for dictionary traverse all the keys and replace hyphen with underscore + elif isinstance(dictionary, dict): + final_dict = {} + for k, v in dictionary.items(): + # If there is a sub dictionary or an array perform this method of it recursively + if isinstance(dictionary[k], (dict, list)): + value = hyphen_to_underscore(v) + final_dict[k.replace('-', '_')] = value + else: + final_dict[k.replace('-', '_')] = v + + return final_dict + + +def expand_preprocessors(data: str) -> str: + """ + Certain pre-processors can be specified globally anywhere in the template that run as soon as + the template is loaded to achieve things like random ids generated for each template run. + + randstr can be suffixed by a number, and new random ids will be created for those names too. + Ex. {{randstr_1}} which will remain same across the template. + randstr is also supported within matchers and can be used to match the inputs. + """ + randstr_to_replace = set(m[0] for m in re.findall(r'({{randstr(_\w+)?}})', data)) + for s in randstr_to_replace: + data = data.replace(s, random_str(27)) + + return data + + +class Nuclei(): + def __init__(self, template, target=''): + self.yaml_template = template + self.json_template = yaml.safe_load(expand_preprocessors(self.yaml_template)) + self.template = dacite.from_dict( + Template, hyphen_to_underscore(self.json_template), + config=dacite.Config(cast=[Severify, ExtractorType, MatcherType, HTTPMethod, AttackType])) + + self.target = target + + self.execute_options = OrderedDict() + self.execute_options['stop_at_first_match'] = self.template.stop_at_first_match + self.execute_options['variables'] = self.template.variables + + self.requests = self.template.requests + + self.dynamic_values = OrderedDict() + + def execute_request(self, request: HttpRequest) -> dict: + results = [] + with requests.Session() as session: + try: + for (method, url, kwargs) in httpRequestGenerator(request, self.dynamic_values): + try: + """ + Redirection conditions can be specified per each template. By default, redirects are not followed. + However, if desired, they can be enabled with redirects: true in request details. + 10 redirects are followed at maximum by default which should be good enough for most use cases. + More fine grained control can be exercised over number of redirects followed by using max-redirects + field. + """ + if request.max_redirects: + session.max_redirects = request.max_redirects + else: + session.max_redirects = 10 + response = session.request(method=method, url=url, **kwargs) + logger.debug(dump.dump_all(response).decode('utf-8')) + except Exception as e1: + logger.debug(str(e1)) + continue + match_res = HttpMatch(request, response) + extractor_res = HttpExtract(request, response) + if match_res and extractor_res: + match_res = str(dict(extractor_res[0])) + if match_res and request.stop_at_first_match: + return match_res + results.append(match_res) + response.close() + except Exception as e: + logger.debug(str(e)) + return results and any(results) + + def execute_template(self): + ''' + Dynamic variables can be placed in the path to modify its behavior on runtime. + Variables start with {{ and end with }} and are case-sensitive. + ''' + + u = urlparse(self.target) + self.dynamic_values['BaseURL'] = self.target + self.dynamic_values['RootURL'] = f'{u.scheme}://{u.netloc}' + self.dynamic_values['Hostname'] = u.netloc + self.dynamic_values['Scheme'] = u.scheme + self.dynamic_values['Host'] = u.hostname + self.dynamic_values['Port'] = u.port + self.dynamic_values['Path'] = '/'.join(u.path.split('/')[0:-1]) + self.dynamic_values['File'] = u.path.split('/')[-1] + + """ + Variables can be used to declare some values which remain constant throughout the template. + The value of the variable once calculated does not change. + Variables can be either simple strings or DSL helper functions. If the variable is a helper function, + it is enclosed in double-curly brackets {{}}. Variables are declared at template level. + + Example variables: + + variables: + a1: "test" # A string variable + a2: "{{to_lower(rand_base(5))}}" # A DSL function variable + """ + for k, v in self.execute_options['variables'].items(): + self.dynamic_values[k] = Evaluate(v) + + """ + Since release of Nuclei v2.3.6, Nuclei supports using the interact.sh API to achieve OOB based vulnerability scanning + with automatic Request correlation built in. It's as easy as writing {{interactsh-url}} anywhere in the request. + """ + if '{{interactsh-url}}' in self.yaml_template or '§interactsh-url§' in self.yaml_template: + from pocsuite3.modules.interactsh import Interactsh + ish = Interactsh() + ish_url, ish_flag = ish.build_request(method='') + self.dynamic_values['interactsh-url'] = ish_url + self.execute_options['interactsh_client'] = ish + + results = [] + for request in self.requests: + res = self.execute_request(request) + results.append(res) + if self.execute_options['stop_at_first_match'] and res: + return res + return all(results) + + def run(self): + return self.execute_template() + + def __str__(self): + ''' + Convert nuclei template to pocsuite3 + ''' + info = [] + key_convert = { + 'description': 'desc', + 'reference': 'references' + } + for k, v in self.json_template['info'].items(): + if k in key_convert: + k = key_convert.get(k) + if type(v) in [str]: + v = f'\'{v.strip()}\'' + if k == 'desc': + v = f'\'\'{v}\'\'' + + info.append(f' {k} = {v}') + + poc_code = [ + 'from pocsuite3.api import POCBase, Nuclei, register_poc\n', + '\n', + '\n', + 'class TestPOC(POCBase):\n', + '\n'.join(info), + '\n', + ' def _verify(self):\n', + ' result = {}\n', + ' if not self._check():\n', + ' return self.parse_output(result)\n', + " template = '''%s'''\n" % self.yaml_template, + ' res = Nuclei(template, self.url).run()\n', + ' if res:\n', + ' result["VerifyInfo"] = {}\n', + ' result["VerifyInfo"]["URL"] = self.url\n', + ' result["VerifyInfo"]["Info"] = {}\n', + ' result["VerifyInfo"]["Info"]["Severity"] = "%s"\n' % self.template.info.severity.value, + ' if not isinstance(res, bool):\n' + ' result["VerifyInfo"]["Info"]["Result"] = {}\n', + ' return self.parse_output(result)\n', + '\n', + '\n', + 'register_poc(TestPOC)\n' + ] + return ''.join(poc_code) diff --git a/pocsuite3/lib/yaml/nuclei/model/__init__.py b/pocsuite3/lib/yaml/nuclei/model/__init__.py new file mode 100644 index 00000000..78d32c75 --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/model/__init__.py @@ -0,0 +1,37 @@ +from dataclasses import dataclass, field +from enum import Enum +from typing import NewType, Union + +StrSlice = NewType('StrSlice', Union[str, list]) + + +class Severify(Enum): + Info = 'info' + Low = 'low' + Medium = 'medium' + High = 'high' + Critical = 'critical' + Unknown = 'unknown' + + +# Classification contains the vulnerability classification data for a template. +@dataclass +class Classification: + cve_id: StrSlice = field(default_factory=list) + cwe_id: StrSlice = field(default_factory=list) + cvss_metrics: str = '' + cvss_score: float = 0.0 + + +# Info contains metadata information abount a template +@dataclass +class Info: + name: str = '' + author: StrSlice = field(default_factory=list) + tags: StrSlice = field(default_factory=list) + description: str = '' + reference: StrSlice = field(default_factory=list) + severity: Severify = 'unknown' + metadata: dict = field(default_factory=dict) + classification: Classification = field(default_factory=Classification) + remediation: str = '' diff --git a/pocsuite3/lib/yaml/nuclei/operators/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/__init__.py new file mode 100644 index 00000000..300ef427 --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/operators/__init__.py @@ -0,0 +1,32 @@ +from pocsuite3.lib.yaml.nuclei.operators.extrators import (ExtractDSL, + ExtractJSON, + ExtractKval, + Extractor, + ExtractorType, + ExtractRegex, + ExtractXPath) +from pocsuite3.lib.yaml.nuclei.operators.matchers import (MatchBinary, + MatchDSL, Matcher, + MatcherType, + MatchRegex, + MatchSize, + MatchStatusCode, + MatchWords) + +__all__ = [ + "ExtractorType", + "Extractor", + "ExtractRegex", + "ExtractKval", + "ExtractXPath", + "ExtractJSON", + "ExtractDSL", + "Matcher", + "MatcherType", + "MatchStatusCode", + "MatchSize", + "MatchWords", + "MatchRegex", + "MatchBinary", + "MatchDSL", +] diff --git a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py new file mode 100644 index 00000000..6fdc9714 --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py @@ -0,0 +1,150 @@ +import re +from collections import defaultdict +from dataclasses import dataclass, field +from enum import Enum + +import jq +import lxml +import requests + +from pocsuite3.lib.core.common import OrderedSet + + +class ExtractorType(Enum): + RegexExtractor = "regex" + KValExtractor = "kval" + XPathExtractor = "xpath" + JSONExtractor = "json" + DSLExtractor = "dsl" + + +# Extractor is used to extract part of response using a regex. +@dataclass +class Extractor: + # Name of the extractor. Name should be lowercase and must not contain spaces or underscores (_). + name: str = '' + + # Type is the type of the extractor. + type: ExtractorType = 'regex' + + # Regex contains the regular expression patterns to extract from a part. + regex: list[str] = field(default_factory=list) + + # Group specifies a numbered group to extract from the regex. + group: int = 0 + + # kval contains the key-value pairs present in the HTTP response header. + kval: list[str] = field(default_factory=list) + + # JSON allows using jq-style syntax to extract items from json response + json: list[str] = field(default_factory=list) + + # XPath allows using xpath expressions to extract items from html response + xpath: list[str] = field(default_factory=list) + + # Attribute is an optional attribute to extract from response XPath. + attribute: str = '' + + # Part is the part of the request response to extract data from. + part: str = '' + + # Internal, when set to true will allow using the value extracted in the next request for some protocols (like HTTP). + internal: bool = False + + # CaseInsensitive enables case-insensitive extractions. Default is false. + case_insensitive: bool = False + + +def ExtractRegex(e: Extractor, corpus: str) -> defaultdict: + """Extract data from response based on a Regular Expression. + """ + results = defaultdict(OrderedSet) + for regex in e.regex: + matches = re.search(regex, corpus) + if not matches: + continue + + lastindex = matches.lastindex + + group = e.group if lastindex and lastindex >= e.group else 0 + res = matches.group(group) + if not res: + continue + + if e.name: + results[e.name].add(res) + else: + results['ExtraInfo'].add(res) + return results + + +def ExtractKval(e: Extractor, headers: requests.structures.CaseInsensitiveDict) -> defaultdict: + """Extract key: value/key=value formatted data from Response Header/Cookie + """ + results = defaultdict(OrderedSet) + for k in e.kval: + res = '' + if k in headers: + res = headers[k] + # kval extractor does not accept dash (-) as input and must be substituted with underscore (_) + elif k.replace('_', '-') in headers: + res = headers[k.replace('_', '-')] + if not res: + continue + + if e.name: + results[e.name].add(res) + else: + results['ExtraInfo'].add(res) + return results + + +def ExtractXPath(e: Extractor, corpus: str) -> defaultdict: + """A xpath extractor example to extract value of href attribute from HTML response + """ + results = defaultdict(OrderedSet) + if corpus.startswith(' defaultdict: + """Extract data from JSON based response in JQ like syntax + """ + results = defaultdict(OrderedSet) + for j in e.json: + res = jq.compile(j).input(corpus).all() + if not res: + continue + + if e.name: + results[e.name].add(res) + else: + results['ExtraInfo'].add(res) + + return results + + +def ExtractDSL(e: Extractor, data: dict) -> defaultdict: + """Extract data from the response based on a DSL expressions + """ + # TODO + raise NotImplementedError diff --git a/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py new file mode 100644 index 00000000..9fed53c6 --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py @@ -0,0 +1,154 @@ +import binascii +import re +from dataclasses import dataclass, field +from enum import Enum + +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import Evaluate + + +class MatcherType(Enum): + StatusMatcher = "status" + SizeMatcher = "size" + WordsMatcher = "word" + RegexMatcher = "regex" + BinaryMatcher = "binary" + DSLMatcher = "dsl" + + +# Matcher is used to match a part in the output from a protocol. +@dataclass +class Matcher: + # Type is the type of the matcher. + type: MatcherType = 'word' + + # Condition is the optional condition between two matcher variables. By default, the condition is assumed to be OR. + condition: str = 'or' + + # Part is the part of the request response to match data from. Each protocol exposes a lot of different parts. + # default matched part is body if not defined. + part: str = 'body' + + # Negative specifies if the match should be reversed. It will only match if the condition is not true. + negative: bool = False + + # Name of the matcher. Name should be lowercase and must not contain spaces or underscores (_). + name: str = '' + + # Status are the acceptable status codes for the response. + status: list[int] = field(default_factory=list) + + # Size is the acceptable size for the response + size: list[int] = field(default_factory=list) + + # Words contains word patterns required to be present in the response part. + words: list[str] = field(default_factory=list) + + # Regex contains Regular Expression patterns required to be present in the response part. + regex: list[str] = field(default_factory=list) + + # Binary are the binary patterns required to be present in the response part. + binary: list[str] = field(default_factory=list) + + # DSL are the dsl expressions that will be evaluated as part of nuclei matching rules. + dsl: list[str] = field(default_factory=list) + + # Encoding specifies the encoding for the words field if any. + encoding: str = '' + + # CaseInsensitive enables case-insensitive matches. Default is false. + case_insensitive: bool = False + + # MatchAll enables matching for all matcher values. Default is false. + match_all: bool = False + + +def MatchStatusCode(matcher: Matcher, statusCode: int): + """MatchStatusCode matches a status code check against a corpus + """ + return statusCode in matcher.status + + +def MatchSize(matcher: Matcher, length: int): + """MatchSize matches a size check against a corpus + """ + return length in matcher.size + + +def MatchWords(matcher: Matcher, corpus: str, data: dict) -> (bool, list): + """MatchWords matches a word check against a corpus + """ + if matcher.case_insensitive: + corpus = corpus.lower() + + matchedWords = [] + for i, word in enumerate(matcher.words): + word = Evaluate(word, data) + + if word not in corpus: + if matcher.condition == 'and': + return False, [] + elif matcher.condition == 'or': + continue + + if matcher.condition == 'or' and not matcher.match_all: + return True, [word] + + matchedWords.append(word) + + if len(matcher.words) - 1 == i and not matcher.match_all: + return True, MatchWords + + if len(matchedWords) > 0 and matcher.match_all: + return True, MatchWords + + return False, [] + + +def MatchRegex(matcher: Matcher, corpus: str) -> (bool, list): + """MatchRegex matches a regex check against a corpus + """ + matchedRegexes = [] + for i, regex in enumerate(matcher.regex): + if not re.search(regex, corpus): + if matcher.condition == 'and': + return False, [] + elif matcher.condition == 'or': + continue + currentMatches = re.findall(regex, corpus) + if matcher.condition == 'or' and not matcher.match_all: + return True, matchedRegexes + + matchedRegexes = matchedRegexes + currentMatches + if len(matcher.regex) - 1 == i and not matcher.match_all: + return True, matchedRegexes + if len(matchedRegexes) > 0 and matcher.match_all: + return True, matchedRegexes + + return False, [] + + +def MatchBinary(matcher: Matcher, corpus: bytes) -> (bool, list): + """MatchBinary matches a binary check against a corpus + """ + matchedBinary = [] + for i, binary in enumerate(matcher.binary): + binary = binascii.unhexlify(binary) + if binary not in corpus: + if matcher.condition == 'and': + return False, [] + elif matcher.condition == 'or': + continue + if matcher.condition == 'or': + return True, [binary] + MatchBinary.append(binary) + if len(matcher.binary) - 1 == i: + return True, matchedBinary + return False, [] + + +def MatchDSL(matcher: Matcher, data: dict) -> bool: + """MatchDSL matches on a generic map result + """ + + # TODO + raise NotImplementedError diff --git a/pocsuite3/lib/yaml/nuclei/protocols/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py new file mode 100644 index 00000000..29dc4992 --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py @@ -0,0 +1,793 @@ +import base64 as py_built_in_base64 +import binascii +import datetime +import gzip as py_built_in_gzip +import hashlib +import hmac as py_hmac +import html +import random +import re +import string +import time +import urllib.parse +import zlib as py_built_in_zlib +from collections import OrderedDict +from typing import Union + +import mmh3 as py_mmh3 +from pkg_resources import parse_version + +from pocsuite3.lib.core.log import LOGGER as logger +from pocsuite3.lib.yaml.nuclei.protocols.common.replacer import Marker + + +def aes_gcm(key: Union[bytes, str], plaintext: Union[bytes, str]) -> bytes: + """ + AES GCM encrypts a string with key + + Example: + Input: {{hex_encode(aes_gcm("AES256Key-32Characters1234567890", "exampleplaintext"))}} + Output: ec183a153b8e8ae7925beed74728534b57a60920c0b009eaa7608a34e06325804c096d7eebccddea3e5ed6c4 + """ + # TODO + raise NotImplementedError + + +def base64(src: Union[bytes, str]) -> str: + """Base64 encodes a string + + Example: + Input: base64("Hello") + Output: SGVsbG8= + """ + if not isinstance(src, bytes): + src = src.encode('utf-8') + return py_built_in_base64.b64encode(src).decode('utf-8') + + +def base64_decode(src: Union[bytes, str]) -> bytes: + """ + Base64 decodes a string + + Example: + Input: base64_decode("SGVsbG8=") + Output: b"Hello" + """ + return py_built_in_base64.b64decode(src) + + +def base64_py(src: Union[bytes, str]) -> str: + """ + Encodes string to base64 like python (with new lines) + + Example: + Input: base64_py("Hello") + Output: SGVsbG8= + """ + return base64(src) + + +def concat(*arguments) -> str: + """ + Concatenates the given number of arguments to form a string + + Example: + Input: concat("Hello", 123, "world) + Output: Hello123world + """ + return ''.join(map(str, arguments)) + + +def compare_versions(versionToCheck: str, *constraints: str) -> bool: + """ + Compares the first version argument with the provided constraints + + Example: + Input: compare_versions('v1.0.0', '>v0.0.1', '=]*', constraint)[0] + if not operator: + operator = '=' + v2 = parse_version(constraint.lstrip('<>=')) + if v1 < v2 and operator not in ['<', '<=']: + return False + elif v1 == v2 and operator not in ['=', '<=', '>=']: + return False + elif v1 > v2 and operator not in ['>', '>=']: + return False + return True + + +def contains(inp: str, substring: str) -> bool: + """ + Verifies if a string contains a substring + + Example: + Input: contains("Hello", "lo") + Output: True + """ + return substring in inp + + +def contains_all(inp: str, *substrings: str) -> bool: + """ + Verifies if any input contains all of the substrings + + Example: + Input: contains_all("Hello everyone", "lo", "every") + Output: True + """ + return all(map(lambda s: s in inp, substrings)) + + +def contains_any(inp: str, *substrings: str) -> bool: + """ + Verifies if an input contains any of substrings + + Example: + Input: contains_any("Hello everyone", "abc", "llo") + Output: True + """ + return any(map(lambda s: s in inp, substrings)) + + +def dec_to_hex(number: Union[str, int]) -> str: + """ + Transforms the input number into hexadecimal format + + Example: + Input: dec_to_hex(7001) + Output: 1b59 + """ + if not isinstance(number, int): + number = int(number) + return hex(number)[2:] + + +def hex_to_dec(hexNumber: Union[str, int]) -> int: + """ + Transforms the input hexadecimal number into decimal format + + Example: + Input: hex_to_dec("ff") + hex_to_dec("0xff") + Output: 255 + """ + return int(str(hexNumber), 16) + + +def bin_to_dec(binaryNumber: Union[str, int]) -> int: + """ + Transforms the input binary number into a decimal format + + Example: + Input: bin_to_dec("0b1010") + bin_to_dec(1010) + Output: 10 + """ + return int(str(binaryNumber), 2) + + +def oct_to_dec(octalNumber: Union[str, int]) -> int: + """ + Transforms the input octal number into a decimal format + + Example: + Input: oct_to_dec("0o1234567") + oct_to_dec(1234567) + Output: 342391 + """ + return int(str(octalNumber), 8) + + +def generate_java_gadget(gadget: str, cmd: str, encoding: str) -> str: + """ + Generates a Java Deserialization Gadget + + Example: + Input: generate_java_gadget("dns", "{{interactsh-url}}", "base64") + """ + # TODO + raise NotImplementedError + + +def gzip(inp: Union[str, bytes]) -> bytes: + """ + Compresses the input using GZip + + Example: + Input: base64(gzip("Hello")) + Output: H4sIAI9GUGMC//NIzcnJBwCCidH3BQAAAA== + """ + if not isinstance(inp, bytes): + inp = inp.encode('utf-8') + return py_built_in_gzip.compress(inp) + + +def gzip_decode(inp: bytes) -> bytes: + """ + Decompresses the input using GZip + + Example: + Input: gzip_decode(hex_decode("1f8b08000000000000fff248cdc9c907040000ffff8289d1f705000000")) + Output: b"Hello" + """ + return py_built_in_gzip.decompress(inp) + + +def zlib(inp: Union[str, bytes]) -> bytes: + """ + Compresses the input using Zlib + + Example: + Input: base64(zlib("Hello")) + Output: eJzzSM3JyQcABYwB9Q== + """ + if not isinstance(inp, bytes): + inp = inp.encode('utf-8') + return py_built_in_zlib.compress(inp) + + +def zlib_decode(inp: bytes) -> bytes: + """ + Decompresses the input using Zlib + + Example: + Input: zlib_decode(hex_decode("789cf248cdc9c907040000ffff058c01f5")) + Output: b"Hello" + """ + return py_built_in_zlib.decompress(inp) + + +def hex_decode(inp: str) -> bytes: + """ + Hex decodes the given input + + Example: + Input: hex_decode("6161") + Output: b"aa" + """ + return binascii.unhexlify(inp) + + +def hex_encode(inp: Union[str, bytes]) -> str: + """ + Hex encodes the given input + + Example: + Input: hex_encode("aa") + Output: 6161 + """ + if not isinstance(inp, bytes): + inp = inp.encode('utf-8') + return binascii.hexlify(inp).decode('utf-8') + + +def html_escape(inp: str) -> str: + """ + HTML escapes the given input + + Example: + Input: html_escape("test") + Output: <body>test</body> + """ + return html.escape(inp) + + +def html_unescape(inp: str) -> str: + """ + HTML un-escapes the given input + + Example: + Input: html_unescape("<body>test</body>") + Output: test + """ + return html.unescape(inp) + + +def md5(inp: Union[str, bytes]) -> str: + """ + Calculates the MD5 (Message Digest) hash of the input + + Example: + Input: md5("Hello") + Output: 8b1a9953c4611296a827abf8c47804d7 + """ + if not isinstance(inp, bytes): + inp = inp.encode('utf-8') + m = hashlib.md5() + m.update(inp) + return m.hexdigest() + + +def mmh3(inp: Union[str, bytes]) -> int: + """ + Calculates the MMH3 (MurmurHash3) hash of an input + + Example: + Input: mmh3("Hello") + Output: 316307400 + """ + return py_mmh3.hash(inp) + + +def print_debug(*args) -> None: + """ + Prints the value of a given input or expression. Used for debugging. + + Example: + Input: print_debug(1+2, "Hello") + Output: 3 Hello + """ + # TODO + raise NotImplementedError + + +def rand_base(length: int, optionalCharSet: str = string.ascii_letters+string.digits) -> str: + """ + Generates a random sequence of given length string from an optional charset (defaults to letters and numbers) + + Example: + Input: rand_base(5, "abc") + Output: caccb + """ + return ''.join(random.choice(optionalCharSet) for _ in range(length)) + + +def rand_char(optionalCharSet: str = string.ascii_letters + string.digits) -> str: + """ + Generates a random character from an optional character set (defaults to letters and numbers) + + Example: + Input: rand_char("abc") + Output: a + """ + return random.choice(optionalCharSet) + + +def rand_int(optionalMin: int = 0, optionalMax: int = 2147483647) -> int: + """ + Generates a random integer between the given optional limits (defaults to 0 - MaxInt32) + + Example: + Input: rand_int(1, 10) + Output: 6 + """ + return random.randint(optionalMin, optionalMax) + + +def rand_text_alpha(length: int, optionalBadChars: str = '') -> str: + """ + Generates a random string of letters, of given length, excluding the optional cutset characters + + Example: + Input: rand_text_alpha(10, "abc") + Output: WKozhjJWlJ + """ + charset = ''.join(i if i not in optionalBadChars else '' for i in string.ascii_letters) + return ''.join(random.choice(charset) for _ in range(length)) + + +def rand_text_alphanumeric(length: int, optionalBadChars: str = '') -> str: + """ + Generates a random alphanumeric string, of given length without the optional cutset characters + + Example: + Input: rand_text_alphanumeric(10, "ab12") + Output: NthI0IiY8r + """ + charset = ''.join(i if i not in optionalBadChars else '' for i in string.ascii_letters + string.digits) + return ''.join(random.choice(charset) for _ in range(length)) + + +def rand_text_numeric(length: int, optionalBadNumbers: str = '') -> str: + """ + Generates a random numeric string of given length without the optional set of undesired numbers + + Example: + Input: rand_text_numeric(10, 123) + Output: 0654087985 + """ + charset = ''.join(i if i not in optionalBadNumbers else '' for i in string.digits) + return ''.join(random.choice(charset) for _ in range(length)) + + +def regex(pattern, inp): + """ + Tests the given regular expression against the input string + + Example: + Input: regex("H([a-z]+)o", "Hello") + Output: True + """ + return re.findall(pattern, inp) != [] + + +def remove_bad_chars(inp: str, cutset: str) -> str: + """ + Removes the desired characters from the input + + Example: + Input: remove_bad_chars("abcd", "bc") + Output: ad + """ + return ''.join(i if i not in cutset else '' for i in inp) + + +def repeat(inp: str, count: int) -> str: + """ + Repeats the input string the given amount of times + + Example: + Input: repeat("../", 5) + Output: ../../../../../ + """ + return inp * count + + +def replace(inp: str, old: str, new: str) -> str: + """ + Replaces a given substring in the given input + + Example: + Input: replace("Hello", "He", "Ha") + Output: Hallo + """ + return inp.replace(old, new) + + +def replace_regex(source: str, regex: str, replacement: str) -> str: + """ + Replaces substrings matching the given regular expression in the input + + Example: + Input: replace_regex("He123llo", "(\\d+)", "") + Output: Hello + """ + return re.sub(regex, replacement, source) + + +def reverse(inp: str) -> str: + """ + Reverses the given input + + Example: + Input: reverse("abc") + Output: cba + """ + return inp[::-1] + + +def sha1(inp: Union[bytes, str]) -> str: + """ + Calculates the SHA1 (Secure Hash 1) hash of the input + + Example: + Input: sha1("Hello") + Output: f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0 + """ + if not isinstance(inp, bytes): + inp = inp.encode('utf-8') + + s = hashlib.sha1() + s.update(inp) + return s.hexdigest() + + +def sha256(inp: Union[bytes, str]) -> str: + """ + Calculates the SHA256 (Secure Hash 256) hash of the input + + Example: + Input: sha256("Hello") + Output: 185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969 + """ + if not isinstance(inp, bytes): + inp = inp.encode('utf-8') + + s = hashlib.sha256() + s.update(inp) + return s.hexdigest() + + +def to_lower(inp: str) -> str: + """ + Transforms the input into lowercase characters + + Example: + Input: to_lower("HELLO") + Output: hello + """ + return inp.lower() + + +def to_upper(inp: str) -> str: + """ + Transforms the input into uppercase characters + + Example: + Input: to_upper("hello") + Output: HELLO + """ + return inp.upper() + + +def trim(inp: str, cutset: str) -> str: + """ + Returns a slice of the input with all leading and trailing Unicode code points contained in cutset removed + + Example: + Input: trim("aaaHelloddd", "ad") + Output: Hello + """ + return inp.strip(cutset) + + +def trim_left(inp: str, cutset: str) -> str: + """ + Returns a slice of the input with all leading Unicode code points contained in cutset removed + + Example: + Input: trim_left("aaaHelloddd", "ad") + Output: Helloddd + """ + return inp.lstrip(cutset) + + +def trim_prefix(inp: str, prefix: str) -> str: + """ + Returns the input without the provided leading prefix string + + Example: + Input: trim_prefix("aaHelloaa", "aa") + Output: Helloaa + """ + if inp.startswith(prefix): + return inp[len(prefix):] + return inp + + +def trim_right(inp: str, cutset: str) -> str: + """ + Returns a string, with all trailing Unicode code points contained in cutset removed + + Example: + Input: trim_right("aaaHelloddd", "ad") + Output: aaaHello + """ + return inp.rstrip(cutset) + + +def trim_space(inp: str) -> str: + """ + Returns a string, with all leading and trailing white space removed, as defined by Unicode + + Example: + Input: trim_space(" Hello ") + Output: Hello + """ + return inp.strip() + + +def trim_suffix(inp: str, suffix: str) -> str: + """ + Returns input without the provided trailing suffix string + + Example: + Input: trim_suffix("aaHelloaa", "aa") + Output: aaHello + """ + if inp.endswith(suffix): + return inp[:-len(suffix)] + return inp + + +def unix_time(optionalSeconds: int = 0) -> int: + """ + Returns the current Unix time (number of seconds elapsed since January 1, 1970 UTC) with the added optional seconds + + Example: + Input: unix_time(10) + Output: 1639568278 + """ + return int(time.time()) + optionalSeconds + + +def url_decode(inp: str) -> str: + """ + URL decodes the input string + Example: + Input: url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1") + Output: https://projectdiscovery.io?test=1 + """ + return urllib.parse.unquote_plus(inp) + + +def url_encode(inp: str) -> str: + """ + URL encodes the input string + + Example: + Input: url_encode("https://projectdiscovery.io/test?a=1") + Output: https%3A%2F%2Fprojectdiscovery.io%2Ftest%3Fa%3D1 + """ + return urllib.parse.quote_plus(inp) + + +def wait_for(seconds: int) -> bool: + """ + Pauses the execution for the given amount of seconds + + Example: + Input: wait_for(10) + Output: True + """ + time.sleep(seconds) + return True + + +def join(separator: str, *elements: str) -> str: + """ + Joins the given elements using the specified separator + + Example: + Input: join("_", 123, "hello", "world") + Output: 123_hello_world + """ + return separator.join(map(str, elements)) + + +def hmac(algorithm: str, data: Union[bytes, str], secret: Union[bytes, str]) -> str: + """ + hmac function that accepts a hashing function type with data and secret + + Example: + Input: hmac("sha1", "test", "scrt") + Output: 8856b111056d946d5c6c92a21b43c233596623c6 + """ + if not isinstance(data, bytes): + data = data.encode('utf-8') + if not isinstance(secret, bytes): + secret = secret.encode('utf-8') + + return py_hmac.new(secret, data, algorithm).hexdigest() + + +def date_time(dateTimeFormat: str, optionalUnixTime: int = int(time.time())) -> str: + """ + Returns the formatted date time using simplified or go style layout for the current or the given unix time + + Example: + Input: date_time("%Y-%m-%d %H:%M") + date_time("%Y-%m-%d %H:%M", 1654870680) + Output: 2022-06-10 14:18 + """ + return datetime.datetime.utcfromtimestamp(optionalUnixTime).strftime(dateTimeFormat) + + +def to_unix_time(inp: str, layout: str = "%Y-%m-%d %H:%M:%S") -> int: + """ + Parses a string date time using default or user given layouts, then returns its Unix timestamp + + Example: + Input: to_unix_time("2022-01-13 16:30:10") + Output: 1642091410 + """ + return int(time.mktime(datetime.datetime.strptime(inp, layout).timetuple())) + + +def starts_with(inp: str, *prefix: str) -> bool: + """ + Checks if the string starts with any of the provided substrings + + Example: + Input: starts_with("Hello", "He") + Output: True + """ + return any(inp.startswith(p) for p in prefix) + + +def line_starts_with(inp: str, *prefix: str) -> bool: + """ + Checks if any line of the string starts with any of the provided substrings + + Example: + Input: line_starts_with("Hi\nHello", "He") + Output: True + """ + for line in inp.splitlines(): + for p in prefix: + if line.startswith(p): + return True + return False + + +def ends_with(inp: str, *suffix: str) -> bool: + """ + Checks if the string ends with any of the provided substrings + + Example: + Input: ends_with("Hello", "lo") + Output: True + """ + return any(inp.endswith(s) for s in suffix) + + +def line_ends_with(inp: str, *suffix: str) -> bool: + """ + Checks if any line of the string ends with any of the provided substrings + + Example: + Input: line_ends_with("Hello\nHi", "lo") + Output: True + """ + for line in inp.splitlines(): + for s in suffix: + if line.endswith(s): + return True + return False + + +def Evaluate(inp: str, dynamic_values: dict = {}) -> str: + """ + Evaluate checks if the match contains a dynamic variable, for each + found one we will check if it's an expression and can be compiled, + it will be evaluated and the results will be returned. + """ + + # find expression and execute + + OpenMarker, CloseMarker = Marker.ParenthesisOpen, Marker.ParenthesisClose + exps = OrderedDict() + maxIterations, iterations = 250, 0 + data = inp + vars().update(dynamic_values) + + while iterations <= maxIterations: + iterations += 1 + indexOpenMarker = data.find(OpenMarker) + if indexOpenMarker < 0: + break + + indexOpenMarkerOffset = indexOpenMarker + len(OpenMarker) + shouldSearchCloseMarker = True + closeMarkerFound = False + innerData = data + skip = indexOpenMarkerOffset + + while shouldSearchCloseMarker: + indexCloseMarker = innerData.find(CloseMarker, skip) + if indexCloseMarker < 0: + shouldSearchCloseMarker = False + continue + indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker) + potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker] + try: + result = eval(potentialMatch) + exps[potentialMatch] = result + closeMarkerFound = True + shouldSearchCloseMarker = False + except Exception as e: + logger.debug(str(e)) + skip = indexCloseMarkerOffset + + if closeMarkerFound: + data = data[indexCloseMarkerOffset:] + else: + data = data[indexOpenMarkerOffset:] + + if exps: + logger.debug('Expressions: ' + str(exps)) + for k, v in exps.items(): + inp = inp.replace(f'{OpenMarker}{k}{CloseMarker}', v) + return inp + + +if __name__ == '__main__': + print(Evaluate("{{to_lower(rand_base(5))}}")) + print(Evaluate("{{base64('World')}}")) + print(Evaluate("{{base64(Hello)}}", {'Hello': 'World'})) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py new file mode 100644 index 00000000..9be98ab9 --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py @@ -0,0 +1,24 @@ +import json + + +class Marker: + # General marker (open/close) + General = "§" + # ParenthesisOpen marker - begin of a placeholder + ParenthesisOpen = "{{" + # ParenthesisClose marker - end of a placeholder + ParenthesisClose = "}}" + + +def marker_replace(data, dynamic_values): + """replaces placeholders in template with values + """ + data = json.dumps(data) + for k, v in dynamic_values.items(): + if k in data: + data = data.replace(f'{Marker.General}{k}{Marker.General}', v) + data = data.replace(f'{Marker.ParenthesisOpen}{k}{Marker.ParenthesisClose}', v) + + # TODO + # various helper functions + return json.loads(data) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py new file mode 100644 index 00000000..3839023b --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py @@ -0,0 +1,273 @@ +import itertools +from collections import OrderedDict +from dataclasses import dataclass, field +from enum import Enum + +import requests + +from pocsuite3.lib.core.common import check_file, get_file_items +from pocsuite3.lib.core.log import LOGGER as logger +from pocsuite3.lib.yaml.nuclei.operators import (ExtractDSL, ExtractJSON, + ExtractKval, Extractor, + ExtractorType, ExtractRegex, + ExtractXPath, MatchBinary, + MatchDSL, Matcher, + MatcherType, MatchRegex, + MatchSize, MatchStatusCode, + MatchWords) +from pocsuite3.lib.yaml.nuclei.protocols.common.replacer import marker_replace + + +class AttackType(Enum): + BatteringRamAttack = "batteringram" + PitchForkAttack = "pitchfork" + ClusterBombAttack = "clusterbomb" + + +class HTTPMethod(Enum): + HTTPGet = "GET" + HTTPHead = "HEAD" + HTTPPost = "POST" + HTTPPut = "PUT" + HTTPDelete = "DELETE" + HTTPConnect = "CONNECT" + HTTPOptions = "OPTIONS" + HTTPTrace = "TRACE" + HTTPPatch = "PATCH" + HTTPPurge = "PURGE" + HTTPDebug = "DEBUG" + + +# HttpRequest contains a http request to be made from a template +@dataclass +class HttpRequest: + # Operators for the current request go here. + matchers: list[Matcher] = field(default_factory=list) + extractors: list[Extractor] = field(default_factory=list) + matchers_condition: str = 'or' + + # Path contains the path/s for the HTTP requests. It supports variables as placeholders. + path: list[str] = field(default_factory=list) + + # Raw contains HTTP Requests in Raw format. + raw: list[str] = field(default_factory=list) + + # ID is the optional id of the request + id: str = '' + + name: str = '' + # Attack is the type of payload combinations to perform. + attack: AttackType = 'batteringram' + + # Method is the HTTP Request Method. + method: HTTPMethod = 'GET' + + # Body is an optional parameter which contains HTTP Request body. + body: str = '' + + # Payloads contains any payloads for the current request. + payloads: dict = field(default_factory=dict) + + # Headers contains HTTP Headers to send with the request. + headers: dict = field(default_factory=dict) + + # RaceCount is the number of times to send a request in Race Condition Attack. + race_count: int = 0 + + # MaxRedirects is the maximum number of redirects that should be followed. + max_redirects: int = 0 + + # PipelineConcurrentConnections is number of connections to create during pipelining. + pipeline_concurrent_connections = 0 + + # PipelineRequestsPerConnection is number of requests to send per connection when pipelining. + pipeline_requests_per_connection = 0 + + # Threads specifies number of threads to use sending requests. This enables Connection Pooling. + threads: int = 0 + + # MaxSize is the maximum size of http response body to read in bytes. + max_size: int = 0 + + # TODO + # cookie-reuse accepts boolean input and false as default, This option not work on pocsuite3 + cookie_reuse: bool = False + + read_all: bool = False + redirects: bool = False + pipeline: bool = False + unsafe: bool = False + race: bool = False + + # TODO + # Request condition allows checking for condition between multiple requests for writing complex checks and + # exploits involving multiple HTTP request to complete the exploit chain. + + req_condition: bool = False + + stop_at_first_match: bool = True + skip_variables_check: bool = False + iterate_all: bool = False + digest_username: str = '' + digest_password: str = '' + + +def getMatchPart(part: str, response: requests.Response, return_bytes: bool = False) -> str: + result = b'' + headers = '\n'.join(f'{k}: {v}' for k, v in response.headers.items()).encode('utf-8') + + if part == 'all': + result = headers + b'\n\n' + response.content + elif part in ['', 'body']: + result = response.content + elif part in ['header', 'all_headers']: + result = headers + + return result if return_bytes else result.decode('utf-8') + + +def HttpMatch(request: HttpRequest, response: requests.Response): + matchers = request.matchers + matchers_result = [] + + for i, matcher in enumerate(matchers): + matcher_res = False + item = getMatchPart(matcher.part, response, return_bytes=matcher.type == MatcherType.BinaryMatcher) + + if matcher.type == MatcherType.StatusMatcher: + matcher_res = MatchStatusCode(matcher, response.status_code) + logger.debug(f'matcher: {matcher}, result: {matcher_res}') + + elif matcher.type == MatcherType.SizeMatcher: + matcher_res = MatchSize(matcher, len(item)) + logger.debug(f'matcher: {matcher}, result: {matcher_res}') + + elif matcher.type == MatcherType.WordsMatcher: + matcher_res, _ = MatchWords(matcher, item, {}) + logger.debug(f'matcher: {matcher}, result: {matcher_res}') + + elif matcher.type == MatcherType.RegexMatcher: + matcher_res, _ = MatchRegex(matcher, item) + logger.debug(f'matcher: {matcher}, result: {matcher_res}') + + elif matcher.type == MatcherType.BinaryMatcher: + matcher_res, _ = MatchBinary(matcher, item) + logger.debug(f'matcher: {matcher}, result: {matcher_res}') + + elif matcher.type == MatcherType.DSLMatcher: + matcher_res = MatchDSL(matcher, {}) + logger.debug(f'matcher: {matcher}, result: {matcher_res}') + + if not matcher_res: + if request.matchers_condition == 'and': + return False + elif request.matchers_condition == 'or': + continue + + if request.matchers_condition == 'or': + return True + + matchers_result.append(matcher_res) + + if len(matchers) - 1 == i: + return True + + return False + + +def HttpExtract(request: HttpRequest, response: requests.Response): + extractors = request.extractors + extractors_result = [] + + for extractor in extractors: + item = getMatchPart(extractor.part, response) + + res = None + if extractor.type == ExtractorType.RegexExtractor: + res = ExtractRegex(extractor, item) + logger.debug(f'extractor: {extractor}, result: {res}') + elif extractor.type == ExtractorType.KValExtractor: + res = ExtractKval(extractor, response.headers) + logger.debug(f'extractor: {extractor}, result: {res}') + elif extractor.type == ExtractorType.XPathExtractor: + res = ExtractXPath(extractor, item) + logger.debug(f'extractor: {extractor}, result: {res}') + elif extractor.type == ExtractorType.JSONExtractor: + res = ExtractJSON(extractor, item) + logger.debug(f'extractor: {extractor}, result: {res}') + elif ExtractorType.type == ExtractorType.DSLExtractor: + res = ExtractDSL(extractor, {}) + logger.debug(f'extractor: {extractor}, result: {res}') + + if res: + extractors_result.append(res) + return extractors_result + + +def extract_dict(text, line_sep='\n', kv_sep='='): + """Split the string into a dictionary according to the split method + """ + _dict = OrderedDict([i.split(kv_sep, 1) for i in text.split(line_sep)]) + return _dict + + +def payloadGenerator(request: HttpRequest) -> OrderedDict: + payloads = OrderedDict() + payloads.update(request.payloads) + + for k, v in payloads.items(): + if isinstance(v, str) and check_file(v): + payloads[k] = get_file_items(v) + + payload_keys, payload_vals = payloads.keys(), payloads.values() + payload_vals = [i if isinstance(i, list) else [i] for i in payload_vals] + + if request.attack == AttackType.PitchForkAttack: + for instance in zip(*payload_vals): + yield dict(zip(payload_keys, instance)) + else: + for instance in itertools.product(*payload_vals): + yield dict(zip(payload_keys, instance)) + + +def httpRequestGenerator(request: HttpRequest, dynamic_values: OrderedDict): + for payload_instance in payloadGenerator(request): + payload_instance.update(dynamic_values) + + for path in request.path + request.raw: + + method, url, headers, data, kwargs = '', '', '', '', OrderedDict() + # base request + if path.startswith('{{'): + method = request.method.value + headers = request.headers + data = request.body + url = path + + # raw + else: + raw = path.strip() + raws = list(map(lambda x: x.strip(), raw.splitlines())) + method, path, _ = raws[0].split(' ') + url = f'{{{{BaseURL}}}}{path}' + + if method == "POST": + index = 0 + for i in raws: + index += 1 + if i.strip() == "": + break + if len(raws) == index: + raise Exception + + headers = raws[1:index - 1] + headers = extract_dict('\n'.join(headers), '\n', ": ") + data = raws[index] + else: + headers = extract_dict('\n'.join(raws[1:]), '\n', ": ") + + kwargs.setdefault('allow_redirects', request.redirects) + kwargs.setdefault('data', data) + kwargs.setdefault('headers', headers) + + yield (method, marker_replace(url, payload_instance), marker_replace(kwargs, payload_instance)) diff --git a/pocsuite3/lib/yaml/nuclei/templates/__init__.py b/pocsuite3/lib/yaml/nuclei/templates/__init__.py new file mode 100644 index 00000000..bb09376e --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/templates/__init__.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass, field +from enum import Enum + +from pocsuite3.lib.yaml.nuclei.model import Info +from pocsuite3.lib.yaml.nuclei.protocols.http import HttpRequest + + +class ProtocolType(Enum): + InvalidProtocol = "invalid" + DNSProtocol = "dns" + FileProtocol = "file" + HTTPProtocol = "http" + HeadlessProtocol = "headless" + NetworkProtocol = "network" + WorkflowProtocol = "workflow" + SSLProtocol = "ssl" + WebsocketProtocol = "websocket" + WHOISProtocol = "whois" + + +# Template is a YAML input file which defines all the requests and other metadata for a template. +@dataclass +class Template: + id: str = '' + info: Info = field(default_factory=Info) + requests: list[HttpRequest] = field(default_factory=list) + stop_at_first_match: bool = True + variables: dict = field(default_factory=dict) diff --git a/tests/test_nuclei_helper_functions.py b/tests/test_nuclei_helper_functions.py new file mode 100644 index 00000000..d51ab7b0 --- /dev/null +++ b/tests/test_nuclei_helper_functions.py @@ -0,0 +1,190 @@ +import unittest +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import * + + +class TestCase(unittest.TestCase): + def test_base64(self): + self.assertEqual(base64("Hello"), "SGVsbG8=") + + def test_base64_decode(self): + self.assertEqual(base64_decode("SGVsbG8="), b"Hello") + + def test_base64_py(self): + self.assertEqual(base64_py("Hello"), "SGVsbG8=") + + def test_concat(self): + self.assertEqual(concat("Hello", 123, "world"), "Hello123world") + + def test_compare_versions(self): + self.assertTrue(compare_versions("v1.0.0", ">v0.0.1", "test"), "<body>test</body>") + + def test_html_unescape(self): + self.assertEqual(html_unescape("<body>test</body>"), "test") + + def test_md5(self): + self.assertEqual(md5("Hello"), "8b1a9953c4611296a827abf8c47804d7") + + def test_mmh3(self): + self.assertEqual(mmh3("Hello"), 316307400) + + def test_rand_base(self): + self.assertRegex(rand_base(5, "abc"), r"[abc]{5}") + + def test_rand_char(self): + self.assertRegex(rand_char("abc"), r"[abc]") + + def test_rand_int(self): + self.assertIn(rand_int(1, 10), range(1, 11)) + + def test_rand_text_alpha(self): + self.assertRegex(rand_text_alpha(10, "abc"), r"[^abc]{10}") + + def test_rand_text_alphanumeric(self): + self.assertRegex(rand_text_alphanumeric(10, "ab12"), r"[^ab12]{10}") + + def test_rand_text_numeric(self): + self.assertRegex(rand_text_numeric(10, "123"), r"[^123]{10}") + + def test_regex(self): + self.assertTrue(regex("H([a-z]+)o", "Hello")) + + def test_remove_bad_chars(self): + self.assertEqual(remove_bad_chars("abcd", "bc"), "ad") + + def test_repeat(self): + self.assertEqual(repeat("../", 5), "../../../../../") + + def test_replace(self): + self.assertEqual(replace("Hello", "He", "Ha"), "Hallo") + + def test_replace_regex(self): + self.assertEqual(replace_regex("He123llo", "(\\d+)", ""), "Hello") + + def test_reverse(self): + self.assertEqual(reverse("abc"), "cba") + + def test_sha1(self): + self.assertEqual(sha1("Hello"), "f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0") + + def test_sha256(self): + self.assertEqual( + sha256("Hello"), + "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", + ) + + def test_to_lower(self): + self.assertEqual(to_lower("HELLO"), "hello") + + def test_to_upper(self): + self.assertEqual(to_upper("hello"), "HELLO") + + def test_trim(self): + self.assertEqual(trim("aaaHelloddd", "ad"), "Hello") + + def test_trim_left(self): + self.assertEqual(trim_left("aaaHelloddd", "ad"), "Helloddd") + + def test_trim_prefix(self): + self.assertEqual(trim_prefix("aaHelloaa", "aa"), "Helloaa") + + def test_trim_right(self): + self.assertEqual(trim_right("aaaHelloddd", "ad"), "aaaHello") + + def test_trim_space(self): + self.assertEqual(trim_space(" Hello "), "Hello") + + def test_trim_suffix(self): + self.assertEqual(trim_suffix("aaHelloaa", "aa"), "aaHello") + + def test_unix_time(self): + self.assertGreater(unix_time(10), 1639568278) + + def test_url_decode(self): + self.assertEqual( + url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1"), + "https://projectdiscovery.io?test=1", + ) + + def test_url_encode(self): + self.assertEqual( + url_encode("https://projectdiscovery.io/test?a=1"), + "https%3A%2F%2Fprojectdiscovery.io%2Ftest%3Fa%3D1", + ) + + def test_join(self): + self.assertEqual(join("_", 123, "hello", "world"), "123_hello_world") + + def test_hmac(self): + self.assertEqual(hmac("sha1", "test", "scrt"), "8856b111056d946d5c6c92a21b43c233596623c6") + + @unittest.skip(reason="timezone") + def test_date_time(self): + self.assertEqual(date_time("%Y-%m-%d %H:%M", 1654870680), "2022-06-10 14:18") + + @unittest.skip(reason="timezone") + def test_to_unix_time(self): + self.assertEqual(to_unix_time("2022-01-13 16:30:10"), 1642120210) + + def test_starts_with(self): + self.assertTrue(starts_with("Hello", "e", "He")) + + def test_line_starts_with(self): + self.assertTrue(line_starts_with("Hi\nHello", "e", "He")) + + def test_ends_with(self): + self.assertTrue(ends_with("Hello", "e", "lo")) + + def test_line_ends_with(self): + self.assertTrue(line_ends_with("Hi\nHello", "e", "lo")) + + +if __name__ == "__main__": + unittest.main() From 5a1bb5e533705a2a057d0205945b1393c5bff323 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 26 Oct 2022 18:08:31 -0700 Subject: [PATCH 2/9] feat(yaml): add oob testing support --- pocsuite3/lib/core/register.py | 2 +- pocsuite3/lib/yaml/nuclei/__init__.py | 16 ++++---- .../protocols/common/interactsh/__init__.py | 19 +++++++++ .../yaml/nuclei/protocols/http/__init__.py | 41 +++++++++++++------ pocsuite3/modules/interactsh/__init__.py | 16 +++++--- 5 files changed, 67 insertions(+), 27 deletions(-) create mode 100644 pocsuite3/lib/yaml/nuclei/protocols/common/interactsh/__init__.py diff --git a/pocsuite3/lib/core/register.py b/pocsuite3/lib/core/register.py index 6ffbaae5..dc63d410 100644 --- a/pocsuite3/lib/core/register.py +++ b/pocsuite3/lib/core/register.py @@ -9,7 +9,6 @@ from pocsuite3.lib.core.data import kb from pocsuite3.lib.core.data import logger from pocsuite3.lib.core.settings import POC_IMPORTDICT -from pocsuite3.lib.yaml.nuclei import Nuclei class PocLoader(Loader): @@ -72,6 +71,7 @@ def exec_module(self, module): # convert yaml template to pocsuite3 poc script if filename.endswith('.yaml') and re.search(r'matchers:\s+-', poc_code): + from pocsuite3.lib.yaml.nuclei import Nuclei poc_code = str(Nuclei(poc_code)) self.check_requires(poc_code) diff --git a/pocsuite3/lib/yaml/nuclei/__init__.py b/pocsuite3/lib/yaml/nuclei/__init__.py index ec1f32a2..ef8abe15 100644 --- a/pocsuite3/lib/yaml/nuclei/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/__init__.py @@ -77,6 +77,7 @@ def __init__(self, template, target=''): self.execute_options = OrderedDict() self.execute_options['stop_at_first_match'] = self.template.stop_at_first_match self.execute_options['variables'] = self.template.variables + self.execute_options['interactsh'] = None self.requests = self.template.requests @@ -103,15 +104,16 @@ def execute_request(self, request: HttpRequest) -> dict: logger.debug(dump.dump_all(response).decode('utf-8')) except Exception as e1: logger.debug(str(e1)) - continue - match_res = HttpMatch(request, response) + response = None + match_res = HttpMatch(request, response, self.execute_options['interactsh']) extractor_res = HttpExtract(request, response) if match_res and extractor_res: match_res = str(dict(extractor_res[0])) if match_res and request.stop_at_first_match: return match_res results.append(match_res) - response.close() + if response: + response.close() except Exception as e: logger.debug(str(e)) return results and any(results) @@ -152,11 +154,9 @@ def execute_template(self): with automatic Request correlation built in. It's as easy as writing {{interactsh-url}} anywhere in the request. """ if '{{interactsh-url}}' in self.yaml_template or '§interactsh-url§' in self.yaml_template: - from pocsuite3.modules.interactsh import Interactsh - ish = Interactsh() - ish_url, ish_flag = ish.build_request(method='') - self.dynamic_values['interactsh-url'] = ish_url - self.execute_options['interactsh_client'] = ish + from pocsuite3.lib.yaml.nuclei.protocols.common.interactsh import InteractshClient + self.execute_options['interactsh'] = InteractshClient() + self.dynamic_values['interactsh-url'] = self.execute_options['interactsh'].client.domain results = [] for request in self.requests: diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/interactsh/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/interactsh/__init__.py new file mode 100644 index 00000000..ecb4604f --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/interactsh/__init__.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass, field +from pocsuite3.modules.interactsh import Interactsh +from pocsuite3.lib.core.log import LOGGER as logger + + +@dataclass +class InteractshClient: + client: Interactsh = field(default_factory=Interactsh) + interactsh_protocol: list = field(default_factory=list) + interactsh_request: list = field(default_factory=list) + interactsh_response: list = field(default_factory=list) + + def poll(self) -> None: + results = self.client.poll() + for result in results: + logger.debug(result) + self.interactsh_protocol.append(result['protocol']) + self.interactsh_request.append(result['raw-request']) + self.interactsh_response.append(result['raw-response']) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py index 3839023b..28770b2d 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py @@ -112,30 +112,47 @@ class HttpRequest: digest_password: str = '' -def getMatchPart(part: str, response: requests.Response, return_bytes: bool = False) -> str: +def getMatchPart(part: str, response: requests.Response, interactsh, return_bytes: bool = False) -> str: result = b'' - headers = '\n'.join(f'{k}: {v}' for k, v in response.headers.items()).encode('utf-8') + headers = b'' + body = b'' + if isinstance(response, requests.Response): + headers = '\n'.join(f'{k}: {v}' for k, v in response.headers.items()).encode('utf-8') + body = response.content if part == 'all': - result = headers + b'\n\n' + response.content + result = headers + b'\n\n' + body elif part in ['', 'body']: - result = response.content + result = body elif part in ['header', 'all_headers']: result = headers - - return result if return_bytes else result.decode('utf-8') - - -def HttpMatch(request: HttpRequest, response: requests.Response): + elif part == 'interactsh_protocol': + interactsh.poll() + result = '\n'.join(interactsh.interactsh_protocol) + elif part == 'interactsh_request': + interactsh.poll() + result = '\n'.join(interactsh.interactsh_request) + elif part == 'interactsh_response': + interactsh.poll() + result = '\n'.join(interactsh.interactsh_response) + + if return_bytes and not isinstance(result, bytes): + result = result.encode() + elif not return_bytes and isinstance(result, bytes): + result = result.decode() + return result + + +def HttpMatch(request: HttpRequest, response: requests.Response, interactsh): matchers = request.matchers matchers_result = [] for i, matcher in enumerate(matchers): matcher_res = False - item = getMatchPart(matcher.part, response, return_bytes=matcher.type == MatcherType.BinaryMatcher) + item = getMatchPart(matcher.part, response, interactsh, return_bytes=matcher.type == MatcherType.BinaryMatcher) if matcher.type == MatcherType.StatusMatcher: - matcher_res = MatchStatusCode(matcher, response.status_code) + matcher_res = MatchStatusCode(matcher, response.status_code if response else 0) logger.debug(f'matcher: {matcher}, result: {matcher_res}') elif matcher.type == MatcherType.SizeMatcher: @@ -187,7 +204,7 @@ def HttpExtract(request: HttpRequest, response: requests.Response): res = ExtractRegex(extractor, item) logger.debug(f'extractor: {extractor}, result: {res}') elif extractor.type == ExtractorType.KValExtractor: - res = ExtractKval(extractor, response.headers) + res = ExtractKval(extractor, response.headers if response else {}) logger.debug(f'extractor: {extractor}, result: {res}') elif extractor.type == ExtractorType.XPathExtractor: res = ExtractXPath(extractor, item) diff --git a/pocsuite3/modules/interactsh/__init__.py b/pocsuite3/modules/interactsh/__init__.py index 7fb26f54..ea28e758 100644 --- a/pocsuite3/modules/interactsh/__init__.py +++ b/pocsuite3/modules/interactsh/__init__.py @@ -5,12 +5,16 @@ import json import random import time -from uuid import uuid4 from base64 import b64encode +from uuid import uuid4 + from Cryptodome.Cipher import AES, PKCS1_OAEP -from Cryptodome.PublicKey import RSA from Cryptodome.Hash import SHA256 -from pocsuite3.api import requests, logger, random_str, conf +from Cryptodome.PublicKey import RSA + +from pocsuite3.lib.core.data import conf, logger +from pocsuite3.lib.request import requests +from pocsuite3.lib.utils import random_str class Interactsh: @@ -53,12 +57,12 @@ def register(self): msg = f"[PLUGIN] Interactsh: Can not initiate {self.server} DNS callback client" try: res = self.session.post( - f"https://{self.server}/register", headers=self.headers, json=data, verify=False) + f"http://{self.server}/register", headers=self.headers, json=data, verify=False) if res.status_code == 401: logger.error("[PLUGIN] Interactsh: auth error") elif 'success' not in res.text: logger.error(msg) - except requests.exceptions.RequestException: + except requests.RequestException: logger.error(msg) def poll(self): @@ -67,7 +71,7 @@ def poll(self): while count: try: - url = f"https://{self.server}/poll?id={self.correlation_id}&secret={self.secret}" + url = f"http://{self.server}/poll?id={self.correlation_id}&secret={self.secret}" res = self.session.get(url, headers=self.headers, verify=False).json() aes_key, data_list = res['aes_key'], res['data'] for i in data_list: From e0b5463825b223de111087b1f248c1b87833f428 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Fri, 28 Oct 2022 14:28:24 -0700 Subject: [PATCH 3/9] feat(yaml): support dsl matcher and extractor, bug fix --- pocsuite3/lib/yaml/nuclei/__init__.py | 91 ++++++++++---- .../nuclei/operators/extrators/__init__.py | 75 ++++++++---- .../nuclei/operators/matchers/__init__.py | 19 ++- .../protocols/common/expressions/__init__.py | 23 ++-- .../protocols/common/replacer/__init__.py | 8 +- .../yaml/nuclei/protocols/http/__init__.py | 114 +++++++++++------- requirements.txt | 5 + setup.py | 7 +- 8 files changed, 235 insertions(+), 107 deletions(-) diff --git a/pocsuite3/lib/yaml/nuclei/__init__.py b/pocsuite3/lib/yaml/nuclei/__init__.py index ef8abe15..6ce8060b 100644 --- a/pocsuite3/lib/yaml/nuclei/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/__init__.py @@ -1,3 +1,4 @@ +import binascii import re from collections import OrderedDict @@ -15,7 +16,8 @@ from pocsuite3.lib.yaml.nuclei.protocols.http import (AttackType, HttpExtract, HttpMatch, HTTPMethod, HttpRequest, - httpRequestGenerator) + httpRequestGenerator, + responseToDSLMap) from pocsuite3.lib.yaml.nuclei.templates import Template @@ -67,6 +69,10 @@ def expand_preprocessors(data: str) -> str: class Nuclei(): def __init__(self, template, target=''): self.yaml_template = template + try: + self.yaml_template = binascii.unhexlify(self.yaml_template).decode() + except binascii.Error: + pass self.json_template = yaml.safe_load(expand_preprocessors(self.yaml_template)) self.template = dacite.from_dict( Template, hyphen_to_underscore(self.json_template), @@ -85,9 +91,11 @@ def __init__(self, template, target=''): def execute_request(self, request: HttpRequest) -> dict: results = [] + resp_data_all = {} with requests.Session() as session: try: - for (method, url, kwargs) in httpRequestGenerator(request, self.dynamic_values): + for (method, url, kwargs, payload, request_count, current_index) in httpRequestGenerator( + request, self.dynamic_values): try: """ Redirection conditions can be specified per each template. By default, redirects are not followed. @@ -101,22 +109,58 @@ def execute_request(self, request: HttpRequest) -> dict: else: session.max_redirects = 10 response = session.request(method=method, url=url, **kwargs) - logger.debug(dump.dump_all(response).decode('utf-8')) - except Exception as e1: - logger.debug(str(e1)) + # for debug purpose + try: + logger.debug(dump.dump_all(response).decode('utf-8')) + except UnicodeDecodeError: + pass + + except Exception: + import traceback + traceback.print_exc() response = None - match_res = HttpMatch(request, response, self.execute_options['interactsh']) - extractor_res = HttpExtract(request, response) - if match_res and extractor_res: - match_res = str(dict(extractor_res[0])) - if match_res and request.stop_at_first_match: - return match_res - results.append(match_res) + + resp_data = responseToDSLMap(response) if response: response.close() - except Exception as e: - logger.debug(str(e)) - return results and any(results) + + extractor_res = HttpExtract(request, resp_data) + self.dynamic_values.update(extractor_res['internal']) + + if request.req_condition: + resp_data_all.update(resp_data) + for k, v in resp_data.items(): + resp_data_all[f'{k}_{current_index}'] = v + if current_index == request_count: + resp_data_all.update(self.dynamic_values) + match_res = HttpMatch(request, resp_data_all, self.execute_options['interactsh']) + resp_data_all = {} + if match_res: + output = {} + output.update(extractor_res['external']) + output.update(payload) + output['extraInfo'] = extractor_res['extraInfo'] + results.append(output) + if request.stop_at_first_match: + return results + else: + resp_data.update(self.dynamic_values) + match_res = HttpMatch(request, resp_data, self.execute_options['interactsh']) + if match_res: + output = {} + output.update(extractor_res['external']) + output.update(payload) + output['extraInfo'] = extractor_res['extraInfo'] + results.append(output) + if request.stop_at_first_match: + return results + except Exception: + import traceback + traceback.print_exc() + if results and any(results): + return results + else: + return False def execute_template(self): ''' @@ -154,17 +198,16 @@ def execute_template(self): with automatic Request correlation built in. It's as easy as writing {{interactsh-url}} anywhere in the request. """ if '{{interactsh-url}}' in self.yaml_template or '§interactsh-url§' in self.yaml_template: - from pocsuite3.lib.yaml.nuclei.protocols.common.interactsh import InteractshClient + from pocsuite3.lib.yaml.nuclei.protocols.common.interactsh import \ + InteractshClient self.execute_options['interactsh'] = InteractshClient() self.dynamic_values['interactsh-url'] = self.execute_options['interactsh'].client.domain - results = [] for request in self.requests: res = self.execute_request(request) - results.append(res) - if self.execute_options['stop_at_first_match'] and res: + if res: return res - return all(results) + return False def run(self): return self.execute_template() @@ -182,9 +225,7 @@ def __str__(self): if k in key_convert: k = key_convert.get(k) if type(v) in [str]: - v = f'\'{v.strip()}\'' - if k == 'desc': - v = f'\'\'{v}\'\'' + v = f'\'\'\'{v.strip()}\'\'\'' info.append(f' {k} = {v}') @@ -199,7 +240,7 @@ def __str__(self): ' result = {}\n', ' if not self._check():\n', ' return self.parse_output(result)\n', - " template = '''%s'''\n" % self.yaml_template, + " template = '%s'\n" % binascii.hexlify(self.yaml_template.encode()).decode(), ' res = Nuclei(template, self.url).run()\n', ' if res:\n', ' result["VerifyInfo"] = {}\n', @@ -207,7 +248,7 @@ def __str__(self): ' result["VerifyInfo"]["Info"] = {}\n', ' result["VerifyInfo"]["Info"]["Severity"] = "%s"\n' % self.template.info.severity.value, ' if not isinstance(res, bool):\n' - ' result["VerifyInfo"]["Info"]["Result"] = {}\n', + ' result["VerifyInfo"]["Info"]["Result"] = res\n', ' return self.parse_output(result)\n', '\n', '\n', diff --git a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py index 6fdc9714..6e68e9a1 100644 --- a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py @@ -1,13 +1,12 @@ import re -from collections import defaultdict from dataclasses import dataclass, field from enum import Enum import jq import lxml -import requests +from requests.structures import CaseInsensitiveDict -from pocsuite3.lib.core.common import OrderedSet +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import Evaluate class ExtractorType(Enum): @@ -55,10 +54,10 @@ class Extractor: case_insensitive: bool = False -def ExtractRegex(e: Extractor, corpus: str) -> defaultdict: +def ExtractRegex(e: Extractor, corpus: str) -> dict: """Extract data from response based on a Regular Expression. """ - results = defaultdict(OrderedSet) + results = {'internal': {}, 'external': {}, 'extraInfo': []} for regex in e.regex: matches = re.search(regex, corpus) if not matches: @@ -72,16 +71,23 @@ def ExtractRegex(e: Extractor, corpus: str) -> defaultdict: continue if e.name: - results[e.name].add(res) + if e.internal: + results['internal'][e.name] = res + else: + results['external'][e.name] = res + return results else: - results['ExtraInfo'].add(res) + results['extraInfo'].append(res) return results -def ExtractKval(e: Extractor, headers: requests.structures.CaseInsensitiveDict) -> defaultdict: +def ExtractKval(e: Extractor, headers: CaseInsensitiveDict) -> dict: """Extract key: value/key=value formatted data from Response Header/Cookie """ - results = defaultdict(OrderedSet) + if not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + results = {'internal': {}, 'external': {}, 'extraInfo': []} for k in e.kval: res = '' if k in headers: @@ -93,16 +99,20 @@ def ExtractKval(e: Extractor, headers: requests.structures.CaseInsensitiveDict) continue if e.name: - results[e.name].add(res) + if e.internal: + results['internal'][e.name] = res + else: + results['external'][e.name] = res + return results else: - results['ExtraInfo'].add(res) + results['extraInfo'].append(res) return results -def ExtractXPath(e: Extractor, corpus: str) -> defaultdict: +def ExtractXPath(e: Extractor, corpus: str) -> dict: """A xpath extractor example to extract value of href attribute from HTML response """ - results = defaultdict(OrderedSet) + results = {'internal': {}, 'external': {}, 'extraInfo': []} if corpus.startswith(' defaultdict: continue if e.name: - results[e.name].add(res) + if e.internal: + results['internal'][e.name] = res + else: + results['external'][e.name] = res + return results else: - results['ExtraInfo'].add(res) + results['extraInfo'].append(res) return results -def ExtractJSON(e: Extractor, corpus: str) -> defaultdict: +def ExtractJSON(e: Extractor, corpus: str) -> dict: """Extract data from JSON based response in JQ like syntax """ - results = defaultdict(OrderedSet) + results = {'internal': {}, 'external': {}, 'extraInfo': []} for j in e.json: res = jq.compile(j).input(corpus).all() if not res: continue if e.name: - results[e.name].add(res) + if e.internal: + results['internal'][e.name] = res + else: + results['external'][e.name] = res + return results else: - results['ExtraInfo'].add(res) - + results['extraInfo'].append(res) return results -def ExtractDSL(e: Extractor, data: dict) -> defaultdict: +def ExtractDSL(e: Extractor, data: dict) -> dict: """Extract data from the response based on a DSL expressions """ - # TODO - raise NotImplementedError + results = {'internal': {}, 'external': {}, 'extraInfo': []} + for expression in e.dsl: + res = Evaluate('{{%s}}' % expression, data) + if res == expression: + continue + if e.name: + if e.internal: + results['internal'][e.name] = res + else: + results['external'][e.name] = res + return results + else: + results['extraInfo'].append(res) + return results diff --git a/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py index 9fed53c6..0fa44b99 100644 --- a/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py @@ -150,5 +150,20 @@ def MatchDSL(matcher: Matcher, data: dict) -> bool: """MatchDSL matches on a generic map result """ - # TODO - raise NotImplementedError + for i, expression in enumerate(matcher.dsl): + result = Evaluate('{{%s}}' % expression, data) + if not isinstance(result, bool): + if matcher.condition == 'and': + return False + elif matcher.condition == 'or': + continue + + if result is False: + if matcher.condition == 'and': + return False + elif matcher.condition == 'or': + continue + + if len(matcher.dsl) - 1 == i: + return True + return False diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py index 29dc4992..bb87771c 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py @@ -18,7 +18,6 @@ from pkg_resources import parse_version from pocsuite3.lib.core.log import LOGGER as logger -from pocsuite3.lib.yaml.nuclei.protocols.common.replacer import Marker def aes_gcm(key: Union[bytes, str], plaintext: Union[bytes, str]) -> bytes: @@ -741,8 +740,8 @@ def Evaluate(inp: str, dynamic_values: dict = {}) -> str: # find expression and execute - OpenMarker, CloseMarker = Marker.ParenthesisOpen, Marker.ParenthesisClose - exps = OrderedDict() + OpenMarker, CloseMarker = '{{', '}}' + exps = {} maxIterations, iterations = 250, 0 data = inp vars().update(dynamic_values) @@ -767,12 +766,16 @@ def Evaluate(inp: str, dynamic_values: dict = {}) -> str: indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker) potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker] try: - result = eval(potentialMatch) + try: + result = eval(potentialMatch) + except SyntaxError: + result = eval(potentialMatch.replace('&&', 'and').replace('||', 'or')) exps[potentialMatch] = result closeMarkerFound = True shouldSearchCloseMarker = False - except Exception as e: - logger.debug(str(e)) + except (SyntaxError, NameError): + import traceback + traceback.print_exc() skip = indexCloseMarkerOffset if closeMarkerFound: @@ -780,11 +783,11 @@ def Evaluate(inp: str, dynamic_values: dict = {}) -> str: else: data = data[indexOpenMarkerOffset:] - if exps: - logger.debug('Expressions: ' + str(exps)) for k, v in exps.items(): - inp = inp.replace(f'{OpenMarker}{k}{CloseMarker}', v) - return inp + logger.debug(f'[+] Expressions: {k} -> {v}') + inp = inp.replace(f'{OpenMarker}{k}{CloseMarker}', str(v)) + + return True if inp == 'True' else False if inp == 'False' else inp if __name__ == '__main__': diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py index 9be98ab9..1549d6c3 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py @@ -1,5 +1,7 @@ import json +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import Evaluate + class Marker: # General marker (open/close) @@ -16,9 +18,9 @@ def marker_replace(data, dynamic_values): data = json.dumps(data) for k, v in dynamic_values.items(): if k in data: - data = data.replace(f'{Marker.General}{k}{Marker.General}', v) - data = data.replace(f'{Marker.ParenthesisOpen}{k}{Marker.ParenthesisClose}', v) + data = data.replace(f'{Marker.General}{k}{Marker.General}', str(v)) + data = data.replace(f'{Marker.ParenthesisOpen}{k}{Marker.ParenthesisClose}', str(v)) - # TODO + data = Evaluate(data, dynamic_values) # various helper functions return json.loads(data) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py index 28770b2d..1238c785 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py @@ -89,20 +89,17 @@ class HttpRequest: # MaxSize is the maximum size of http response body to read in bytes. max_size: int = 0 - # TODO - # cookie-reuse accepts boolean input and false as default, This option not work on pocsuite3 cookie_reuse: bool = False read_all: bool = False redirects: bool = False + host_redirects: bool = False pipeline: bool = False unsafe: bool = False race: bool = False - # TODO # Request condition allows checking for condition between multiple requests for writing complex checks and # exploits involving multiple HTTP request to complete the exploit chain. - req_condition: bool = False stop_at_first_match: bool = True @@ -112,20 +109,46 @@ class HttpRequest: digest_password: str = '' -def getMatchPart(part: str, response: requests.Response, interactsh, return_bytes: bool = False) -> str: - result = b'' - headers = b'' - body = b'' - if isinstance(response, requests.Response): - headers = '\n'.join(f'{k}: {v}' for k, v in response.headers.items()).encode('utf-8') - body = response.content - - if part == 'all': - result = headers + b'\n\n' + body - elif part in ['', 'body']: - result = body - elif part in ['header', 'all_headers']: - result = headers +def responseToDSLMap(resp: requests.Response): + """responseToDSLMap converts an HTTP response to a map for use in DSL matching + """ + data = {} + if not isinstance(resp, requests.Response): + return data + + for k, v in resp.cookies.items(): + data[k.lower()] = v + for k, v in resp.headers.items(): + data[k.lower().replace('-', '_')] = v + + req_headers_raw = '\n'.join(f'{k}: {v}' for k, v in resp.request.headers.items()) + req_body = resp.request.body + if not req_body: + req_body = b'' + if not isinstance(req_body, bytes): + req_body = req_body.encode() + resp_headers_raw = '\n'.join(f'{k}: {v}' for k, v in resp.headers.items()) + resp_body = resp.content + + data['request'] = req_headers_raw.encode() + b'\n\n' + req_body + data['response'] = resp_headers_raw.encode() + b'\n\n' + resp_body + data['status_code'] = resp.status_code + data['body'] = resp_body + data['all_headers'] = resp_headers_raw + data['header'] = resp_headers_raw + data['kval_extractor_dict'] = {} + data['kval_extractor_dict'].update(resp.cookies) + data['kval_extractor_dict'].update(resp.headers) + + return data + + +def getMatchPart(part: str, resp_data: dict, interactsh=None, return_bytes: bool = False) -> str: + if part == '': + part = 'body' + + if part in resp_data: + result = resp_data[part] elif part == 'interactsh_protocol': interactsh.poll() result = '\n'.join(interactsh.interactsh_protocol) @@ -135,6 +158,8 @@ def getMatchPart(part: str, response: requests.Response, interactsh, return_byte elif part == 'interactsh_response': interactsh.poll() result = '\n'.join(interactsh.interactsh_response) + else: + result = '' if return_bytes and not isinstance(result, bytes): result = result.encode() @@ -143,37 +168,37 @@ def getMatchPart(part: str, response: requests.Response, interactsh, return_byte return result -def HttpMatch(request: HttpRequest, response: requests.Response, interactsh): +def HttpMatch(request: HttpRequest, resp_data: dict, interactsh=None): matchers = request.matchers matchers_result = [] for i, matcher in enumerate(matchers): matcher_res = False - item = getMatchPart(matcher.part, response, interactsh, return_bytes=matcher.type == MatcherType.BinaryMatcher) + item = getMatchPart(matcher.part, resp_data, interactsh, matcher.type == MatcherType.BinaryMatcher) if matcher.type == MatcherType.StatusMatcher: - matcher_res = MatchStatusCode(matcher, response.status_code if response else 0) - logger.debug(f'matcher: {matcher}, result: {matcher_res}') + matcher_res = MatchStatusCode(matcher, resp_data['status_code']) + logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.SizeMatcher: matcher_res = MatchSize(matcher, len(item)) - logger.debug(f'matcher: {matcher}, result: {matcher_res}') + logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.WordsMatcher: matcher_res, _ = MatchWords(matcher, item, {}) - logger.debug(f'matcher: {matcher}, result: {matcher_res}') + logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.RegexMatcher: matcher_res, _ = MatchRegex(matcher, item) - logger.debug(f'matcher: {matcher}, result: {matcher_res}') + logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.BinaryMatcher: matcher_res, _ = MatchBinary(matcher, item) - logger.debug(f'matcher: {matcher}, result: {matcher_res}') + logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.DSLMatcher: - matcher_res = MatchDSL(matcher, {}) - logger.debug(f'matcher: {matcher}, result: {matcher_res}') + matcher_res = MatchDSL(matcher, resp_data) + logger.debug(f'[+] {matcher} -> {matcher_res}') if not matcher_res: if request.matchers_condition == 'and': @@ -192,32 +217,33 @@ def HttpMatch(request: HttpRequest, response: requests.Response, interactsh): return False -def HttpExtract(request: HttpRequest, response: requests.Response): +def HttpExtract(request: HttpRequest, resp_data: dict): extractors = request.extractors - extractors_result = [] + extractors_result = {'internal': {}, 'external': {}, 'extraInfo': []} for extractor in extractors: - item = getMatchPart(extractor.part, response) + item = getMatchPart(extractor.part, resp_data) res = None if extractor.type == ExtractorType.RegexExtractor: res = ExtractRegex(extractor, item) - logger.debug(f'extractor: {extractor}, result: {res}') + logger.debug(f'[+] {extractor} -> {res}') elif extractor.type == ExtractorType.KValExtractor: - res = ExtractKval(extractor, response.headers if response else {}) - logger.debug(f'extractor: {extractor}, result: {res}') + res = ExtractKval(extractor, resp_data['kval_extractor_dict']) + logger.debug(f'[+] {extractor} -> {res}') elif extractor.type == ExtractorType.XPathExtractor: res = ExtractXPath(extractor, item) - logger.debug(f'extractor: {extractor}, result: {res}') + logger.debug(f'[+] {extractor} -> {res}') elif extractor.type == ExtractorType.JSONExtractor: res = ExtractJSON(extractor, item) - logger.debug(f'extractor: {extractor}, result: {res}') + logger.debug(f'[+] {extractor} -> {res}') elif ExtractorType.type == ExtractorType.DSLExtractor: res = ExtractDSL(extractor, {}) - logger.debug(f'extractor: {extractor}, result: {res}') + logger.debug(f'[+] {extractor} -> {res}') - if res: - extractors_result.append(res) + extractors_result['internal'].update(res['internal']) + extractors_result['external'].update(res['external']) + extractors_result['extraInfo'] += res['extraInfo'] return extractors_result @@ -248,11 +274,12 @@ def payloadGenerator(request: HttpRequest) -> OrderedDict: def httpRequestGenerator(request: HttpRequest, dynamic_values: OrderedDict): + request_count = len(request.path + request.raw) for payload_instance in payloadGenerator(request): - payload_instance.update(dynamic_values) - + current_index = 0 + dynamic_values.update(payload_instance) for path in request.path + request.raw: - + current_index += 1 method, url, headers, data, kwargs = '', '', '', '', OrderedDict() # base request if path.startswith('{{'): @@ -287,4 +314,5 @@ def httpRequestGenerator(request: HttpRequest, dynamic_values: OrderedDict): kwargs.setdefault('data', data) kwargs.setdefault('headers', headers) - yield (method, marker_replace(url, payload_instance), marker_replace(kwargs, payload_instance)) + yield (method, marker_replace(url, dynamic_values), marker_replace(kwargs, dynamic_values), + payload_instance, request_count, current_index) diff --git a/requirements.txt b/requirements.txt index 351b61ca..2d2f310f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,8 @@ scapy >= 2.4.4 pyOpenSSL >= 20.0.0 Faker >= 0.7.7 pycryptodomex >= 3.9.0 +dacite >= 1.6.0 +PyYAML >= 6.0 +jq >= 1.2.1 +lxml >= 4.6.0 +mmh3 >= 3.0.0 diff --git a/setup.py b/setup.py index 405c3e83..86ccab13 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,12 @@ def find_packages(where='.'): "colorlog", "scapy", "Faker", - "pycryptodomex" + "pycryptodomex", + "dacite", + "PyYAML", + "jq", + "lxml", + "mmh3" ], extras_require={ 'complete': [ From c24efe7e477cca06139e9992a00abbe6d0415c23 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:11:41 -0700 Subject: [PATCH 4/9] feat: code optimization & network request support --- pocsuite3/lib/yaml/nuclei/__init__.py | 168 +++-------- pocsuite3/lib/yaml/nuclei/model/__init__.py | 6 +- .../lib/yaml/nuclei/operators/__init__.py | 48 +-- .../nuclei/operators/extrators/__init__.py | 83 ++++-- .../nuclei/operators/matchers/__init__.py | 72 ++--- .../protocols/common/expressions/__init__.py | 220 +++++++++----- .../protocols/common/expressions/safe_eval.py | 221 ++++++++++++++ .../protocols/common/generators/__init__.py | 30 ++ .../protocols/common/interactsh/__init__.py | 3 +- .../protocols/common/replacer/__init__.py | 19 +- .../yaml/nuclei/protocols/http/__init__.py | 196 ++++++++----- .../yaml/nuclei/protocols/network/__init__.py | 277 ++++++++++++++++++ .../lib/yaml/nuclei/templates/__init__.py | 5 +- 13 files changed, 994 insertions(+), 354 deletions(-) create mode 100644 pocsuite3/lib/yaml/nuclei/protocols/common/expressions/safe_eval.py create mode 100644 pocsuite3/lib/yaml/nuclei/protocols/common/generators/__init__.py create mode 100644 pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py diff --git a/pocsuite3/lib/yaml/nuclei/__init__.py b/pocsuite3/lib/yaml/nuclei/__init__.py index 6ce8060b..b15ec495 100644 --- a/pocsuite3/lib/yaml/nuclei/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/__init__.py @@ -4,20 +4,15 @@ import dacite import yaml -from requests_toolbelt.utils import dump from pocsuite3.lib.core.common import urlparse -from pocsuite3.lib.core.log import LOGGER as logger -from pocsuite3.lib.request import requests from pocsuite3.lib.utils import random_str from pocsuite3.lib.yaml.nuclei.model import Severify from pocsuite3.lib.yaml.nuclei.operators import ExtractorType, MatcherType -from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import Evaluate -from pocsuite3.lib.yaml.nuclei.protocols.http import (AttackType, HttpExtract, - HttpMatch, HTTPMethod, - HttpRequest, - httpRequestGenerator, - responseToDSLMap) +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import evaluate, Marker +from pocsuite3.lib.yaml.nuclei.protocols.common.generators import AttackType +from pocsuite3.lib.yaml.nuclei.protocols.http import HTTPMethod, execute_http_request +from pocsuite3.lib.yaml.nuclei.protocols.network import NetworkInputType, execute_network_request from pocsuite3.lib.yaml.nuclei.templates import Template @@ -27,7 +22,7 @@ def hyphen_to_underscore(dictionary): :param dictionary: :return: the same object with all hyphens replaced by underscore """ - # By default return the same object + # By default, return the same object final_dict = dictionary # for Array perform this method on every object @@ -59,14 +54,15 @@ def expand_preprocessors(data: str) -> str: Ex. {{randstr_1}} which will remain same across the template. randstr is also supported within matchers and can be used to match the inputs. """ - randstr_to_replace = set(m[0] for m in re.findall(r'({{randstr(_\w+)?}})', data)) + randstr_to_replace = set(m[0] for m in re.findall( + fr'({Marker.ParenthesisOpen}randstr(_\w+)?{Marker.ParenthesisClose})', data)) for s in randstr_to_replace: data = data.replace(s, random_str(27)) return data -class Nuclei(): +class Nuclei: def __init__(self, template, target=''): self.yaml_template = template try: @@ -76,97 +72,15 @@ def __init__(self, template, target=''): self.json_template = yaml.safe_load(expand_preprocessors(self.yaml_template)) self.template = dacite.from_dict( Template, hyphen_to_underscore(self.json_template), - config=dacite.Config(cast=[Severify, ExtractorType, MatcherType, HTTPMethod, AttackType])) + config=dacite.Config(cast=[Severify, ExtractorType, MatcherType, HTTPMethod, AttackType, NetworkInputType])) self.target = target - - self.execute_options = OrderedDict() - self.execute_options['stop_at_first_match'] = self.template.stop_at_first_match - self.execute_options['variables'] = self.template.variables - self.execute_options['interactsh'] = None - - self.requests = self.template.requests - + self.interactsh = None self.dynamic_values = OrderedDict() - def execute_request(self, request: HttpRequest) -> dict: - results = [] - resp_data_all = {} - with requests.Session() as session: - try: - for (method, url, kwargs, payload, request_count, current_index) in httpRequestGenerator( - request, self.dynamic_values): - try: - """ - Redirection conditions can be specified per each template. By default, redirects are not followed. - However, if desired, they can be enabled with redirects: true in request details. - 10 redirects are followed at maximum by default which should be good enough for most use cases. - More fine grained control can be exercised over number of redirects followed by using max-redirects - field. - """ - if request.max_redirects: - session.max_redirects = request.max_redirects - else: - session.max_redirects = 10 - response = session.request(method=method, url=url, **kwargs) - # for debug purpose - try: - logger.debug(dump.dump_all(response).decode('utf-8')) - except UnicodeDecodeError: - pass - - except Exception: - import traceback - traceback.print_exc() - response = None - - resp_data = responseToDSLMap(response) - if response: - response.close() - - extractor_res = HttpExtract(request, resp_data) - self.dynamic_values.update(extractor_res['internal']) - - if request.req_condition: - resp_data_all.update(resp_data) - for k, v in resp_data.items(): - resp_data_all[f'{k}_{current_index}'] = v - if current_index == request_count: - resp_data_all.update(self.dynamic_values) - match_res = HttpMatch(request, resp_data_all, self.execute_options['interactsh']) - resp_data_all = {} - if match_res: - output = {} - output.update(extractor_res['external']) - output.update(payload) - output['extraInfo'] = extractor_res['extraInfo'] - results.append(output) - if request.stop_at_first_match: - return results - else: - resp_data.update(self.dynamic_values) - match_res = HttpMatch(request, resp_data, self.execute_options['interactsh']) - if match_res: - output = {} - output.update(extractor_res['external']) - output.update(payload) - output['extraInfo'] = extractor_res['extraInfo'] - results.append(output) - if request.stop_at_first_match: - return results - except Exception: - import traceback - traceback.print_exc() - if results and any(results): - return results - else: - return False - def execute_template(self): - ''' - Dynamic variables can be placed in the path to modify its behavior on runtime. - Variables start with {{ and end with }} and are case-sensitive. - ''' + # Dynamic variables can be placed in the path to modify its behavior on runtime. + # Variables start with {{ and end with }} and are case-sensitive. u = urlparse(self.target) self.dynamic_values['BaseURL'] = self.target @@ -178,44 +92,48 @@ def execute_template(self): self.dynamic_values['Path'] = '/'.join(u.path.split('/')[0:-1]) self.dynamic_values['File'] = u.path.split('/')[-1] - """ - Variables can be used to declare some values which remain constant throughout the template. - The value of the variable once calculated does not change. - Variables can be either simple strings or DSL helper functions. If the variable is a helper function, - it is enclosed in double-curly brackets {{}}. Variables are declared at template level. + # Variables can be used to declare some values which remain constant throughout the template. + # The value of the variable once calculated does not change. + # Variables can be either simple strings or DSL helper functions. If the variable is a helper function, + # it is enclosed in double-curly brackets {{}}. Variables are declared at template level. - Example variables: + # Example variables: - variables: - a1: "test" # A string variable - a2: "{{to_lower(rand_base(5))}}" # A DSL function variable - """ - for k, v in self.execute_options['variables'].items(): - self.dynamic_values[k] = Evaluate(v) + # variables: + # a1: "test" # A string variable + # a2: "{{to_lower(rand_base(5))}}" # A DSL function variable - """ - Since release of Nuclei v2.3.6, Nuclei supports using the interact.sh API to achieve OOB based vulnerability scanning - with automatic Request correlation built in. It's as easy as writing {{interactsh-url}} anywhere in the request. - """ - if '{{interactsh-url}}' in self.yaml_template or '§interactsh-url§' in self.yaml_template: - from pocsuite3.lib.yaml.nuclei.protocols.common.interactsh import \ - InteractshClient - self.execute_options['interactsh'] = InteractshClient() - self.dynamic_values['interactsh-url'] = self.execute_options['interactsh'].client.domain + for k, v in self.template.variables.items(): + self.dynamic_values[k] = evaluate(v) + + # Since release of Nuclei v2.3.6, Nuclei supports using the interact.sh API to achieve OOB based + # vulnerability scanning with automatic Request correlation built in. It's as easy as writing + # {{interactsh-url}} anywhere in the request. + + if (f'{Marker.ParenthesisOpen}interactsh-url{Marker.ParenthesisClose}' in self.yaml_template or + f'{Marker.General}interactsh-url{Marker.General}' in self.yaml_template): + from pocsuite3.lib.yaml.nuclei.protocols.common.interactsh import InteractshClient + self.interactsh = InteractshClient() + self.dynamic_values['interactsh-url'] = self.interactsh.client.domain - for request in self.requests: - res = self.execute_request(request) + for request in self.template.requests: + res = execute_http_request(request, self.dynamic_values, self.interactsh) if res: return res + for request in self.template.network: + res = execute_network_request(request, self.dynamic_values, self.interactsh) + if res: + return res + return False def run(self): return self.execute_template() def __str__(self): - ''' - Convert nuclei template to pocsuite3 - ''' + """ + Convert nuclei template to Pocsuite3 + """ info = [] key_convert = { 'description': 'desc', @@ -238,7 +156,7 @@ def __str__(self): '\n', ' def _verify(self):\n', ' result = {}\n', - ' if not self._check():\n', + ' if not self._check(is_http=%s):\n' % (len(self.template.requests) > 0), ' return self.parse_output(result)\n', " template = '%s'\n" % binascii.hexlify(self.yaml_template.encode()).decode(), ' res = Nuclei(template, self.url).run()\n', diff --git a/pocsuite3/lib/yaml/nuclei/model/__init__.py b/pocsuite3/lib/yaml/nuclei/model/__init__.py index 78d32c75..463b3ed1 100644 --- a/pocsuite3/lib/yaml/nuclei/model/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/model/__init__.py @@ -14,18 +14,20 @@ class Severify(Enum): Unknown = 'unknown' -# Classification contains the vulnerability classification data for a template. @dataclass class Classification: + """Classification contains the vulnerability classification data for a template. + """ cve_id: StrSlice = field(default_factory=list) cwe_id: StrSlice = field(default_factory=list) cvss_metrics: str = '' cvss_score: float = 0.0 -# Info contains metadata information abount a template @dataclass class Info: + """Info contains metadata information abount a template + """ name: str = '' author: StrSlice = field(default_factory=list) tags: StrSlice = field(default_factory=list) diff --git a/pocsuite3/lib/yaml/nuclei/operators/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/__init__.py index 300ef427..351b8503 100644 --- a/pocsuite3/lib/yaml/nuclei/operators/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/operators/__init__.py @@ -1,32 +1,32 @@ -from pocsuite3.lib.yaml.nuclei.operators.extrators import (ExtractDSL, - ExtractJSON, - ExtractKval, - Extractor, +from pocsuite3.lib.yaml.nuclei.operators.extrators import (Extractor, ExtractorType, - ExtractRegex, - ExtractXPath) -from pocsuite3.lib.yaml.nuclei.operators.matchers import (MatchBinary, - MatchDSL, Matcher, - MatcherType, - MatchRegex, - MatchSize, - MatchStatusCode, - MatchWords) + extract_dsl, + extract_json, + extract_kval, + extract_regex, + extract_xpath) +from pocsuite3.lib.yaml.nuclei.operators.matchers import (Matcher, MatcherType, + match_binary, + match_dsl, + match_regex, + match_size, + match_status_code, + match_words) __all__ = [ "ExtractorType", "Extractor", - "ExtractRegex", - "ExtractKval", - "ExtractXPath", - "ExtractJSON", - "ExtractDSL", + "extract_regex", + "extract_kval", + "extract_xpath", + "extract_json", + "extract_dsl", "Matcher", "MatcherType", - "MatchStatusCode", - "MatchSize", - "MatchWords", - "MatchRegex", - "MatchBinary", - "MatchDSL", + "match_status_code", + "match_size", + "match_words", + "match_regex", + "match_binary", + "match_dsl", ] diff --git a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py index 6e68e9a1..a982e915 100644 --- a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py @@ -1,12 +1,13 @@ +import json import re from dataclasses import dataclass, field from enum import Enum import jq -import lxml +from lxml import etree from requests.structures import CaseInsensitiveDict -from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import Evaluate +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import evaluate, UNRESOLVED_VARIABLE, Marker class ExtractorType(Enum): @@ -17,9 +18,10 @@ class ExtractorType(Enum): DSLExtractor = "dsl" -# Extractor is used to extract part of response using a regex. @dataclass class Extractor: + """Extractor is used to extract part of response using a regex. + """ # Name of the extractor. Name should be lowercase and must not contain spaces or underscores (_). name: str = '' @@ -44,20 +46,28 @@ class Extractor: # Attribute is an optional attribute to extract from response XPath. attribute: str = '' + # Extracts using DSL expressions + dsl: list[str] = field(default_factory=list) + # Part is the part of the request response to extract data from. part: str = '' - # Internal, when set to true will allow using the value extracted in the next request for some protocols (like HTTP). + # Internal, when set to true will allow using the value extracted in the next request for some protocols (like + # HTTP). internal: bool = False # CaseInsensitive enables case-insensitive extractions. Default is false. case_insensitive: bool = False -def ExtractRegex(e: Extractor, corpus: str) -> dict: +def extract_regex(e: Extractor, corpus: str) -> dict: """Extract data from response based on a Regular Expression. """ - results = {'internal': {}, 'external': {}, 'extraInfo': []} + results = {'internal': {}, 'external': {}, 'extra_info': []} + + if e.internal and e.name: + results['internal'][e.name] = UNRESOLVED_VARIABLE + for regex in e.regex: matches = re.search(regex, corpus) if not matches: @@ -77,17 +87,21 @@ def ExtractRegex(e: Extractor, corpus: str) -> dict: results['external'][e.name] = res return results else: - results['extraInfo'].append(res) + results['extra_info'].append(res) return results -def ExtractKval(e: Extractor, headers: CaseInsensitiveDict) -> dict: +def extract_kval(e: Extractor, headers: CaseInsensitiveDict) -> dict: """Extract key: value/key=value formatted data from Response Header/Cookie """ if not isinstance(headers, CaseInsensitiveDict): headers = CaseInsensitiveDict(headers) - results = {'internal': {}, 'external': {}, 'extraInfo': []} + results = {'internal': {}, 'external': {}, 'extra_info': []} + + if e.internal and e.name: + results['internal'][e.name] = UNRESOLVED_VARIABLE + for k in e.kval: res = '' if k in headers: @@ -105,18 +119,25 @@ def ExtractKval(e: Extractor, headers: CaseInsensitiveDict) -> dict: results['external'][e.name] = res return results else: - results['extraInfo'].append(res) + results['extra_info'].append(res) return results -def ExtractXPath(e: Extractor, corpus: str) -> dict: +def extract_xpath(e: Extractor, corpus: str) -> dict: """A xpath extractor example to extract value of href attribute from HTML response """ - results = {'internal': {}, 'external': {}, 'extraInfo': []} + results = {'internal': {}, 'external': {}, 'extra_info': []} + + if e.internal and e.name: + results['internal'][e.name] = UNRESOLVED_VARIABLE + if corpus.startswith(' dict: results['external'][e.name] = res return results else: - results['extraInfo'].append(res) + results['extra_info'].append(res) return results -def ExtractJSON(e: Extractor, corpus: str) -> dict: +def extract_json(e: Extractor, corpus: str) -> dict: """Extract data from JSON based response in JQ like syntax """ - results = {'internal': {}, 'external': {}, 'extraInfo': []} + results = {'internal': {}, 'external': {}, 'extra_info': []} + + if e.internal and e.name: + results['internal'][e.name] = UNRESOLVED_VARIABLE + + try: + corpus = json.loads(corpus) + except json.JSONDecodeError: + return results + for j in e.json: - res = jq.compile(j).input(corpus).all() + try: + res = jq.compile(j).input(corpus).all() + except ValueError: + continue if not res: continue @@ -156,16 +189,20 @@ def ExtractJSON(e: Extractor, corpus: str) -> dict: results['external'][e.name] = res return results else: - results['extraInfo'].append(res) + results['extra_info'].append(res) return results -def ExtractDSL(e: Extractor, data: dict) -> dict: +def extract_dsl(e: Extractor, data: dict) -> dict: """Extract data from the response based on a DSL expressions """ - results = {'internal': {}, 'external': {}, 'extraInfo': []} + results = {'internal': {}, 'external': {}, 'extra_info': []} + + if e.internal and e.name: + results['internal'][e.name] = UNRESOLVED_VARIABLE + for expression in e.dsl: - res = Evaluate('{{%s}}' % expression, data) + res = evaluate(f'{Marker.ParenthesisOpen}{expression}{Marker.ParenthesisClose}', data) if res == expression: continue if e.name: @@ -175,5 +212,5 @@ def ExtractDSL(e: Extractor, data: dict) -> dict: results['external'][e.name] = res return results else: - results['extraInfo'].append(res) + results['extra_info'].append(res) return results diff --git a/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py index 0fa44b99..c0368e83 100644 --- a/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from enum import Enum -from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import Evaluate +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import evaluate, Marker class MatcherType(Enum): @@ -15,9 +15,11 @@ class MatcherType(Enum): DSLMatcher = "dsl" -# Matcher is used to match a part in the output from a protocol. @dataclass class Matcher: + """Matcher is used to match a part in the output from a protocol. + """ + # Type is the type of the matcher. type: MatcherType = 'word' @@ -62,27 +64,27 @@ class Matcher: match_all: bool = False -def MatchStatusCode(matcher: Matcher, statusCode: int): - """MatchStatusCode matches a status code check against a corpus +def match_status_code(matcher: Matcher, status_code: int): + """Matches a status code check against a corpus """ - return statusCode in matcher.status + return status_code in matcher.status -def MatchSize(matcher: Matcher, length: int): - """MatchSize matches a size check against a corpus +def match_size(matcher: Matcher, length: int): + """Matches a size check against a corpus """ return length in matcher.size -def MatchWords(matcher: Matcher, corpus: str, data: dict) -> (bool, list): - """MatchWords matches a word check against a corpus +def match_words(matcher: Matcher, corpus: str, data: dict) -> (bool, list): + """Matches a word check against a corpus """ if matcher.case_insensitive: corpus = corpus.lower() - matchedWords = [] + matched_words = [] for i, word in enumerate(matcher.words): - word = Evaluate(word, data) + word = evaluate(word, data) if word not in corpus: if matcher.condition == 'and': @@ -93,44 +95,46 @@ def MatchWords(matcher: Matcher, corpus: str, data: dict) -> (bool, list): if matcher.condition == 'or' and not matcher.match_all: return True, [word] - matchedWords.append(word) + matched_words.append(word) if len(matcher.words) - 1 == i and not matcher.match_all: - return True, MatchWords + return True, matched_words - if len(matchedWords) > 0 and matcher.match_all: - return True, MatchWords + if len(matched_words) > 0 and matcher.match_all: + return True, matched_words return False, [] -def MatchRegex(matcher: Matcher, corpus: str) -> (bool, list): - """MatchRegex matches a regex check against a corpus +def match_regex(matcher: Matcher, corpus: str) -> (bool, list): + """Matches a regex check against a corpus """ - matchedRegexes = [] + matched_regexes = [] for i, regex in enumerate(matcher.regex): if not re.search(regex, corpus): if matcher.condition == 'and': return False, [] elif matcher.condition == 'or': continue - currentMatches = re.findall(regex, corpus) + + current_matches = re.findall(regex, corpus) if matcher.condition == 'or' and not matcher.match_all: - return True, matchedRegexes + return True, matched_regexes - matchedRegexes = matchedRegexes + currentMatches + matched_regexes = matched_regexes + current_matches if len(matcher.regex) - 1 == i and not matcher.match_all: - return True, matchedRegexes - if len(matchedRegexes) > 0 and matcher.match_all: - return True, matchedRegexes + return True, matched_regexes + + if len(matched_regexes) > 0 and matcher.match_all: + return True, matched_regexes return False, [] -def MatchBinary(matcher: Matcher, corpus: bytes) -> (bool, list): - """MatchBinary matches a binary check against a corpus +def match_binary(matcher: Matcher, corpus: bytes) -> (bool, list): + """Matches a binary check against a corpus """ - matchedBinary = [] + matched_binary = [] for i, binary in enumerate(matcher.binary): binary = binascii.unhexlify(binary) if binary not in corpus: @@ -138,20 +142,22 @@ def MatchBinary(matcher: Matcher, corpus: bytes) -> (bool, list): return False, [] elif matcher.condition == 'or': continue + if matcher.condition == 'or': return True, [binary] - MatchBinary.append(binary) + + matched_binary.append(binary) if len(matcher.binary) - 1 == i: - return True, matchedBinary + return True, matched_binary + return False, [] -def MatchDSL(matcher: Matcher, data: dict) -> bool: - """MatchDSL matches on a generic map result +def match_dsl(matcher: Matcher, data: dict) -> bool: + """Matches on a generic map result """ - for i, expression in enumerate(matcher.dsl): - result = Evaluate('{{%s}}' % expression, data) + result = evaluate(f'{Marker.ParenthesisOpen}{expression}{Marker.ParenthesisClose}', data) if not isinstance(result, bool): if matcher.condition == 'and': return False diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py index bb87771c..1a9f1b02 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py @@ -11,13 +11,24 @@ import time import urllib.parse import zlib as py_built_in_zlib -from collections import OrderedDict from typing import Union import mmh3 as py_mmh3 from pkg_resources import parse_version from pocsuite3.lib.core.log import LOGGER as logger +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions.safe_eval import safe_eval + +UNRESOLVED_VARIABLE = '---UNRESOLVED-VARIABLE---' + + +class Marker: + # General marker (open/close) + General = "§" + # ParenthesisOpen marker - begin of a placeholder + ParenthesisOpen = "{{" + # ParenthesisClose marker - end of a placeholder + ParenthesisClose = "}}" def aes_gcm(key: Union[bytes, str], plaintext: Union[bytes, str]) -> bytes: @@ -77,7 +88,7 @@ def concat(*arguments) -> str: return ''.join(map(str, arguments)) -def compare_versions(versionToCheck: str, *constraints: str) -> bool: +def compare_versions(version_to_check: str, *constraints: str) -> bool: """ Compares the first version argument with the provided constraints @@ -85,7 +96,7 @@ def compare_versions(versionToCheck: str, *constraints: str) -> bool: Input: compare_versions('v1.0.0', '>v0.0.1', '=]*', constraint)[0] @@ -114,7 +125,7 @@ def contains(inp: str, substring: str) -> bool: def contains_all(inp: str, *substrings: str) -> bool: """ - Verifies if any input contains all of the substrings + Verify if any input contains all the substrings Example: Input: contains_all("Hello everyone", "lo", "every") @@ -147,7 +158,7 @@ def dec_to_hex(number: Union[str, int]) -> str: return hex(number)[2:] -def hex_to_dec(hexNumber: Union[str, int]) -> int: +def hex_to_dec(hex_number: Union[str, int]) -> int: """ Transforms the input hexadecimal number into decimal format @@ -156,10 +167,10 @@ def hex_to_dec(hexNumber: Union[str, int]) -> int: hex_to_dec("0xff") Output: 255 """ - return int(str(hexNumber), 16) + return int(str(hex_number), 16) -def bin_to_dec(binaryNumber: Union[str, int]) -> int: +def bin_to_dec(binary_number: Union[str, int]) -> int: """ Transforms the input binary number into a decimal format @@ -168,10 +179,10 @@ def bin_to_dec(binaryNumber: Union[str, int]) -> int: bin_to_dec(1010) Output: 10 """ - return int(str(binaryNumber), 2) + return int(str(binary_number), 2) -def oct_to_dec(octalNumber: Union[str, int]) -> int: +def oct_to_dec(octal_number: Union[str, int]) -> int: """ Transforms the input octal number into a decimal format @@ -180,7 +191,7 @@ def oct_to_dec(octalNumber: Union[str, int]) -> int: oct_to_dec(1234567) Output: 342391 """ - return int(str(octalNumber), 8) + return int(str(octal_number), 8) def generate_java_gadget(gadget: str, cmd: str, encoding: str) -> str: @@ -326,7 +337,7 @@ def print_debug(*args) -> None: raise NotImplementedError -def rand_base(length: int, optionalCharSet: str = string.ascii_letters+string.digits) -> str: +def rand_base(length: int, optional_charset: str = string.ascii_letters+string.digits) -> str: """ Generates a random sequence of given length string from an optional charset (defaults to letters and numbers) @@ -334,10 +345,10 @@ def rand_base(length: int, optionalCharSet: str = string.ascii_letters+string.di Input: rand_base(5, "abc") Output: caccb """ - return ''.join(random.choice(optionalCharSet) for _ in range(length)) + return ''.join(random.choice(optional_charset) for _ in range(length)) -def rand_char(optionalCharSet: str = string.ascii_letters + string.digits) -> str: +def rand_char(optional_charset: str = string.ascii_letters + string.digits) -> str: """ Generates a random character from an optional character set (defaults to letters and numbers) @@ -345,10 +356,10 @@ def rand_char(optionalCharSet: str = string.ascii_letters + string.digits) -> st Input: rand_char("abc") Output: a """ - return random.choice(optionalCharSet) + return random.choice(optional_charset) -def rand_int(optionalMin: int = 0, optionalMax: int = 2147483647) -> int: +def rand_int(optional_min: int = 0, optional_max: int = 2147483647) -> int: """ Generates a random integer between the given optional limits (defaults to 0 - MaxInt32) @@ -356,10 +367,10 @@ def rand_int(optionalMin: int = 0, optionalMax: int = 2147483647) -> int: Input: rand_int(1, 10) Output: 6 """ - return random.randint(optionalMin, optionalMax) + return random.randint(optional_min, optional_max) -def rand_text_alpha(length: int, optionalBadChars: str = '') -> str: +def rand_text_alpha(length: int, optional_bad_chars: str = '') -> str: """ Generates a random string of letters, of given length, excluding the optional cutset characters @@ -367,11 +378,11 @@ def rand_text_alpha(length: int, optionalBadChars: str = '') -> str: Input: rand_text_alpha(10, "abc") Output: WKozhjJWlJ """ - charset = ''.join(i if i not in optionalBadChars else '' for i in string.ascii_letters) + charset = ''.join(i if i not in optional_bad_chars else '' for i in string.ascii_letters) return ''.join(random.choice(charset) for _ in range(length)) -def rand_text_alphanumeric(length: int, optionalBadChars: str = '') -> str: +def rand_text_alphanumeric(length: int, optional_bad_chars: str = '') -> str: """ Generates a random alphanumeric string, of given length without the optional cutset characters @@ -379,11 +390,11 @@ def rand_text_alphanumeric(length: int, optionalBadChars: str = '') -> str: Input: rand_text_alphanumeric(10, "ab12") Output: NthI0IiY8r """ - charset = ''.join(i if i not in optionalBadChars else '' for i in string.ascii_letters + string.digits) + charset = ''.join(i if i not in optional_bad_chars else '' for i in string.ascii_letters + string.digits) return ''.join(random.choice(charset) for _ in range(length)) -def rand_text_numeric(length: int, optionalBadNumbers: str = '') -> str: +def rand_text_numeric(length: int, optional_bad_numbers: str = '') -> str: """ Generates a random numeric string of given length without the optional set of undesired numbers @@ -391,7 +402,7 @@ def rand_text_numeric(length: int, optionalBadNumbers: str = '') -> str: Input: rand_text_numeric(10, 123) Output: 0654087985 """ - charset = ''.join(i if i not in optionalBadNumbers else '' for i in string.digits) + charset = ''.join(i if i not in optional_bad_numbers else '' for i in string.digits) return ''.join(random.choice(charset) for _ in range(length)) @@ -439,7 +450,7 @@ def replace(inp: str, old: str, new: str) -> str: return inp.replace(old, new) -def replace_regex(source: str, regex: str, replacement: str) -> str: +def replace_regex(source: str, pattern: str, replacement: str) -> str: """ Replaces substrings matching the given regular expression in the input @@ -447,7 +458,7 @@ def replace_regex(source: str, regex: str, replacement: str) -> str: Input: replace_regex("He123llo", "(\\d+)", "") Output: Hello """ - return re.sub(regex, replacement, source) + return re.sub(pattern, replacement, source) def reverse(inp: str) -> str: @@ -585,7 +596,7 @@ def trim_suffix(inp: str, suffix: str) -> str: return inp -def unix_time(optionalSeconds: int = 0) -> int: +def unix_time(optional_seconds: int = 0) -> int: """ Returns the current Unix time (number of seconds elapsed since January 1, 1970 UTC) with the added optional seconds @@ -593,7 +604,7 @@ def unix_time(optionalSeconds: int = 0) -> int: Input: unix_time(10) Output: 1639568278 """ - return int(time.time()) + optionalSeconds + return int(time.time()) + optional_seconds def url_decode(inp: str) -> str: @@ -656,7 +667,7 @@ def hmac(algorithm: str, data: Union[bytes, str], secret: Union[bytes, str]) -> return py_hmac.new(secret, data, algorithm).hexdigest() -def date_time(dateTimeFormat: str, optionalUnixTime: int = int(time.time())) -> str: +def date_time(date_time_format: str, optional_unix_time: int = int(time.time())) -> str: """ Returns the formatted date time using simplified or go style layout for the current or the given unix time @@ -665,7 +676,7 @@ def date_time(dateTimeFormat: str, optionalUnixTime: int = int(time.time())) -> date_time("%Y-%m-%d %H:%M", 1654870680) Output: 2022-06-10 14:18 """ - return datetime.datetime.utcfromtimestamp(optionalUnixTime).strftime(dateTimeFormat) + return datetime.datetime.utcfromtimestamp(optional_unix_time).strftime(date_time_format) def to_unix_time(inp: str, layout: str = "%Y-%m-%d %H:%M:%S") -> int: @@ -731,66 +742,137 @@ def line_ends_with(inp: str, *suffix: str) -> bool: return False -def Evaluate(inp: str, dynamic_values: dict = {}) -> str: +def evaluate(inp: str, dynamic_values: dict = None) -> str: """ - Evaluate checks if the match contains a dynamic variable, for each + evaluate checks if the match contains a dynamic variable, for each found one we will check if it's an expression and can be compiled, it will be evaluated and the results will be returned. """ - # find expression and execute - - OpenMarker, CloseMarker = '{{', '}}' - exps = {} - maxIterations, iterations = 250, 0 + if dynamic_values is None: + dynamic_values = {} + + variables = { + 'aes_gcm': aes_gcm, + 'base64': base64, + 'base64_decode': base64_decode, + 'base64_py': base64_py, + 'concat': concat, + 'compare_versions': compare_versions, + 'contains': contains, + 'contains_all': contains_all, + 'contains_any': contains_any, + 'dec_to_hex': dec_to_hex, + 'hex_to_dec': hex_to_dec, + 'bin_to_dec': bin_to_dec, + 'oct_to_dec': oct_to_dec, + 'generate_java_gadget': generate_java_gadget, + 'gzip': gzip, + 'gzip_decode': gzip_decode, + 'zlib': zlib, + 'zlib_decode': zlib_decode, + 'hex_decode': hex_decode, + 'hex_encode': hex_encode, + 'html_escape': html_escape, + 'html_unescape': html_unescape, + 'md5': md5, + 'mmh3': mmh3, + 'print_debug': print_debug, + 'rand_base': rand_base, + 'rand_char': rand_char, + 'rand_int': rand_int, + 'rand_text_alpha': rand_text_alpha, + 'rand_text_alphanumeric': rand_text_alphanumeric, + 'rand_text_numeric': rand_text_numeric, + 'regex': regex, + 'remove_bad_chars': remove_bad_chars, + 'repeat': repeat, + 'replace': replace, + 'replace_regex': replace_regex, + 'reverse': reverse, + 'sha1': sha1, + 'sha256': sha256, + 'to_lower': to_lower, + 'to_upper': to_upper, + 'trim': trim, + 'trim_left': trim_left, + 'trim_prefix': trim_prefix, + 'trim_right': trim_right, + 'trim_space': trim_space, + 'trim_suffix': trim_suffix, + 'unix_time': unix_time, + 'url_decode': url_decode, + 'url_encode': url_encode, + 'wait_for': wait_for, + 'join': join, + 'hmac': hmac, + 'date_time': date_time, + 'to_unix_time': to_unix_time, + 'starts_with': starts_with, + 'line_starts_with': line_starts_with, + 'ends_with': ends_with, + 'line_ends_with': line_ends_with, + } + variables.update(dynamic_values) + open_marker, close_marker = Marker.ParenthesisOpen, Marker.ParenthesisClose + expressions = {} + max_iterations, iterations = 250, 0 data = inp - vars().update(dynamic_values) - while iterations <= maxIterations: + while iterations <= max_iterations: iterations += 1 - indexOpenMarker = data.find(OpenMarker) - if indexOpenMarker < 0: + index_open_marker = data.find(open_marker) + if index_open_marker < 0: break - indexOpenMarkerOffset = indexOpenMarker + len(OpenMarker) - shouldSearchCloseMarker = True - closeMarkerFound = False - innerData = data - skip = indexOpenMarkerOffset + index_open_marker_offset = index_open_marker + len(open_marker) + should_search_close_marker = True + close_marker_found = False + inner_data = data + skip = index_open_marker_offset - while shouldSearchCloseMarker: - indexCloseMarker = innerData.find(CloseMarker, skip) - if indexCloseMarker < 0: - shouldSearchCloseMarker = False + while should_search_close_marker: + index_close_marker = inner_data.find(close_marker, skip) + if index_close_marker < 0: + should_search_close_marker = False continue - indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker) - potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker] + index_close_marker_offset = index_close_marker + len(close_marker) + potential_match = inner_data[index_open_marker_offset:index_close_marker] try: - try: - result = eval(potentialMatch) - except SyntaxError: - result = eval(potentialMatch.replace('&&', 'and').replace('||', 'or')) - exps[potentialMatch] = result - closeMarkerFound = True - shouldSearchCloseMarker = False - except (SyntaxError, NameError): + result = safe_eval(potential_match, variables) + + if UNRESOLVED_VARIABLE in str(result): + return UNRESOLVED_VARIABLE + expressions[potential_match] = result + close_marker_found = True + should_search_close_marker = False + except Exception: import traceback traceback.print_exc() - skip = indexCloseMarkerOffset + skip = index_close_marker_offset - if closeMarkerFound: - data = data[indexCloseMarkerOffset:] + if close_marker_found: + data = data[index_close_marker_offset:] else: - data = data[indexOpenMarkerOffset:] + data = data[index_open_marker_offset:] - for k, v in exps.items(): + result = inp + for k, v in expressions.items(): logger.debug(f'[+] Expressions: {k} -> {v}') - inp = inp.replace(f'{OpenMarker}{k}{CloseMarker}', str(v)) - - return True if inp == 'True' else False if inp == 'False' else inp + k = f'{open_marker}{k}{close_marker}' + if k == inp: + return v + elif isinstance(v, bytes): + if not isinstance(result, bytes): + result = result.encode() + k = k.encode() + result = result.replace(k, v) + else: + result = result.replace(k, str(v)) + return result if __name__ == '__main__': - print(Evaluate("{{to_lower(rand_base(5))}}")) - print(Evaluate("{{base64('World')}}")) - print(Evaluate("{{base64(Hello)}}", {'Hello': 'World'})) + print(evaluate("{{to_lower(rand_base(5))}}")) + print(evaluate("{{base64('World')}}")) + print(evaluate("{{base64(Hello)}}", {'Hello': 'World'})) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/safe_eval.py b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/safe_eval.py new file mode 100644 index 00000000..38d5fb6a --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/safe_eval.py @@ -0,0 +1,221 @@ +# https://raw.githubusercontent.com/galaxyproject/galaxy/a981724c22f5bc6b3696c25faa896b3e96db88db/lib/galaxy/jobs/runners/state_handlers/_safe_eval.py + +from ast import ( + Module, + parse, + walk, +) + + +AST_NODE_TYPE_ALLOWLIST = [ + "Expr", + "Load", + "Str", + "Num", + "BoolOp", + "Compare", + "And", + "Eq", + "NotEq", + "Or", + "GtE", + "LtE", + "Lt", + "Gt", + "BinOp", + "Add", + "Div", + "Sub", + "Mult", + "Mod", + "Pow", + "LShift", + "GShift", + "BitAnd", + "BitOr", + "BitXor", + "UnaryOp", + "Invert", + "Not", + "NotIn", + "In", + "Is", + "IsNot", + "List", + "Index", + "Subscript", + "Constant", + # Further checks + "Name", + "Call", + "Attribute", +] + + +BUILTIN_AND_MATH_FUNCTIONS = [ + 'abs', + 'all', + 'any', + 'bin', + 'chr', + 'cmp', + 'complex', + 'divmod', + 'float', + 'hex', + 'int', + 'len', + 'long', + 'max', + 'min', + 'oct', + 'ord', + 'pow', + 'range', + 'reversed', + 'round', + 'sorted', + 'str', + 'sum', + 'type', + 'unichr', + 'unicode', + 'log', + 'exp', + 'sqrt', + 'ceil', + 'floor' +] + +STRING_AND_LIST_METHODS = [name for name in dir("") + dir([]) if not name.startswith("_")] +VALID_FUNCTIONS = BUILTIN_AND_MATH_FUNCTIONS + STRING_AND_LIST_METHODS + + +def _check_name(ast_node, allowed_variables=None): + if allowed_variables is None: + allowed_variables = [] + name = ast_node.id + return name in VALID_FUNCTIONS + allowed_variables + + +def _check_attribute(ast_node): + attribute_name = ast_node.attr + if attribute_name not in STRING_AND_LIST_METHODS: + return False + return True + + +def _check_call(ast_node, allowed_variables=None): + if allowed_variables is None: + allowed_variables = [] + # If we are calling a function or method, it better be a math, + # string or list function. + ast_func = ast_node.func + ast_func_class = ast_func.__class__.__name__ + if ast_func_class == "Name": + if ast_func.id not in BUILTIN_AND_MATH_FUNCTIONS + allowed_variables: + return False + elif ast_func_class == "Attribute": + if not _check_attribute(ast_func): + return False + else: + return False + + return True + + +def _check_expression(text, allowed_variables=None): + """ + + >>> allowed_variables = ["c1", "c2", "c3", "c4", "c5"] + >>> _check_expression("c1", allowed_variables) + True + >>> _check_expression("eval('1+1')", allowed_variables) + False + >>> _check_expression("import sys", allowed_variables) + False + >>> _check_expression("[].__str__", allowed_variables) + False + >>> _check_expression("__builtins__", allowed_variables) + False + >>> _check_expression("'x' in globals", allowed_variables) + False + >>> _check_expression("'x' in [1,2,3]", allowed_variables) + True + >>> _check_expression("c3=='chr1' and c5>5", allowed_variables) + True + >>> _check_expression("c3=='chr1' and d5>5", allowed_variables) # Invalid d5 reference + False + >>> _check_expression("c3=='chr1' and c5>5 or exec", allowed_variables) + False + >>> _check_expression("type(c1) != type(1)", allowed_variables) + True + >>> _check_expression("c1.split(',')[1] == '1'", allowed_variables) + True + >>> _check_expression("exec 1", allowed_variables) + False + >>> _check_expression("str(c2) in [\\\"a\\\",\\\"b\\\"]", allowed_variables) + True + """ + if allowed_variables is None: + allowed_variables = [] + try: + module = parse(text) + except SyntaxError: + return False + + if not isinstance(module, Module): + return False + statements = module.body + if not len(statements) == 1: + return False + expression = statements[0] + if expression.__class__.__name__ != "Expr": + return False + + for ast_node in walk(expression): + ast_node_class = ast_node.__class__.__name__ + + # Toss out everything that is not a "simple" expression, + # imports, error handling, etc... + if ast_node_class not in AST_NODE_TYPE_ALLOWLIST: + return False + + # White-list more potentially dangerous types AST elements. + if ast_node_class == "Name": + # In order to prevent loading 'exec', 'eval', etc... + # put string restriction on names allowed. + if not _check_name(ast_node, allowed_variables): + return False + # Check only valid, white-listed functions are called. + elif ast_node_class == "Call": + if not _check_call(ast_node, allowed_variables): + return False + # Check only valid, white-listed attributes are accessed + elif ast_node_class == "Attribute": + if not _check_attribute(ast_node): + return False + + return True + + +def safe_eval(expression, variables): + """ + + >>> safe_eval("moo", {"moo": 5}) + 5 + >>> exception_thrown = False + >>> try: safe_eval("moo", {"cow": 5}) + ... except Exception as e: exception_thrown = True + >>> exception_thrown + True + """ + if not _check_expression(expression, allowed_variables=list(variables.keys())): + expression = expression.replace(' && ', ' and ').replace(' || ', ' or ') + if not _check_expression(expression, allowed_variables=list(variables.keys())): + raise Exception(f"Invalid expression [{expression}], only a very simple subset of Python is allowed.") + return eval(expression, globals(), variables) + + +if __name__ == '__main__': + print(safe_eval("moo", {"moo": 5})) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/generators/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/generators/__init__.py new file mode 100644 index 00000000..d48d680e --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/generators/__init__.py @@ -0,0 +1,30 @@ +import itertools +from collections import OrderedDict +from enum import Enum + +from pocsuite3.lib.core.common import check_file, get_file_items + + +class AttackType(Enum): + BatteringRamAttack = "batteringram" + PitchForkAttack = "pitchfork" + ClusterBombAttack = "clusterbomb" + + +def payload_generator(payloads: dict, attack_type: AttackType) -> OrderedDict: + payloads_final = OrderedDict() + payloads_final.update(payloads) + + for k, v in payloads_final.items(): + if isinstance(v, str) and check_file(v): + payloads_final[k] = get_file_items(v) + + payload_keys, payload_vals = payloads_final.keys(), payloads_final.values() + payload_vals = [i if isinstance(i, list) else [i] for i in payload_vals] + + if attack_type == AttackType.PitchForkAttack: + for instance in zip(*payload_vals): + yield dict(zip(payload_keys, instance)) + else: + for instance in itertools.product(*payload_vals): + yield dict(zip(payload_keys, instance)) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/interactsh/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/interactsh/__init__.py index ecb4604f..edb2b337 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/interactsh/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/interactsh/__init__.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field -from pocsuite3.modules.interactsh import Interactsh + from pocsuite3.lib.core.log import LOGGER as logger +from pocsuite3.modules.interactsh import Interactsh @dataclass diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py index 1549d6c3..0a3bc704 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/replacer/__init__.py @@ -1,15 +1,10 @@ import json -from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import Evaluate +from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import evaluate, UNRESOLVED_VARIABLE, Marker -class Marker: - # General marker (open/close) - General = "§" - # ParenthesisOpen marker - begin of a placeholder - ParenthesisOpen = "{{" - # ParenthesisClose marker - end of a placeholder - ParenthesisClose = "}}" +class UnresolvedVariableException(Exception): + pass def marker_replace(data, dynamic_values): @@ -21,6 +16,10 @@ def marker_replace(data, dynamic_values): data = data.replace(f'{Marker.General}{k}{Marker.General}', str(v)) data = data.replace(f'{Marker.ParenthesisOpen}{k}{Marker.ParenthesisClose}', str(v)) - data = Evaluate(data, dynamic_values) - # various helper functions + # execute various helper functions + data = evaluate(data, dynamic_values) + + if UNRESOLVED_VARIABLE in data: + raise UnresolvedVariableException + return json.loads(data) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py index 1238c785..8a222815 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py @@ -1,27 +1,23 @@ -import itertools from collections import OrderedDict from dataclasses import dataclass, field from enum import Enum +from typing import Union -import requests +from requests_toolbelt.utils import dump -from pocsuite3.lib.core.common import check_file, get_file_items from pocsuite3.lib.core.log import LOGGER as logger -from pocsuite3.lib.yaml.nuclei.operators import (ExtractDSL, ExtractJSON, - ExtractKval, Extractor, - ExtractorType, ExtractRegex, - ExtractXPath, MatchBinary, - MatchDSL, Matcher, - MatcherType, MatchRegex, - MatchSize, MatchStatusCode, - MatchWords) -from pocsuite3.lib.yaml.nuclei.protocols.common.replacer import marker_replace - - -class AttackType(Enum): - BatteringRamAttack = "batteringram" - PitchForkAttack = "pitchfork" - ClusterBombAttack = "clusterbomb" +from pocsuite3.lib.request import requests +from pocsuite3.lib.yaml.nuclei.operators import (Extractor, ExtractorType, + Matcher, MatcherType, + extract_dsl, extract_json, + extract_kval, extract_regex, + extract_xpath, match_binary, + match_dsl, match_regex, + match_size, match_status_code, + match_words) +from pocsuite3.lib.yaml.nuclei.protocols.common.generators import AttackType, payload_generator +from pocsuite3.lib.yaml.nuclei.protocols.common.replacer import ( + UnresolvedVariableException, UNRESOLVED_VARIABLE, marker_replace, Marker) class HTTPMethod(Enum): @@ -38,9 +34,11 @@ class HTTPMethod(Enum): HTTPDebug = "DEBUG" -# HttpRequest contains a http request to be made from a template @dataclass class HttpRequest: + """HttpRequest contains a http request to be made from a template + """ + # Operators for the current request go here. matchers: list[Matcher] = field(default_factory=list) extractors: list[Extractor] = field(default_factory=list) @@ -109,8 +107,8 @@ class HttpRequest: digest_password: str = '' -def responseToDSLMap(resp: requests.Response): - """responseToDSLMap converts an HTTP response to a map for use in DSL matching +def http_response_to_dsl_map(resp: requests.Response): + """Converts an HTTP response to a map for use in DSL matching """ data = {} if not isinstance(resp, requests.Response): @@ -143,7 +141,7 @@ def responseToDSLMap(resp: requests.Response): return data -def getMatchPart(part: str, resp_data: dict, interactsh=None, return_bytes: bool = False) -> str: +def http_get_match_part(part: str, resp_data: dict, interactsh=None, return_bytes: bool = False) -> str: if part == '': part = 'body' @@ -164,40 +162,43 @@ def getMatchPart(part: str, resp_data: dict, interactsh=None, return_bytes: bool if return_bytes and not isinstance(result, bytes): result = result.encode() elif not return_bytes and isinstance(result, bytes): - result = result.decode() + try: + result = result.decode() + except UnicodeDecodeError: + result = str(result) return result -def HttpMatch(request: HttpRequest, resp_data: dict, interactsh=None): +def http_match(request: HttpRequest, resp_data: dict, interactsh=None): matchers = request.matchers matchers_result = [] for i, matcher in enumerate(matchers): matcher_res = False - item = getMatchPart(matcher.part, resp_data, interactsh, matcher.type == MatcherType.BinaryMatcher) + item = http_get_match_part(matcher.part, resp_data, interactsh, matcher.type == MatcherType.BinaryMatcher) if matcher.type == MatcherType.StatusMatcher: - matcher_res = MatchStatusCode(matcher, resp_data['status_code']) + matcher_res = match_status_code(matcher, resp_data['status_code']) logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.SizeMatcher: - matcher_res = MatchSize(matcher, len(item)) + matcher_res = match_size(matcher, len(item)) logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.WordsMatcher: - matcher_res, _ = MatchWords(matcher, item, {}) + matcher_res, _ = match_words(matcher, item, {}) logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.RegexMatcher: - matcher_res, _ = MatchRegex(matcher, item) + matcher_res, _ = match_regex(matcher, item) logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.BinaryMatcher: - matcher_res, _ = MatchBinary(matcher, item) + matcher_res, _ = match_binary(matcher, item) logger.debug(f'[+] {matcher} -> {matcher_res}') elif matcher.type == MatcherType.DSLMatcher: - matcher_res = MatchDSL(matcher, resp_data) + matcher_res = match_dsl(matcher, resp_data) logger.debug(f'[+] {matcher} -> {matcher_res}') if not matcher_res: @@ -217,33 +218,33 @@ def HttpMatch(request: HttpRequest, resp_data: dict, interactsh=None): return False -def HttpExtract(request: HttpRequest, resp_data: dict): +def http_extract(request: HttpRequest, resp_data: dict): extractors = request.extractors - extractors_result = {'internal': {}, 'external': {}, 'extraInfo': []} + extractors_result = {'internal': {}, 'external': {}, 'extra_info': []} for extractor in extractors: - item = getMatchPart(extractor.part, resp_data) + item = http_get_match_part(extractor.part, resp_data) res = None if extractor.type == ExtractorType.RegexExtractor: - res = ExtractRegex(extractor, item) + res = extract_regex(extractor, item) logger.debug(f'[+] {extractor} -> {res}') elif extractor.type == ExtractorType.KValExtractor: - res = ExtractKval(extractor, resp_data['kval_extractor_dict']) + res = extract_kval(extractor, resp_data['kval_extractor_dict']) logger.debug(f'[+] {extractor} -> {res}') elif extractor.type == ExtractorType.XPathExtractor: - res = ExtractXPath(extractor, item) + res = extract_xpath(extractor, item) logger.debug(f'[+] {extractor} -> {res}') elif extractor.type == ExtractorType.JSONExtractor: - res = ExtractJSON(extractor, item) + res = extract_json(extractor, item) logger.debug(f'[+] {extractor} -> {res}') - elif ExtractorType.type == ExtractorType.DSLExtractor: - res = ExtractDSL(extractor, {}) + elif extractor.type == ExtractorType.DSLExtractor: + res = extract_dsl(extractor, resp_data) logger.debug(f'[+] {extractor} -> {res}') extractors_result['internal'].update(res['internal']) extractors_result['external'].update(res['external']) - extractors_result['extraInfo'] += res['extraInfo'] + extractors_result['extra_info'] += res['extra_info'] return extractors_result @@ -254,35 +255,16 @@ def extract_dict(text, line_sep='\n', kv_sep='='): return _dict -def payloadGenerator(request: HttpRequest) -> OrderedDict: - payloads = OrderedDict() - payloads.update(request.payloads) - - for k, v in payloads.items(): - if isinstance(v, str) and check_file(v): - payloads[k] = get_file_items(v) - - payload_keys, payload_vals = payloads.keys(), payloads.values() - payload_vals = [i if isinstance(i, list) else [i] for i in payload_vals] - - if request.attack == AttackType.PitchForkAttack: - for instance in zip(*payload_vals): - yield dict(zip(payload_keys, instance)) - else: - for instance in itertools.product(*payload_vals): - yield dict(zip(payload_keys, instance)) - - -def httpRequestGenerator(request: HttpRequest, dynamic_values: OrderedDict): +def http_request_generator(request: HttpRequest, dynamic_values: OrderedDict): request_count = len(request.path + request.raw) - for payload_instance in payloadGenerator(request): + for payload_instance in payload_generator(request.payloads, request.attack): current_index = 0 dynamic_values.update(payload_instance) for path in request.path + request.raw: current_index += 1 method, url, headers, data, kwargs = '', '', '', '', OrderedDict() # base request - if path.startswith('{{'): + if path.startswith(Marker.ParenthesisOpen): method = request.method.value headers = request.headers data = request.body @@ -293,7 +275,7 @@ def httpRequestGenerator(request: HttpRequest, dynamic_values: OrderedDict): raw = path.strip() raws = list(map(lambda x: x.strip(), raw.splitlines())) method, path, _ = raws[0].split(' ') - url = f'{{{{BaseURL}}}}{path}' + url = f'{Marker.ParenthesisOpen}BaseURL{Marker.ParenthesisClose}{path}' if method == "POST": index = 0 @@ -314,5 +296,87 @@ def httpRequestGenerator(request: HttpRequest, dynamic_values: OrderedDict): kwargs.setdefault('data', data) kwargs.setdefault('headers', headers) - yield (method, marker_replace(url, dynamic_values), marker_replace(kwargs, dynamic_values), - payload_instance, request_count, current_index) + try: + url = marker_replace(url, dynamic_values) + kwargs = marker_replace(kwargs, dynamic_values) + except UnresolvedVariableException: + continue + + yield method, url, kwargs, payload_instance, request_count, current_index + + +def execute_http_request(request: HttpRequest, dynamic_values, interactsh) -> Union[bool, list]: + results = [] + resp_data_all = {} + with requests.Session() as session: + try: + for (method, url, kwargs, payload, request_count, current_index) in http_request_generator( + request, dynamic_values): + try: + # Redirection conditions can be specified per each template. By default, redirects are not + # followed. However, if desired, they can be enabled with redirects: true in request details. 10 + # redirects are followed at maximum by default which should be good enough for most use cases. + # More fine grained control can be exercised over number of redirects followed by using + # max-redirects field. + + if request.max_redirects: + session.max_redirects = request.max_redirects + else: + session.max_redirects = 10 + response = session.request(method=method, url=url, **kwargs) + # for debug purpose + try: + logger.debug(dump.dump_all(response).decode('utf-8')) + except UnicodeDecodeError: + logger.debug(dump.dump_all(response)) + + except Exception: + import traceback + traceback.print_exc() + response = None + + resp_data = http_response_to_dsl_map(response) + if response: + response.close() + + extractor_res = http_extract(request, resp_data) + for k, v in extractor_res['internal'].items(): + if v == UNRESOLVED_VARIABLE and k in dynamic_values: + continue + else: + dynamic_values[k] = v + + if request.req_condition: + resp_data_all.update(resp_data) + for k, v in resp_data.items(): + resp_data_all[f'{k}_{current_index}'] = v + if current_index == request_count: + resp_data_all.update(dynamic_values) + match_res = http_match(request, resp_data_all, interactsh) + resp_data_all = {} + if match_res: + output = {} + output.update(extractor_res['external']) + output.update(payload) + output['extra_info'] = extractor_res['extra_info'] + results.append(output) + if request.stop_at_first_match: + return results + else: + resp_data.update(dynamic_values) + match_res = http_match(request, resp_data, interactsh) + if match_res: + output = {} + output.update(extractor_res['external']) + output.update(payload) + output['extra_info'] = extractor_res['extra_info'] + results.append(output) + if request.stop_at_first_match: + return results + except Exception: + import traceback + traceback.print_exc() + if results and any(results): + return results + else: + return False diff --git a/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py new file mode 100644 index 00000000..c4da12e9 --- /dev/null +++ b/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py @@ -0,0 +1,277 @@ +import binascii +import json +import socket +import ssl +import time +from collections import OrderedDict +from dataclasses import dataclass, field +from enum import Enum +from typing import Union + +from pocsuite3.lib.core.common import urlparse +from pocsuite3.lib.core.log import LOGGER as logger +from pocsuite3.lib.yaml.nuclei.operators import (Extractor, ExtractorType, + Matcher, MatcherType, + extract_dsl, extract_kval, extract_regex, + match_binary, + match_dsl, match_regex, + match_size, match_words) +from pocsuite3.lib.yaml.nuclei.protocols.common.generators import AttackType, payload_generator +from pocsuite3.lib.yaml.nuclei.protocols.common.replacer import ( + UNRESOLVED_VARIABLE, marker_replace) + + +class NetworkInputType(Enum): + HexType = 'hex' + TextType = 'text' + + +@dataclass +class AddressKV: + address: str = '' + host: str = '' + port: int = 0 + tls: bool = False + + +@dataclass +class Input: + """Input is the input to send on the network + """ + + # Data is the data to send as the input. + # It supports DSL Helper Functions as well as normal expressions. + data: str = '' + + # Type is the type of input specified in `data` field. + type: NetworkInputType = 'text' + + # Read is the number of bytes to read from socket. + read: int = 0 + + # Name is the optional name of the data read to provide matching on. + name: str = '' + + +@dataclass +class NetworkRequest: + """NetworkRequest contains a Network protocol request to be made from a template + """ + + # Operators for the current request go here. + matchers: list[Matcher] = field(default_factory=list) + extractors: list[Extractor] = field(default_factory=list) + matchers_condition: str = 'or' + + # Host to send network requests to. + host: list[str] = field(default_factory=list) + + addresses: list[AddressKV] = field(default_factory=list) + + # ID is the optional id of the request + id: str = '' + + # Attack is the type of payload combinations to perform. + attack: AttackType = 'batteringram' + + # Payloads contains any payloads for the current request. + payloads: dict = field(default_factory=dict) + + # Inputs contains inputs for the network socket + inputs: list[Input] = field(default_factory=list) + + # ReadSize is the size of response to read at the end, Default value for read-size is 1024. + read_size: int = 1024 + + # ReadAll determines if the data stream should be read till the end regardless of the size + read_all: bool = False + + +def network_get_match_part(part: str, resp_data: dict, interactsh=None, return_bytes: bool = False) -> str: + if part in ['', 'all', 'body']: + part = 'data' + + if part in resp_data: + result = resp_data[part] + elif part == 'interactsh_protocol': + interactsh.poll() + result = '\n'.join(interactsh.interactsh_protocol) + elif part == 'interactsh_request': + interactsh.poll() + result = '\n'.join(interactsh.interactsh_request) + elif part == 'interactsh_response': + interactsh.poll() + result = '\n'.join(interactsh.interactsh_response) + else: + result = '' + + if return_bytes and not isinstance(result, bytes): + result = result.encode() + elif not return_bytes and isinstance(result, bytes): + try: + result = result.decode() + except UnicodeDecodeError: + result = str(result) + return result + + +def network_extract(request: NetworkRequest, resp_data: dict): + extractors = request.extractors + extractors_result = {'internal': {}, 'external': {}, 'extra_info': []} + + for extractor in extractors: + item = network_get_match_part(extractor.part, resp_data) + + res = None + if extractor.type == ExtractorType.RegexExtractor: + res = extract_regex(extractor, item) + logger.debug(f'[+] {extractor} -> {res}') + elif extractor.type == ExtractorType.KValExtractor: + try: + item = json.loads(item) + except json.JSONDecodeError: + continue + res = extract_kval(extractor, item) + logger.debug(f'[+] {extractor} -> {res}') + elif extractor.type == ExtractorType.DSLExtractor: + res = extract_dsl(extractor, resp_data) + logger.debug(f'[+] {extractor} -> {res}') + + extractors_result['internal'].update(res['internal']) + extractors_result['external'].update(res['external']) + extractors_result['extra_info'] += res['extra_info'] + return extractors_result + + +def network_match(request: NetworkRequest, resp_data: dict, interactsh=None): + matchers = request.matchers + matchers_result = [] + + for i, matcher in enumerate(matchers): + matcher_res = False + item = network_get_match_part(matcher.part, resp_data, interactsh, matcher.type == MatcherType.BinaryMatcher) + + if matcher.type == MatcherType.SizeMatcher: + matcher_res = match_size(matcher, len(item)) + logger.debug(f'[+] {matcher} -> {matcher_res}') + + elif matcher.type == MatcherType.WordsMatcher: + matcher_res, _ = match_words(matcher, item, {}) + logger.debug(f'[+] {matcher} -> {matcher_res}') + + elif matcher.type == MatcherType.RegexMatcher: + matcher_res, _ = match_regex(matcher, item) + logger.debug(f'[+] {matcher} -> {matcher_res}') + + elif matcher.type == MatcherType.BinaryMatcher: + matcher_res, _ = match_binary(matcher, item) + logger.debug(f'[+] {matcher} -> {matcher_res}') + + elif matcher.type == MatcherType.DSLMatcher: + matcher_res = match_dsl(matcher, resp_data) + logger.debug(f'[+] {matcher} -> {matcher_res}') + + if not matcher_res: + if request.matchers_condition == 'and': + return False + elif request.matchers_condition == 'or': + continue + + if request.matchers_condition == 'or': + return True + + matchers_result.append(matcher_res) + + if len(matchers) - 1 == i: + return True + + return False + + +def network_request_generator(request: NetworkRequest, dynamic_values: OrderedDict): + request_count = len(request.addresses) + for payload_instance in payload_generator(request.payloads, request.attack): + current_index = 0 + dynamic_values.update(payload_instance) + for address in request.addresses: + current_index += 1 + yield address, request.inputs, payload_instance, request_count, current_index + + +def execute_network_request(request: NetworkRequest, dynamic_values, interactsh) -> Union[bool, list]: + results = [] + for h in request.host: + use_tls = False + if h.startswith('tls://'): + use_tls = True + h = h.replace('tls://', '') + address = marker_replace(h, dynamic_values) + host, port = urlparse(address).hostname, urlparse(address).port + address = AddressKV(address=address, host=host, port=port, tls=use_tls) + request.addresses.append(address) + + for (address, inputs, payload, request_count, current_index) in network_request_generator(request, dynamic_values): + try: + req_buf, resp_buf = [], [] + resp_data = {'host': address.address} + s = socket.socket() + s.connect((address.host, address.port)) + if address.tls: + ssl.wrap_socket(s) + for inp in inputs: + data = marker_replace(inp.data, dynamic_values) + if inp.type == NetworkInputType.HexType: + data = binascii.unhexlify(data) + elif not isinstance(data, bytes): + data = data.encode('utf-8') + + if inp.read > 0: + chunk = s.recv(inp.read) + resp_buf.append(chunk) + if inp.name: + resp_data[inp.name] = chunk + + req_buf.append(data) + s.send(data) + time.sleep(0.1) + + last_bytes = [] + if request.read_all: + while True: + chunk = s.recv(1024) + if not chunk: + break + last_bytes.append(chunk) + else: + chunk = s.recv(request.read_size) + last_bytes.append(chunk) + + # response to DSL Map + resp_buf += last_bytes + resp_data['request'] = b''.join(req_buf) + resp_data['data'] = b''.join(last_bytes) + resp_data['raw'] = b''.join(resp_buf) + logger.debug(resp_data) + + extractor_res = network_extract(request, resp_data) + + for k, v in extractor_res['internal'].items(): + if v == UNRESOLVED_VARIABLE and k in dynamic_values: + continue + else: + dynamic_values[k] = v + + resp_data.update(dynamic_values) + match_res = network_match(request, resp_data, interactsh) + if match_res: + output = {} + output.update(extractor_res['external']) + output.update(payload) + output['extra_info'] = extractor_res['extra_info'] + results.append(output) + return results + except Exception: + import traceback + traceback.print_exc() + + return False diff --git a/pocsuite3/lib/yaml/nuclei/templates/__init__.py b/pocsuite3/lib/yaml/nuclei/templates/__init__.py index bb09376e..c82407e8 100644 --- a/pocsuite3/lib/yaml/nuclei/templates/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/templates/__init__.py @@ -3,6 +3,7 @@ from pocsuite3.lib.yaml.nuclei.model import Info from pocsuite3.lib.yaml.nuclei.protocols.http import HttpRequest +from pocsuite3.lib.yaml.nuclei.protocols.network import NetworkRequest class ProtocolType(Enum): @@ -18,11 +19,13 @@ class ProtocolType(Enum): WHOISProtocol = "whois" -# Template is a YAML input file which defines all the requests and other metadata for a template. @dataclass class Template: + """Template is a YAML input file which defines all the requests and other metadata for a template. + """ id: str = '' info: Info = field(default_factory=Info) requests: list[HttpRequest] = field(default_factory=list) + network: list[NetworkRequest] = field(default_factory=list) stop_at_first_match: bool = True variables: dict = field(default_factory=dict) From 659901d5f53f20b8a71b6db4ffe1835f48a5d253 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:37:51 -0700 Subject: [PATCH 5/9] fix: flake8 error --- .../yaml/nuclei/protocols/common/expressions/__init__.py | 2 +- pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py | 2 +- setup.cfg | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py index 1a9f1b02..e45ef3b0 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/common/expressions/__init__.py @@ -337,7 +337,7 @@ def print_debug(*args) -> None: raise NotImplementedError -def rand_base(length: int, optional_charset: str = string.ascii_letters+string.digits) -> str: +def rand_base(length: int, optional_charset: str = string.ascii_letters + string.digits) -> str: """ Generates a random sequence of given length string from an optional charset (defaults to letters and numbers) diff --git a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py index 8a222815..2c2ed866 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py @@ -42,7 +42,7 @@ class HttpRequest: # Operators for the current request go here. matchers: list[Matcher] = field(default_factory=list) extractors: list[Extractor] = field(default_factory=list) - matchers_condition: str = 'or' + matchers_condition: str = 'or' # Path contains the path/s for the HTTP requests. It supports variables as placeholders. path: list[str] = field(default_factory=list) diff --git a/setup.cfg b/setup.cfg index 60847fcc..ae8bd6c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,4 +2,11 @@ description-file = README.md [bdist_wheel] -universal = 1 \ No newline at end of file +universal = 1 + +[flake8] +per-file-ignores = + # imported but unused + pocsuite3/api/__init__.py: F401 + # may be undefined, or defined from star imports + tests/test_nuclei_helper_functions.py: F405 From 08f2d06058549e1b28977bb38d9e4a6a6452b5ea Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:41:25 -0700 Subject: [PATCH 6/9] fix: flake8 error --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ae8bd6c8..930b98c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,4 +9,4 @@ per-file-ignores = # imported but unused pocsuite3/api/__init__.py: F401 # may be undefined, or defined from star imports - tests/test_nuclei_helper_functions.py: F405 + tests/test_nuclei_helper_functions.py: F405, F403 From 448b731bf8bdb600b19f9ee56e5526eeac9fdfe4 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 2 Nov 2022 19:20:58 -0700 Subject: [PATCH 7/9] fix: compatible with Python 3.7 --- pocsuite3/lib/core/interpreter.py | 1 + .../lib/yaml/nuclei/operators/extrators/__init__.py | 11 ++++++----- .../lib/yaml/nuclei/operators/matchers/__init__.py | 13 +++++++------ .../lib/yaml/nuclei/protocols/http/__init__.py | 10 +++++----- .../lib/yaml/nuclei/protocols/network/__init__.py | 12 ++++++------ pocsuite3/lib/yaml/nuclei/templates/__init__.py | 5 +++-- 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/pocsuite3/lib/core/interpreter.py b/pocsuite3/lib/core/interpreter.py index c30d7eee..206fe31a 100644 --- a/pocsuite3/lib/core/interpreter.py +++ b/pocsuite3/lib/core/interpreter.py @@ -1,3 +1,4 @@ +# pylint: disable=E0202 import os import re import chardet diff --git a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py index a982e915..40fe677c 100644 --- a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py @@ -2,6 +2,7 @@ import re from dataclasses import dataclass, field from enum import Enum +from typing import List import jq from lxml import etree @@ -29,25 +30,25 @@ class Extractor: type: ExtractorType = 'regex' # Regex contains the regular expression patterns to extract from a part. - regex: list[str] = field(default_factory=list) + regex: List[str] = field(default_factory=list) # Group specifies a numbered group to extract from the regex. group: int = 0 # kval contains the key-value pairs present in the HTTP response header. - kval: list[str] = field(default_factory=list) + kval: List[str] = field(default_factory=list) # JSON allows using jq-style syntax to extract items from json response - json: list[str] = field(default_factory=list) + json: List[str] = field(default_factory=list) # XPath allows using xpath expressions to extract items from html response - xpath: list[str] = field(default_factory=list) + xpath: List[str] = field(default_factory=list) # Attribute is an optional attribute to extract from response XPath. attribute: str = '' # Extracts using DSL expressions - dsl: list[str] = field(default_factory=list) + dsl: List[str] = field(default_factory=list) # Part is the part of the request response to extract data from. part: str = '' diff --git a/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py index c0368e83..6e3ea50d 100644 --- a/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/operators/matchers/__init__.py @@ -2,6 +2,7 @@ import re from dataclasses import dataclass, field from enum import Enum +from typing import List from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import evaluate, Marker @@ -37,22 +38,22 @@ class Matcher: name: str = '' # Status are the acceptable status codes for the response. - status: list[int] = field(default_factory=list) + status: List[int] = field(default_factory=list) # Size is the acceptable size for the response - size: list[int] = field(default_factory=list) + size: List[int] = field(default_factory=list) # Words contains word patterns required to be present in the response part. - words: list[str] = field(default_factory=list) + words: List[str] = field(default_factory=list) # Regex contains Regular Expression patterns required to be present in the response part. - regex: list[str] = field(default_factory=list) + regex: List[str] = field(default_factory=list) # Binary are the binary patterns required to be present in the response part. - binary: list[str] = field(default_factory=list) + binary: List[str] = field(default_factory=list) # DSL are the dsl expressions that will be evaluated as part of nuclei matching rules. - dsl: list[str] = field(default_factory=list) + dsl: List[str] = field(default_factory=list) # Encoding specifies the encoding for the words field if any. encoding: str = '' diff --git a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py index 2c2ed866..d29b4d0c 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/http/__init__.py @@ -1,7 +1,7 @@ from collections import OrderedDict from dataclasses import dataclass, field from enum import Enum -from typing import Union +from typing import Union, List from requests_toolbelt.utils import dump @@ -40,15 +40,15 @@ class HttpRequest: """ # Operators for the current request go here. - matchers: list[Matcher] = field(default_factory=list) - extractors: list[Extractor] = field(default_factory=list) + matchers: List[Matcher] = field(default_factory=list) + extractors: List[Extractor] = field(default_factory=list) matchers_condition: str = 'or' # Path contains the path/s for the HTTP requests. It supports variables as placeholders. - path: list[str] = field(default_factory=list) + path: List[str] = field(default_factory=list) # Raw contains HTTP Requests in Raw format. - raw: list[str] = field(default_factory=list) + raw: List[str] = field(default_factory=list) # ID is the optional id of the request id: str = '' diff --git a/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py b/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py index c4da12e9..fd1f1bd3 100644 --- a/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/protocols/network/__init__.py @@ -6,7 +6,7 @@ from collections import OrderedDict from dataclasses import dataclass, field from enum import Enum -from typing import Union +from typing import Union, List from pocsuite3.lib.core.common import urlparse from pocsuite3.lib.core.log import LOGGER as logger @@ -59,14 +59,14 @@ class NetworkRequest: """ # Operators for the current request go here. - matchers: list[Matcher] = field(default_factory=list) - extractors: list[Extractor] = field(default_factory=list) + matchers: List[Matcher] = field(default_factory=list) + extractors: List[Extractor] = field(default_factory=list) matchers_condition: str = 'or' # Host to send network requests to. - host: list[str] = field(default_factory=list) + host: List[str] = field(default_factory=list) - addresses: list[AddressKV] = field(default_factory=list) + addresses: List[AddressKV] = field(default_factory=list) # ID is the optional id of the request id: str = '' @@ -78,7 +78,7 @@ class NetworkRequest: payloads: dict = field(default_factory=dict) # Inputs contains inputs for the network socket - inputs: list[Input] = field(default_factory=list) + inputs: List[Input] = field(default_factory=list) # ReadSize is the size of response to read at the end, Default value for read-size is 1024. read_size: int = 1024 diff --git a/pocsuite3/lib/yaml/nuclei/templates/__init__.py b/pocsuite3/lib/yaml/nuclei/templates/__init__.py index c82407e8..fb31b704 100644 --- a/pocsuite3/lib/yaml/nuclei/templates/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/templates/__init__.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field from enum import Enum +from typing import List from pocsuite3.lib.yaml.nuclei.model import Info from pocsuite3.lib.yaml.nuclei.protocols.http import HttpRequest @@ -25,7 +26,7 @@ class Template: """ id: str = '' info: Info = field(default_factory=Info) - requests: list[HttpRequest] = field(default_factory=list) - network: list[NetworkRequest] = field(default_factory=list) + requests: List[HttpRequest] = field(default_factory=list) + network: List[NetworkRequest] = field(default_factory=list) stop_at_first_match: bool = True variables: dict = field(default_factory=dict) From c85930039a42d3a7f43a7b167bd50fff7b3b0c36 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 2 Nov 2022 19:59:54 -0700 Subject: [PATCH 8/9] fix: compatible with windows --- pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py | 8 +++++++- requirements.txt | 1 - setup.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py index 40fe677c..ced8c189 100644 --- a/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py +++ b/pocsuite3/lib/yaml/nuclei/operators/extrators/__init__.py @@ -4,10 +4,10 @@ from enum import Enum from typing import List -import jq from lxml import etree from requests.structures import CaseInsensitiveDict +from pocsuite3.lib.core.log import LOGGER as logger from pocsuite3.lib.yaml.nuclei.protocols.common.expressions import evaluate, UNRESOLVED_VARIABLE, Marker @@ -175,6 +175,12 @@ def extract_json(e: Extractor, corpus: str) -> dict: except json.JSONDecodeError: return results + try: + import jq + except ImportError: + logger.error('Python bindings for jq not installed, it only supports linux and macos, https://pypi.org/project/jq/') + return results + for j in e.json: try: res = jq.compile(j).input(corpus).all() diff --git a/requirements.txt b/requirements.txt index 2d2f310f..679d11c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,5 @@ Faker >= 0.7.7 pycryptodomex >= 3.9.0 dacite >= 1.6.0 PyYAML >= 6.0 -jq >= 1.2.1 lxml >= 4.6.0 mmh3 >= 3.0.0 diff --git a/setup.py b/setup.py index 86ccab13..59fc4d8d 100644 --- a/setup.py +++ b/setup.py @@ -56,13 +56,13 @@ def find_packages(where='.'): "pycryptodomex", "dacite", "PyYAML", - "jq", "lxml", "mmh3" ], extras_require={ 'complete': [ - 'pyOpenSSL' + 'pyOpenSSL', + 'jq' ], } ) From 43d3548420b9a03fc565f8684b0ab51729af35f2 Mon Sep 17 00:00:00 2001 From: 13ph03nix <17541483+13ph03nix@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:11:56 -0700 Subject: [PATCH 9/9] chore: bump version to 2.0.0 --- CHANGELOG.md | 6 ++++++ manpages/poc-console.1 | 2 +- manpages/pocsuite.1 | 2 +- pocsuite3/__init__.py | 2 +- setup.py | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1469c0a0..f816e4f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# version 2.0.0 +---------------- +* yaml poc support, compatible with nuclei +* fix httpserver module hangs on macos platform +* auto correction of url protocol based on status code + # version 1.9.11 ---------------- * support customize poc protocol and default port #321 diff --git a/manpages/poc-console.1 b/manpages/poc-console.1 index 7f384dae..76599364 100644 --- a/manpages/poc-console.1 +++ b/manpages/poc-console.1 @@ -31,7 +31,7 @@ is maintained at: .I https://pocsuite.org .PP .SH VERSION -This manual page documents pocsuite3 version 1.9.11 +This manual page documents pocsuite3 version 2.0.0 .SH AUTHOR .br (c) 2014-present by Knownsec 404 Team diff --git a/manpages/pocsuite.1 b/manpages/pocsuite.1 index bcb5140c..c6bd60cb 100644 --- a/manpages/pocsuite.1 +++ b/manpages/pocsuite.1 @@ -286,7 +286,7 @@ is maintained at: .I https://pocsuite.org .PP .SH VERSION -This manual page documents pocsuite3 version 1.9.11 +This manual page documents pocsuite3 version 2.0.0 .SH AUTHOR .br (c) 2014-present by Knownsec 404 Team diff --git a/pocsuite3/__init__.py b/pocsuite3/__init__.py index a4d4de65..71715679 100644 --- a/pocsuite3/__init__.py +++ b/pocsuite3/__init__.py @@ -1,5 +1,5 @@ __title__ = 'pocsuite3' -__version__ = '1.9.11' +__version__ = '2.0.0' __author__ = 'Knownsec 404 Team' __author_email__ = '404-team@knownsec.com' __license__ = 'GPLv2' diff --git a/setup.py b/setup.py index 59fc4d8d..8184f084 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def find_packages(where='.'): setup( name='pocsuite3', - version='1.9.11', + version='2.0.0', url='https://pocsuite.org', description='Open-sourced remote vulnerability testing framework.', long_description=long_description,