diff --git a/honeytraps/waf_elk/logstash/pipeline/logstash.conf b/honeytraps/waf_elk/logstash/pipeline/logstash.conf index eb09f36..a7c0c1e 100644 --- a/honeytraps/waf_elk/logstash/pipeline/logstash.conf +++ b/honeytraps/waf_elk/logstash/pipeline/logstash.conf @@ -6,13 +6,19 @@ input { } } -#filter { -# if [type] == "mod_security" { -# grok { -# match => { "[audit_data][messages]" => 'ARGS:%{DATA}: %{DATA:m_data}"]'} -# } -# } -#} +filter { + ruby { + code => " + event.to_hash.clone.each_value{|v| + if v.is_a? Hash + v.each_pair{|k,v| + event[k] = v + } + end + } + " + } +} output { elasticsearch { diff --git a/honeytraps/waf_elk/misp-push/kibana-client.py b/honeytraps/waf_elk/misp-push/kibana-client.py index 9ee3369..a37c93b 100755 --- a/honeytraps/waf_elk/misp-push/kibana-client.py +++ b/honeytraps/waf_elk/misp-push/kibana-client.py @@ -93,15 +93,13 @@ def generate_misp_tags(): - +log.info("Waiting for Elasticsearch to be Up...") while (True): try: res = requests.get('http://elasticsearch:9200') break except Exception as e: - log.info("Waiting for Elasticsearch to be Up...") time.sleep(1) - log.info("Elasticsearch is up") es = Elasticsearch([{'host': 'elasticsearch', 'port': 9200}]) diff --git a/honeytraps/waf_modsec/Dockerfile b/honeytraps/waf_modsec/Dockerfile index cb05482..6529fa1 100644 --- a/honeytraps/waf_modsec/Dockerfile +++ b/honeytraps/waf_modsec/Dockerfile @@ -1,15 +1,16 @@ FROM owasp/modsecurity-crs -RUN apt install -y wget nano curl +RUN apt install -y wget nano curl python3-watchdog RUN wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.4.2-amd64.deb RUN dpkg -i filebeat-7.4.2-amd64.deb COPY filebeat.yml /etc/filebeat/filebeat.yml -COPY filebeat.template.json /etc/filebeat/filebeat.template.json COPY modsec_entry.sh / COPY robots.txt /home/ COPY index.html /usr/local/apache2/htdocs/ COPY login.html /usr/local/apache2/htdocs/ COPY httpd-extension.conf /app/httpd-extension.conf COPY modsecurity-extension.conf /app/modsecurity-extension.conf +COPY preprocess-modsec-log.py /app/preprocess-modsec-log.py +RUN touch /var/log/modsec_audit_processed.log RUN cat /app/httpd-extension.conf >> /usr/local/apache2/conf/httpd.conf RUN cat /app/modsecurity-extension.conf >> /etc/modsecurity.d/modsecurity.conf RUN chmod +x /modsec_entry.sh diff --git a/honeytraps/waf_modsec/docker-entrypoint.sh b/honeytraps/waf_modsec/docker-entrypoint.sh deleted file mode 100644 index a4baf83..0000000 --- a/honeytraps/waf_modsec/docker-entrypoint.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -curl -H 'Content-Type: application/json' -XPUT 'http://elk:9200/_template/filebeat' -d@/etc/filebeat/filebeat.template.json \ No newline at end of file diff --git a/honeytraps/waf_modsec/example-formatted-log-message.txt b/honeytraps/waf_modsec/example-formatted-log-message.txt deleted file mode 100644 index 40c3d2a..0000000 --- a/honeytraps/waf_modsec/example-formatted-log-message.txt +++ /dev/null @@ -1,202 +0,0 @@ -Warning. Found 9 byte(s) in ARGS:q outside range: 38,44-46,48-58,61,65-90,95,97-122. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf"] - [line "1391"] - [id "920273"] - [msg "Invalid character in request (outside of very strict set)"] - [data "ARGS:q=\x22>"] - [severity "CRITICAL"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-protocol"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/PROTOCOL_VIOLATION/EVASION"] - [tag "paranoia-level/4"], -Warning. detected XSS using libinjection. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-941-APPLICATION-ATTACK-XSS.conf"] - [line "59"] - [id "941100"] - [msg "XSS Attack Detected via libinjection"] - [data "Matched Data: XSS data found within ARGS:q: \x22>"] - [severity "CRITICAL"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-xss"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/XSS"] - [tag "WASCTC/WASC-8"] - [tag "WASCTC/WASC-22"] - [tag "OWASP_TOP_10/A3"] - [tag "OWASP_AppSensor/IE1"] - [tag "CAPEC-242"], -Warning. Pattern match "(?i)]*>[\\s\\S]*?" at ARGS:q. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-941-APPLICATION-ATTACK-XSS.conf"] - [line "90"] - [id "941110"] - [msg "XSS Filter - Category 1: Script Tag Vector"] - [data "Matched Data: "] - [severity "CRITICAL"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-xss"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/XSS"] - [tag "WASCTC/WASC-8"] - [tag "WASCTC/WASC-22"] - [tag "OWASP_TOP_10/A3"] - [tag "OWASP_AppSensor/IE1"] - [tag "CAPEC-242"], -Warning. Pattern match "(?i:(?:<\\w[\\s\\S]*[\\s\\/]|['\"](?:[\\s\\S]*[\\s\\/])?)(?:on(?:d(?:e(?:vice(?:(?:orienta|mo)tion|proximity|found|light)|livery(?:success|error)|activate)|r(?:ag(?:e(?:n(?:ter|d)|xit)|(?:gestur|leav)e|start|drop|over)|op)|i(?:s(?:c(?:hargingtimechange ..." at ARGS:q. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-941-APPLICATION-ATTACK-XSS.conf"] - [line "218"] - [id "941160"] - [msg "NoScript XSS InjectionChecker: HTML Injection"] - [data "Matched Data: "] - [severity "CRITICAL"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-xss"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/XSS"] - [tag "WASCTC/WASC-8"] - [tag "WASCTC/WASC-22"] - [tag "OWASP_TOP_10/A3"] - [tag "OWASP_AppSensor/IE1"] - [tag "CAPEC-242"], -Warning. Pattern match "<(?:a|abbr|acronym|address|applet|area|audioscope|b|base|basefront|bdo|bgsound|big|blackface|blink|blockquote|body|bq|br|button|caption|center|cite|code|col|colgroup|comment|dd|del|dfn|dir|div|dl|dt|em|embed|fieldset|fn|font|form|frame|frameset|h1|head ..." at ARGS:q. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-941-APPLICATION-ATTACK-XSS.conf"] - [line "879"] - [id "941320"] - [msg "Possible XSS Attack Detected - HTML Tag Handler"] - [data "Matched Data: "] - [severity "CRITICAL"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-xss"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/XSS"] - [tag "WASCTC/WASC-8"] - [tag "WASCTC/WASC-22"] - [tag "OWASP_TOP_10/A2"] - [tag "OWASP_AppSensor/IE1"] - [tag "PCI/6.5.1"] - [tag "paranoia-level/2"], -Warning. Pattern match "(?:^\\s*[\"'`;]+|[\"'`]+\\s*$)" at ARGS:q. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-942-APPLICATION-ATTACK-SQLI.conf"] - [line "550"] - [id "942110"] - [msg "SQL Injection Attack: Common Injection Testing Detected"] - [data "Matched Data: \x22 found within ARGS:q: \x22>"] - [severity "WARNING"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-sqli"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/SQL_INJECTION"] - [tag "WASCTC/WASC-19"] - [tag "OWASP_TOP_10/A1"] - [tag "OWASP_AppSensor/CIE1"] - [tag "PCI/6.5.2"] - [tag "paranoia-level/2"], -Warning. Pattern match "(?i:[\\s'\"`()]*?([\\d\\w]++)[\\s'\"`()]*?(?:<(?:=(?:[\\s'\"`()]*?(?!\\1)[\\d\\w]+|>[\\s'\"`()]*?(?:\\1))|>?[\\s'\"`()]*?(?!\\1)[\\d\\w]+)|(?:not\\s+(?:regexp|like)|is\\s+not|>=?|!=|\\^)[\\s'\"`()]*?(?!\\1)[\\d\\w]+|(?:(?:sounds\\s+)?like|r(?:egexp|lik ..." at ARGS:q. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-942-APPLICATION-ATTACK-SQLI.conf"] - [line "628"] - [id "942130"] - [msg "SQL Injection Attack: SQL Tautology Detected."] - [data "Matched Data: script>alert found within ARGS:q: \x22>"] - [severity "CRITICAL"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-sqli"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/SQL_INJECTION"] - [tag "WASCTC/WASC-19"] - [tag "OWASP_TOP_10/A1"] - [tag "OWASP_AppSensor/CIE1"] - [tag "PCI/6.5.2"] - [tag "paranoia-level/2"], -Warning. Pattern match "(?i:[\"'`]\\s*?(?:(?:n(?:and|ot)|(?:x?x)?or|between|\\|\\||and|div|&&)\\s+[\\s\\w]+=\\s*?\\w+\\s*?having\\s+|like(?:\\s+[\\s\\w]+=\\s*?\\w+\\s*?having\\s+|\\W*?[\"'`\\d])|[^?\\w\\s=.,;)(]++\\s*?[(@\"'`]*?\\s*?\\w+\\W+\\w|\\*\\s*?\\w+\\W+[\"'`])|(?:unio ..." at ARGS:q. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-942-APPLICATION-ATTACK-SQLI.conf"] - [line "803"] - [id "942260"] - [msg "Detects basic SQL authentication bypass attempts 2/3"] - [data "Matched Data: \x22>"] - [severity "CRITICAL"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-sqli"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/SQL_INJECTION"] - [tag "WASCTC/WASC-19"] - [tag "OWASP_TOP_10/A1"] - [tag "OWASP_AppSensor/CIE1"] - [tag "PCI/6.5.2"] - [tag "paranoia-level/2"], -Warning. Pattern match "((?:[~!@#\\$%\\^&\\*\\(\\)\\-\\+=\\{\\}\\[\\]\\|:;\"'\xc2\xb4\xe2\x80\x99\xe2\x80\x98`<>][^~!@#\\$%\\^&\\*\\(\\)\\-\\+=\\{\\}\\[\\]\\|:;\"'\xc2\xb4\xe2\x80\x99\xe2\x80\x98`<>]*?){6})" at ARGS:q. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-942-APPLICATION-ATTACK-SQLI.conf"] - [line "1526"] - [id "942431"] - [msg "Restricted SQL Character Anomaly Detection (args): # of special characters exceeded (6)"] - [data "Matched Data: \x22>"] - [severity "WARNING"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-sqli"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/SQL_INJECTION"] - [tag "WASCTC/WASC-19"] - [tag "OWASP_TOP_10/A1"] - [tag "OWASP_AppSensor/CIE1"] - [tag "PCI/6.5.2"] - [tag "paranoia-level/3"], -Warning. Pattern match "((?:[~!@#\\$%\\^&\\*\\(\\)\\-\\+=\\{\\}\\[\\]\\|:;\"'\xc2\xb4\xe2\x80\x99\xe2\x80\x98`<>][^~!@#\\$%\\^&\\*\\(\\)\\-\\+=\\{\\}\\[\\]\\|:;\"'\xc2\xb4\xe2\x80\x99\xe2\x80\x98`<>]*?){2})" at ARGS:q. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-942-APPLICATION-ATTACK-SQLI.conf"] - [line "1717"] - [id "942432"] - [msg "Restricted SQL Character Anomaly Detection (args): # of special characters exceeded (2)"] - [data "Matched Data: \x22> found within ARGS:q: \x22>"] - [severity "WARNING"] - [ver "OWASP_CRS/3.2.0"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-sqli"] - [tag "OWASP_CRS"] - [tag "OWASP_CRS/WEB_ATTACK/SQL_INJECTION"] - [tag "WASCTC/WASC-19"] - [tag "OWASP_TOP_10/A1"] - [tag "OWASP_AppSensor/CIE1"] - [tag "PCI/6.5.2"] - [tag "paranoia-level/4"], -Access denied with code 403 (phase 2). Operator GE matched 5 at TX:anomaly_score. - [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-949-BLOCKING-EVALUATION.conf"] - [line "91"] - [id "949110"] - [msg "Inbound Anomaly Score Exceeded (Total Score: 44)"] [severity "CRITICAL"] - [tag "application-multi"] - [tag "language-multi"] - [tag "platform-multi"] - [tag "attack-generic"], -Warning. Operator GE matched 5 at TX:inbound_anomaly_score. - [file "/etc/modsecurity.d/owasp-crs/rules/RESPONSE-980-CORRELATION.conf"] - [line "86"] - [id "980130"] - [msg "Inbound Anomaly Score Exceeded (Total Inbound Score: 44 - SQLI=19,XSS=20,RFI=0,LFI=0,RCE=0,PHPI=0,HTTP=0,SESS=0): individual paranoia level scores: 15, 18, 3, 8"] - [tag "event-correlation"] \ No newline at end of file diff --git a/honeytraps/waf_modsec/filebeat.template.json b/honeytraps/waf_modsec/filebeat.template.json deleted file mode 100644 index 4b589c4..0000000 --- a/honeytraps/waf_modsec/filebeat.template.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "mappings": { - "_default_": { - "_all": { - "norms": false - }, - "_meta": { - "version": "5.1.2" - }, - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "@timestamp": { - "type": "date" - }, - "beat": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "input_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "norms": false, - "type": "text" - }, - "meta": { - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "instance_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine_type": { - "ignore_above": 1024, - "type": "keyword" - }, - "project_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "offset": { - "type": "long" - }, - "source": { - "ignore_above": 1024, - "type": "keyword" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - }, - "order": 0, - "settings": { - "index.refresh_interval": "5s" - }, - "template": "filebeat-*" -} \ No newline at end of file diff --git a/honeytraps/waf_modsec/filebeat.yml b/honeytraps/waf_modsec/filebeat.yml index 2598860..c008b26 100644 --- a/honeytraps/waf_modsec/filebeat.yml +++ b/honeytraps/waf_modsec/filebeat.yml @@ -9,7 +9,7 @@ filebeat: inputs: - paths: - - /var/log/modsec_audit.log + - /var/log/modsec_audit_processed.log type: log json.keys_under_root: true json.add_error_key: true diff --git a/honeytraps/waf_modsec/modsec_entry.sh b/honeytraps/waf_modsec/modsec_entry.sh index 856e0d5..1922e81 100755 --- a/honeytraps/waf_modsec/modsec_entry.sh +++ b/honeytraps/waf_modsec/modsec_entry.sh @@ -1,3 +1,4 @@ # ~/bin/sh -apachectl +apachectl +python3 /app/preprocess-modsec-log.py & filebeat -e -c filebeat.yml -d "publish" diff --git a/honeytraps/waf_modsec/preprocess-modsec-log.py b/honeytraps/waf_modsec/preprocess-modsec-log.py new file mode 100755 index 0000000..18a97cf --- /dev/null +++ b/honeytraps/waf_modsec/preprocess-modsec-log.py @@ -0,0 +1,155 @@ +#!/usr/bin/python +## --------------------------------------------------------------------- +## +## A hack to preprocess Modsecurity JSON logs into a proper JSON format +## Hopefully this can be removed as soon as ModSecurity updates to not to +## have a shitty format for the actual audit_data +## +## ---------------------------------------------------------------------- + +### +### This script watches var/log/modsec_audit.log and processes logs as they +### come in, +### + +import json +import re +import sys +import time +import logging +from typing import Tuple, List, Dict, Any +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +class LineParser: + '''parses audit_log.messages and audit_log.error_messages''' + __tagRegex__ = ' \[.*? .*?\]|^\[.*? .*?\]' + + def __parseTag__(self, tag) -> Tuple[str, str]: + tag = tag.strip() # Strip leading/trailing whitespace + tag = tag[1:-1] # remove brackets + tag = tag.split(" ", 1) # split by first space to get key and value + key = tag[0] + value = tag[1][1:-1] if tag[1].startswith("\"") else tag[1] # strip quotes from value if its a string + return (key, value) + + def __parseMessages__(self, messages) -> Dict[str, Any]: + events = dict() + index = 0 + for message in messages: + event = dict() + # Get all tags in square brackets + tags = re.findall(self.__tagRegex__, message) + for chunk in tags: + [key, value] = self.__parseTag__(chunk) + if (key == "tag"): + if (event.get('tags', None) == None): + event['tags'] = [] + event['tags'].append(value) + pass + else: + if (key != "data" and key != "msg"): + event[key] = value + pass + + # Get type and pattern + leftovers = (re.sub(self.__tagRegex__, '', message)) + leftovers = leftovers.split(".") + + event['type'] = leftovers[0].strip() + event['pattern'] = leftovers[1].strip() + events['message-' + str(index)] = event # Logstash can't process nested object arrays so we use dicts + index+= 1 + return events + + def parse(self, logLine) -> Dict[str, Any]: + parsed = json.loads(logLine) + #parsed['audit_data']['messages'] = self.__parseMessages__(parsed['audit_data']['messages']) + #parsed['audit_data']['error_messages'] = self.__parseMessages__(parsed['audit_data']['error_messages']) + # Same reason, Logstash can't process deeply nested JSON, putting them into root level instead + parsed['messages'] = self.__parseMessages__(parsed['audit_data']['messages']) + parsed['error_messages'] = self.__parseMessages__(parsed['audit_data']['error_messages']) + del parsed['audit_data']['messages'] + del parsed['audit_data']['error_messages'] + return parsed + +class fileHandler(FileSystemEventHandler): + parser = LineParser() + lineNum = None + processdLineNum = None + originalPath = "/var/log/modsec_audit.log" + processedPath = "/var/log/modsec_audit_processed.log" + + def on_modified(self, event): + if "modsec_audit.log" in event.src_path: + log.info("Logfile changed") + self.processLog() + else: + #log.info("log did not change, path: " + event.src_path) + pass + + def processLog(self) -> None: + # Getting lines and total number of lines + processedLines = self.getLines(self.processedPath) + lines = self.getLines(self.originalPath) + procIndex = 0 if processedLines is None else len(processedLines) + log.debug("Processed file lines", procIndex) + origIndex = 0 if lines is None else len(lines) + log.debug("Original file lines", origIndex) + + # Getting current head to process from + [index, writeMode] = self.getHead(origIndex, procIndex) + log.debug("Head", index) + log.debug("Write Mode", writeMode) + + # Log line prasing/modification + newLines = [] + for i in range(index, len(lines)): + newLines.append(json.dumps(self.parser.parse(lines[i]))) + + # Write to the new file + try: + file = open(self.processedPath, writeMode) + for line in newLines: + file.write(line + "\n") + #log.info(newLines) + file.close() + except Exception as e: + log.error("Error writing to file", e) + + def getLines(self, path) -> List[str]: + try: + file = open(path, "r") + lines = file.readlines() + file.close() + return lines + except FileNotFoundError as e: + return None + + def getHead(self, origFileIndex, newFileIndex) -> Tuple[int, str]: + # returns (index, write mode) + if (origFileIndex <= newFileIndex): return (0, "w") + if (newFileIndex == 0): return (0, "a") + if (newFileIndex > 0 and origFileIndex > newFileIndex): return (newFileIndex, "a") + raise NotImplementedError("Case not implemented") + +if __name__ == "__main__": + PATH="/var/log/" + logging.basicConfig(level=logging.INFO, format='[%(asctime)s][%(levelname)s][%(name)s] %(message)s') + log = logging.getLogger("Python-Script") + handler = logging.FileHandler('/var/log/preprocess-script.log') + formatter = logging.Formatter('[%(asctime)s][%(levelname)s][%(name)s] %(message)s') + handler.setFormatter(formatter) + handler.setLevel(logging.INFO) + log.addHandler(handler) + log.info("Python script is starting up...") + event_handler = fileHandler() + observer = Observer() + observer.schedule(event_handler, PATH, recursive=True) + observer.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + observer.join()