Add Modsecurity logfile preprocessing to be digestible by Logstash

OWASP · Jan 11, 2020 · d32d0b0 · d32d0b0
1 parent b7eb468
commit d32d0b0
Show file tree

Hide file tree

Showing 9 changed files with 175 additions and 322 deletions.
diff --git a/honeytraps/waf_elk/logstash/pipeline/logstash.conf b/honeytraps/waf_elk/logstash/pipeline/logstash.conf
@@ -6,13 +6,19 @@ input {
  }
 }
 
-#filter {
-# if [type] == "mod_security" {
-# grok {
-# match => { "[audit_data][messages]" => 'ARGS:%{DATA}: %{DATA:m_data}"]'}
-# }
-# }
-#}
+filter {
+ ruby {
+ code => "
+ event.to_hash.clone.each_value{|v|
+ if v.is_a? Hash
+ v.each_pair{|k,v|
+ event[k] = v
+ }
+ end
+ }
+ "
+ }
+}
 
 output {
  elasticsearch {

diff --git a/honeytraps/waf_elk/misp-push/kibana-client.py b/honeytraps/waf_elk/misp-push/kibana-client.py
@@ -93,15 +93,13 @@ def generate_misp_tags():
 
 
 
-
+log.info("Waiting for Elasticsearch to be Up...")
 while (True):
  try:
  res = requests.get('http://elasticsearch:9200')
  break
  except Exception as e:
- log.info("Waiting for Elasticsearch to be Up...")
  time.sleep(1)
-
 log.info("Elasticsearch is up")
 es = Elasticsearch([{'host': 'elasticsearch', 'port': 9200}])
 

diff --git a/honeytraps/waf_modsec/Dockerfile b/honeytraps/waf_modsec/Dockerfile
@@ -1,15 +1,16 @@
 FROM owasp/modsecurity-crs
-RUN apt install -y wget nano curl
+RUN apt install -y wget nano curl python3-watchdog
 RUN wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.4.2-amd64.deb
 RUN dpkg -i filebeat-7.4.2-amd64.deb
 COPY filebeat.yml /etc/filebeat/filebeat.yml
-COPY filebeat.template.json /etc/filebeat/filebeat.template.json
 COPY modsec_entry.sh /
 COPY robots.txt /home/
 COPY index.html /usr/local/apache2/htdocs/
 COPY login.html /usr/local/apache2/htdocs/
 COPY httpd-extension.conf /app/httpd-extension.conf
 COPY modsecurity-extension.conf /app/modsecurity-extension.conf
+COPY preprocess-modsec-log.py /app/preprocess-modsec-log.py
+RUN touch /var/log/modsec_audit_processed.log
 RUN cat /app/httpd-extension.conf >> /usr/local/apache2/conf/httpd.conf
 RUN cat /app/modsecurity-extension.conf >> /etc/modsecurity.d/modsecurity.conf
 RUN chmod +x /modsec_entry.sh

diff --git a/honeytraps/waf_modsec/docker-entrypoint.sh b/honeytraps/waf_modsec/docker-entrypoint.sh
diff --git a/honeytraps/waf_modsec/example-formatted-log-message.txt b/honeytraps/waf_modsec/example-formatted-log-message.txt
diff --git a/honeytraps/waf_modsec/filebeat.template.json b/honeytraps/waf_modsec/filebeat.template.json
diff --git a/honeytraps/waf_modsec/filebeat.yml b/honeytraps/waf_modsec/filebeat.yml
@@ -9,7 +9,7 @@ filebeat:
  inputs:
  -
  paths:
- - /var/log/modsec_audit.log
+ - /var/log/modsec_audit_processed.log
  type: log
  json.keys_under_root: true
  json.add_error_key: true
diff --git a/honeytraps/waf_modsec/modsec_entry.sh b/honeytraps/waf_modsec/modsec_entry.sh
@@ -1,3 +1,4 @@
 # ~/bin/sh
-apachectl 
+apachectl
+python3 /app/preprocess-modsec-log.py &
 filebeat -e -c filebeat.yml -d "publish"