Feature/218 elastalert

README.md

@@ -43,3 +43,30 @@ these/pems/are/ok/*.pem
 ```
 
 [See example in this repo](./.secignore).
+
+
+-----------------------------------------------------------
+### Notifications
+Work in progress.
+
+#### Local testing of notifications
+```
+# docker-compose.yml -> comment out elastalert container
+. .env && docker-compose up --build
+
+# new CLI tab
+# Elastalert expects the elasticsearch index it is monitoring to exist otherwise it will error. The server creates the index in elasticsearch the first time it writes a log.
+. .env && make test-run-offenses
+
+# docker-compose.yml -> uncomment out elastalert container
+. .env && docker-compose up
+
+# new CLI tab
+. .env && make test-run-offenses
+
+# attach to elastalert container
+doc run --entrypoint sh elastalert
+OR
+docker exec -it <___> sh
+elastalert-test-rule --config $ELASTALERT_CONFIG --count-only "$RULES_DIRECTORY/new_violation.yaml"
+```

docker-compose.yml

@@ -18,7 +18,7 @@ services:
 
   elasticsearch:
     restart: always
-    image: elasticsearch:2
+    image: elasticsearch:5
     ports:
       - 9200:9200
     expose:
@@ -28,3 +28,14 @@ services:
       interval: 5s
       timeout: 3s
       retries: 5
+
+  elastalert:
+    build:
+      context: elastalert
+    environment:
+      - ELASTICSEARCH_HOST=elasticsearch
+      - ELASTICSEARCH_PORT=9200
+      - NOTIFICATION_EMAILS=test@example.com,test2@example.com
+    depends_on:
+      elasticsearch:
+        condition: service_healthy

elastalert/Dockerfile

@@ -0,0 +1,32 @@
+FROM iron/python:2
+
+WORKDIR /opt
+RUN apk update && \
+    apk upgrade && \
+    apk add ca-certificates openssl-dev libffi-dev python-dev gcc musl-dev tzdata openntpd && \
+    rm -rf /var/cache/apk/* && \
+# Install pip - required for installation of Elastalert.
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py && \
+    rm get-pip.py && \
+# Download and unpack Elastalert.
+    wget https://github.com/Yelp/elastalert/archive/master.zip && \
+    unzip *.zip && \
+    rm *.zip && \
+    mv e* elastalert
+
+# Install Elastalert.
+WORKDIR /opt/elastalert
+RUN python setup.py install && \
+    pip install -e . && \
+    pip uninstall twilio --yes && \
+    pip install twilio==6.0.0
+
+ENV RULES_DIRECTORY /opt/rules
+COPY rules ${RULES_DIRECTORY}
+ENV ELASTALERT_CONFIG /opt/elastalert_config.yaml
+COPY elastalert_config.yaml ${ELASTALERT_CONFIG}
+
+COPY start-elastalert.sh /opt/
+RUN chmod +x /opt/start-elastalert.sh
+CMD ["/opt/start-elastalert.sh"]

elastalert/elastalert_config.yaml

@@ -0,0 +1,32 @@
+# This is the folder that contains the rule yaml files
+# Any .yaml file will be loaded as a rule
+rules_folder: /opt/rules
+
+# How often ElastAlert will query Elasticsearch
+# The unit can be anything from weeks to seconds
+run_every:
+  seconds: 10
+
+# ElastAlert will buffer results from the most recent
+# period of time, in case some log sources are not in real time
+buffer_time:
+  minutes: 15
+
+# The Elasticsearch hostname for metadata writeback
+# Note that every rule can have its own Elasticsearch host
+es_host:
+
+# The Elasticsearch port
+es_port:
+
+# The index on es_host which is used for metadata storage
+# This can be a unmapped index, but it is recommended that you run
+# elastalert-create-index to set a mapping
+writeback_index: elastalert_status
+
+# If an alert fails for some reason, ElastAlert will retry
+# sending the alert until this time period has elapsed
+alert_time_limit:
+  days: 2
+
+timestamp_field: Timestamp

elastalert/rules/new_violation.yaml

@@ -0,0 +1,17 @@
+# Alert when a new violation is written to the githubintegration elasticsearch index
+
+name: New violation added to githubintegration index rule
+
+type: new_term
+
+index: githubintegration
+
+fields:
+  - Level
+
+filter:
+- term:
+    _type: "log"
+
+alert:
+- "email"

elastalert/start-elastalert.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+set -e
+
+####################
+# Main config
+####################
+# Set the rule directory in the Elastalert config file to external rules directory.
+sed -i -e"s|rules_folder: [[:print:]]*|rules_folder: ${RULES_DIRECTORY}|g" "$ELASTALERT_CONFIG"
+# Set the Elasticsearch host that Elastalert is to query.
+sed -i -e"s|es_host: [[:print:]]*|es_host: ${ELASTICSEARCH_HOST}|g" "$ELASTALERT_CONFIG"
+# Set the port used by Elasticsearch at the above address.
+sed -i -e"s|es_port: [0-9]*|es_port: ${ELASTICSEARCH_PORT}|g" "$ELASTALERT_CONFIG"
+
+####################
+# Rules config
+####################
+# NOTIFICATION_EMAILS = comma separated list of email addresses
+echo "email:" >> "$RULES_DIRECTORY/new_violation.yaml"
+for i in $(echo $NOTIFICATION_EMAILS | tr "," "\n"); do
+  echo "- $i" >> "$RULES_DIRECTORY/new_violation.yaml"
+done
+
+
+# Wait until Elasticsearch is online since otherwise Elastalert will fail.
+rm -f garbage_file
+while ! wget -O garbage_file "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT" 2>/dev/null
+do
+	echo "Waiting for Elasticsearch..."
+	rm -f garbage_file
+	sleep 1
+done
+rm -f garbage_file
+sleep 5
+
+# Check if the Elastalert index exists in Elasticsearch and create it if it does not.
+if ! wget -O garbage_file "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/elastalert_status" 2>/dev/null
+then
+	echo "Creating Elastalert index in Elasticsearch..."
+    elastalert-create-index \
+    	--host "$ELASTICSEARCH_HOST" \
+    	--port "$ELASTICSEARCH_PORT" \
+    	--config "$ELASTALERT_CONFIG" \
+    	--index elastalert_status \
+    	--old-index ""
+else
+    echo "Elastalert index already exists in Elasticsearch."
+fi
+rm -f garbage_file
+
+echo "Starting Elastalert..."
+exec elastalert --config "$ELASTALERT_CONFIG"

log.go

@@ -3,10 +3,10 @@ package main
 import (
 	"fmt"
 
-	"gopkg.in/olivere/elastic.v3"
-	"gopkg.in/sohlich/elogrus.v1"
-
 	"github.com/Sirupsen/logrus"
+	elastic "gopkg.in/olivere/elastic.v5"
+	elogrus "gopkg.in/sohlich/elogrus.v2"
+
 	"github.com/techjacker/diffence"
 )
 

main.go

@@ -13,6 +13,7 @@ import (
 )
 
 const (
+	elasticSearchIndex  = "githubintegration"
 	headerGithubMAC     = "X-Hub-Signature"
 	headerGithubEvt     = "X-Github-Event"
 	githubWebhookSecret = "GITHUB_WEBHOOKSECRET"
@@ -41,7 +42,7 @@ func getLogger() Log {
 	esURL, ok := os.LookupEnv("ELASTICSEARCH_URL")
 	if ok {
 		fmt.Printf("Logging to elasticsearch: %s\n", esURL)
-		logger, err := NewESLogger(esURL, "githubintegration")
+		logger, err := NewESLogger(esURL, elasticSearchIndex)
 		if err != nil {
 			panic(err)
 		}