Analytics Dashboard

.gitignore

@@ -3,4 +3,5 @@ goguerrilla.conf
 goguerrilla.conf.json
 /guerrillad
 vendor
-go-guerrilla.wiki
+go-guerrilla.wiki
+statik

.travis.yml

@@ -7,8 +7,16 @@ go:
   - 1.10.x
   - master
 
+before_install:
+  - if [[ `npm -v` != 3* ]]; then npm i -g npm@3; fi
+  - nvm install 6 && nvm use 6
+
 install:
   - export GO15VENDOREXPERIMENT=1
+  - go get github.com/rakyll/statik
+  - go install github.com/rakyll/statik
+  - cd dashboard/js && npm install && npm run build && cd ../..
+  - statik -src=dashboard/js/build -dest=dashboard
   - go get github.com/Masterminds/glide
   - go install github.com/Masterminds/glide
   - glide up

Makefile

@@ -16,10 +16,17 @@ help:
 
 clean:
 	rm -f guerrillad
+	rm -rf dashboard/js/node_modules
+	rm -rf dashboard/js/build
 
 dependencies:
 	$(GO_VARS) $(GO) list -f='{{ join .Deps "\n" }}' $(ROOT)/cmd/guerrillad | grep -v $(ROOT) | tr '\n' ' ' | $(GO_VARS) xargs $(GO) get -u -v
 	$(GO_VARS) $(GO) list -f='{{ join .Deps "\n" }}' $(ROOT)/cmd/guerrillad | grep -v $(ROOT) | tr '\n' ' ' | $(GO_VARS) xargs $(GO) install -v
+	cd dashboard/js && npm install && cd ../..
+
+dashboard: dashboard/*.go */*/*/*.js */*/*/*/*.js
+	cd dashboard/js && npm run build && cd ../..
+	statik -src=dashboard/js/build -dest=dashboard
 
 guerrillad: *.go */*.go */*/*.go
 	$(GO_VARS) $(GO) build -o="guerrillad" -ldflags="$(LD_FLAGS)" $(ROOT)/cmd/guerrillad
@@ -41,4 +48,4 @@ testrace: *.go */*.go */*/*.go
 	$(GO_VARS) $(GO) test -v ./tests -race
 	$(GO_VARS) $(GO) test -v ./cmd/guerrillad -race
 	$(GO_VARS) $(GO) test -v ./response -race
-	$(GO_VARS) $(GO) test -v ./backends -race
+	$(GO_VARS) $(GO) test -v ./backends -race

backends/gateway.go

@@ -7,11 +7,12 @@ import (
 	"sync"
 	"time"
 
+	"runtime/debug"
+	"strings"
+
 	"github.com/flashmob/go-guerrilla/log"
 	"github.com/flashmob/go-guerrilla/mail"
 	"github.com/flashmob/go-guerrilla/response"
-	"runtime/debug"
-	"strings"
 )
 
 var ErrProcessorNotFound error

build.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Build frontend to `dashboard/js/build`
+cd dashboard/js && npm install && cd ../../
+cd dashboard/js && npm run build && cd ../../
+# Build statik file system in `dashboard/statik`
+statik -src=dashboard/js/build -dest=dashboard

cmd/guerrillad/backend_test.go.no

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"testing"
+	"os"
+	"time"
+	"io/ioutil"
+	"github.com/flashmob/go-guerrilla/tests/testcert"
+	"github.com/flashmob/go-guerrilla/log"
+	"runtime"
+	"github.com/spf13/cobra"
+	"sync"
+	"strings"
+	"fmt"
+)
+
+func TestBadBackendReload2(t *testing.T) {
+
+	testcert.GenerateCert("mail2.guerrillamail.com", "", 365*24*time.Hour, false, 2048, "P256", "../../tests/")
+	os.Truncate("../../tests/testlog", 0)
+	//mainlog, _ = log.GetLogger("../../tests/testlog")
+	mainlog, _ = log.GetLogger("stdout")
+	mainlog.SetLevel("debug")
+	mainlog.Info("are u sure")
+	mainlog.Info("not another word")
+
+	select {
+
+	case <-time.After(10 * time.Second):
+		mainlog.Info("paabix")
+		stacktrace := make([]byte, 8192)
+		length := runtime.Stack(stacktrace, true)
+		_ = length
+		fmt.Fprintf(ioutil.Discard, (string(stacktrace[:length])))
+
+		panic("timed out")
+	}
+
+	mainlog.Info("not another word")
+	sigKill()
+	ioutil.WriteFile("configJsonA.json", []byte(configJsonA), 0644)
+	cmd := &cobra.Command{}
+	configPath = "configJsonA.json"
+	var serveWG sync.WaitGroup
+	serveWG.Add(1)
+	go func() {
+		mainlog.Info("start serve")
+		serve(cmd, []string{})
+		serveWG.Done()
+	}()
+	mainlog.Info("after start")
+	time.Sleep(testPauseDuration)
+
+	// change the config file to the one with a broken backend
+	ioutil.WriteFile("configJsonA.json", []byte(configJsonE), 0644)
+
+	// test SIGHUP via the kill command
+	// Would not work on windows as kill is not available.
+	// TODO: Implement an alternative test for windows.
+	if runtime.GOOS != "windows" {
+		sigHup()
+		time.Sleep(testPauseDuration) // allow sighup to do its job
+		// did the pidfile change as expected?
+		if _, err := os.Stat("./pidfile2.pid"); os.IsNotExist(err) {
+			t.Error("pidfile not changed after sighup SIGHUP", err)
+		}
+	}
+
+	// send kill signal and wait for exit
+	sigKill()
+	serveWG.Wait()
+	//time.Sleep(time.Second * 3)
+	// did backend started as expected?
+	fd, err := os.Open("../../tests/testlog")
+	if err != nil {
+		t.Error(err)
+	}
+	if read, err := ioutil.ReadAll(fd); err == nil {
+		logOutput := string(read)
+		if i := strings.Index(logOutput, "reverted to old backend config"); i < 0 {
+			t.Error("did not revert to old backend config")
+		}
+	}
+
+	// cleanup
+	//os.Truncate("../../tests/testlog", 0)
+	os.Remove("configJsonA.json")
+	os.Remove("./pidfile.pid")
+	os.Remove("./pidfile2.pid")
+
+}

config.go

@@ -5,12 +5,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/flashmob/go-guerrilla/backends"
-	"github.com/flashmob/go-guerrilla/log"
 	"os"
 	"reflect"
 	"strings"
 	"time"
+
+	"github.com/flashmob/go-guerrilla/backends"
+	"github.com/flashmob/go-guerrilla/dashboard"
+	"github.com/flashmob/go-guerrilla/log"
 )
 
 // AppConfig is the holder of the configuration of the app
@@ -30,6 +32,8 @@ type AppConfig struct {
 	LogLevel string `json:"log_level,omitempty"`
 	// BackendConfig configures the email envelope processing backend
 	BackendConfig backends.BackendConfig `json:"backend_config"`
+	// Dashboard config configures how analytics are gathered and displayed
+	Dashboard dashboard.Config `json:"dashboard"`
 }
 
 // ServerConfig specifies config options for a single server

dashboard/README.md

@@ -0,0 +1,82 @@
+# About the Dashboard
+
+The dashboard package gathers data about Guerrilla while it is running 
+and provides an analytics web dashboard. To activate the dashboard, checkout
+the dashboard branch, then build it, then edit your configuration 
+file as specified in the example configuration.
+
+tl/dr
+
+```
+$ git checkout dashboard
+$ cd dashboard/js
+$ npm install
+$ npm run build
+$ cd ..
+$ statik -src=./js/build
+$ cd ..
+$ make guerrillad
+```
+
+Then see the Config section below how to enable it!
+
+## Screenshot
+
+![](https://cloud.githubusercontent.com/assets/10557821/23277291/233efc0c-f9c2-11e6-8374-f2515f7d868c.png)
+
+## The Backend
+
+The backend is a Go package that collects and stores data from guerrillad, 
+serves the dashboard to web clients, and updates clients with new analytics data 
+over WebSockets. 
+
+The backend uses [statik](https://github.com/rakyll/statik) to convert the `build` 
+folder into a http-servable Go package. When deploying, the frontend should be 
+built first, then the `statik` package should be created. 
+An example of this process is in the `.travis.yml`.
+
+`To build the statik Go package, cd to the `dashboard` dir, then run
+
+ `statik -src=./js/build` 
+
+## The Frontend
+
+The front-end is written in React and uses WebSockets to accept data 
+from the backend and [Victory](https://formidable.com/open-source/victory/) to render charts. 
+The `js` directory is an NPM module that contains all frontend code. 
+All commands below should be run within the `js` directory.
+
+To install frontend dependencies:
+`npm install`
+
+To build the frontend code:
+`npm run build`
+
+To run the HMR development server (serves frontend on port 3000 rather than through `dashboard` package):
+`npm start`
+
+## Config
+
+Add `dashboard` to your goguerrilla.json config file
+
+```
+"dashboard": {
+    "is_enabled": true,
+    "listen_interface": ":8081",
+    "tick_interval": "5s",
+    "max_window": "24h",
+    "ranking_aggregation_interval": "6h"
+  }
+```
+
+## Security considerations
+
+Warning: The dashboard does not have any authentication. It is also served over HTTP.
+
+Assuming that the host will open the dashboard http port only to the local network or VPN. 
+However, if you need to access the dashboard securely from a remote connection and
+don't have a VPN, then maybe an SSH tunnel could do:
+
+`ssh you@example.com -L 8081:127.0.0.1:8081 -N`
+
+Then point your browser to http://127.0.0.1:8081

dashboard/conn_record.go

@@ -0,0 +1,51 @@
+package dashboard
+
+import "container/heap"
+
+// Records ranking of one unique connection in domain/ip/helo rankings.
+type connRecord struct {
+	// Number of records of this type
+	count int
+	// Name of the record, either the domain, IP, or helo
+	value string
+}
+
+// Contains all ranking of a particular type (domain/ip/helo).
+// Tracks ranking ordering by implementing heap.Interface
+type connRecordHeap []connRecord
+
+func (crh connRecordHeap) Len() int {
+	return len(crh)
+}
+
+func (crh connRecordHeap) Less(i, j int) bool {
+	return crh[i].count > crh[j].count
+}
+
+func (crh connRecordHeap) Swap(i, j int) {
+	crh[i], crh[j] = crh[j], crh[i]
+}
+
+func (crh *connRecordHeap) Push(x interface{}) {
+	*crh = append(*crh, x.(connRecord))
+}
+
+func (crh *connRecordHeap) Pop() interface{} {
+	old := *crh
+	l := len(old)
+	toPop := old[l-1]
+	*crh = old[:l-1]
+	return toPop
+}
+
+// Gets N records with the greatest counts, maintaining the state of the heap
+func (crh *connRecordHeap) GetN(n int) []connRecord {
+	nHighest := make([]connRecord, n)
+	for i := 0; i < n; i++ {
+		nHighest[i] = heap.Pop(crh).(connRecord)
+	}
+	for _, cr := range nHighest {
+		heap.Push(crh, cr)
+	}
+	return nHighest
+}

dashboard/conn_record_test.go

@@ -0,0 +1,28 @@
+package dashboard
+
+import (
+	"container/heap"
+	"math/rand"
+	"testing"
+)
+
+const (
+	nConnRecords = 100
+)
+
+func TestConnRecord(t *testing.T) {
+	var crHeap connRecordHeap = make([]connRecord, nConnRecords)
+	var max connRecord
+	for i := 0; i < nConnRecords; i++ {
+		crHeap[i] = connRecord{rand.Int(), "abc"}
+		if crHeap[i].count > max.count {
+			max = crHeap[i]
+		}
+	}
+
+	heap.Init(&crHeap)
+
+	if max.count != heap.Pop(&crHeap).(connRecord).count {
+		t.Error("Pop did not return maximum value")
+	}
+}