From e4f36982f1c94433507c5a40f35e61794d7ba165 Mon Sep 17 00:00:00 2001 From: Tuan Nguyen Date: Thu, 25 Jul 2019 16:06:07 +0700 Subject: [PATCH 1/2] POSTing with http-endpoint as application/json instead of uncommon application/x-www-form-urlencoded --- notifier/http-endpoint-notifier.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/notifier/http-endpoint-notifier.go b/notifier/http-endpoint-notifier.go index f7562c0d..e066caee 100644 --- a/notifier/http-endpoint-notifier.go +++ b/notifier/http-endpoint-notifier.go @@ -2,9 +2,10 @@ package notifier import ( "fmt" + "bytes" + "encoding/json" "io/ioutil" "net/http" - "net/url" log "github.com/AcalephStorage/consul-alerts/Godeps/_workspace/src/github.com/Sirupsen/logrus" ) @@ -38,17 +39,25 @@ func (notifier *HttpEndpointNotifier) Notify(messages Messages) bool { PassCount: pass, Nodes: mapByNodes(messages), } - values := url.Values{} + values := map[string]string{} + for key, val := range notifier.Payload { data, err := renderTemplate(t, "", val) if err != nil { log.Println("Error rendering template: ", err) return false } - values.Set(key, string(data)) + values[key] = string(data) } + + requestBody, err := json.Marshal(values) + if err != nil { + log.Println("Unable to encode POST data") + return false + } + endpoint := fmt.Sprintf("%s%s", notifier.BaseURL, notifier.Endpoint) - if res, err := http.PostForm(endpoint, values); err != nil { + if res, err := http.Post(endpoint, "application/json", bytes.NewBuffer(requestBody)); err != nil { log.Println("Unable to send data to HTTP endpoint:", err) return false } else { From 48b61194659f650c018a351083c7de5bb592ee00 Mon Sep 17 00:00:00 2001 From: Tuan Nguyen Date: Wed, 25 Sep 2019 13:00:27 +0700 Subject: [PATCH 2/2] Prometheus's alertmanager notifier --- README.md | 29 +++++++ consul-alerts.go | 6 +- consul/client.go | 16 ++++ consul/interface.go | 9 ++- notifier/notifier.go | 3 + notifier/prometheus-notifier.go | 134 ++++++++++++++++++++++++++++++++ 6 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 notifier/prometheus-notifier.go diff --git a/README.md b/README.md index 2b8dfb94..89e7cb14 100644 --- a/README.md +++ b/README.md @@ -607,6 +607,35 @@ prefix: `consul-alerts/config/notifiers/ilert/` | api-key | The API key of the alert source. (mandatory) | | incident-key-template | Format of the incident key. [Default: `{{.Node}}:{{.Service}}:{{.Check}}` | +#### Prometheus + +To enable the Prometheus built-in notifier, set +`consul-alerts/config/notifiers/prometheus/enabled` to `true`. Prometheus details +needs to be configured. + +prefix: `consul-alerts/config/notifiers/prometheus/` + +| key | description | +|--------------|-----------------------------------------------------------------------| +| enabled | Enable the prometheus notifier. [Default: false] | +| cluster-name | The name of the cluster. [Default: "Consul Alerts"] | +| base-urls | Base URLs of the Prometheus cluster (mandatory) | +| endpoint | The endpoint to append to the end of each base-url | +| payload | The payload to send to the Prometheus server (mandatory) | + +The value of 'payload' must be a json map of type string. Value will be rendered as a template. +``` +{ + "alertName": "{{ .Check }}/{{ .Service }}/{{ .Node }}", + "host": "{{ .Node }}", + "service": "{{ .Service }}", + "severity": "{{ .Status }}", + "output": "{{ .Output }}", + "notes": "{{ .Notes }}", + "time": "{{ .Timestamp }}" +} +``` + Health Check via API -------------------- diff --git a/consul-alerts.go b/consul-alerts.go index 91be3598..ddd116a6 100644 --- a/consul-alerts.go +++ b/consul-alerts.go @@ -250,6 +250,7 @@ func builtinNotifiers() map[string]notifier.Notifier { awssnsNotifier := consulClient.AwsSnsNotifier() victoropsNotifier := consulClient.VictorOpsNotifier() httpEndpointNotifier := consulClient.HttpEndpointNotifier() + prometheusNotifier := consulClient.PrometheusNotifier() ilertNotifier := consulClient.ILertNotifier() notifiers := map[string]notifier.Notifier{} @@ -288,7 +289,10 @@ func builtinNotifiers() map[string]notifier.Notifier { } if httpEndpointNotifier.Enabled { notifiers[httpEndpointNotifier.NotifierName()] = httpEndpointNotifier - } + } + if prometheusNotifier.Enabled { + notifiers[prometheusNotifier.NotifierName()] = prometheusNotifier + } if ilertNotifier.Enabled { notifiers[ilertNotifier.NotifierName()] = ilertNotifier } diff --git a/consul/client.go b/consul/client.go index c8ac8989..1e9a930d 100644 --- a/consul/client.go +++ b/consul/client.go @@ -253,6 +253,18 @@ func (c *ConsulAlertClient) LoadConfig() { case "consul-alerts/config/notifiers/http-endpoint/payload": valErr = loadCustomValue(&config.Notifiers.HttpEndpoint.Payload, val, ConfigTypeStrMap) + // Prometheus notifier config + case "consul-alerts/config/notifiers/prometheus/enabled": + valErr = loadCustomValue(&config.Notifiers.Prometheus.Enabled, val, ConfigTypeBool) + case "consul-alerts/config/notifiers/prometheus/cluster-name": + valErr = loadCustomValue(&config.Notifiers.Prometheus.ClusterName, val, ConfigTypeString) + case "consul-alerts/config/notifiers/prometheus/base-urls": + valErr = loadCustomValue(&config.Notifiers.Prometheus.BaseURLs, val, ConfigTypeStrArray) + case "consul-alerts/config/notifiers/prometheus/endpoint": + valErr = loadCustomValue(&config.Notifiers.Prometheus.Endpoint, val, ConfigTypeString) + case "consul-alerts/config/notifiers/prometheus/payload": + valErr = loadCustomValue(&config.Notifiers.Prometheus.Payload, val, ConfigTypeStrMap) + // iLert notfier config case "consul-alerts/config/notifiers/ilert/enabled": valErr = loadCustomValue(&config.Notifiers.ILert.Enabled, val, ConfigTypeBool) @@ -566,6 +578,10 @@ func (c *ConsulAlertClient) HttpEndpointNotifier() *notifier.HttpEndpointNotifie return c.config.Notifiers.HttpEndpoint } +func (c *ConsulAlertClient) PrometheusNotifier() *notifier.PrometheusNotifier { + return c.config.Notifiers.Prometheus +} + func (c *ConsulAlertClient) ILertNotifier() *notifier.ILertNotifier { return c.config.Notifiers.ILert } diff --git a/consul/interface.go b/consul/interface.go index 5d43cf9b..f286451f 100644 --- a/consul/interface.go +++ b/consul/interface.go @@ -81,6 +81,7 @@ type Consul interface { AwsSnsNotifier() *notifier.AwsSnsNotifier VictorOpsNotifier() *notifier.VictorOpsNotifier HttpEndpointNotifier() *notifier.HttpEndpointNotifier + PrometheusNotifier() *notifier.PrometheusNotifier ILertNotifier() *notifier.ILertNotifier CheckChangeThreshold() int @@ -173,7 +174,12 @@ func DefaultAlertConfig() *ConsulAlertConfig { httpEndpoint := ¬ifier.HttpEndpointNotifier{ Enabled: false, ClusterName: "Consul-Alerts", - } + } + + prometheus := ¬ifier.PrometheusNotifier{ + Enabled: false, + ClusterName: "Consul-Alerts", + } ilert := ¬ifier.ILertNotifier{ Enabled: false, @@ -193,6 +199,7 @@ func DefaultAlertConfig() *ConsulAlertConfig { AwsSns: awsSns, VictorOps: victorOps, HttpEndpoint: httpEndpoint, + Prometheus: prometheus, ILert: ilert, Custom: []string{}, } diff --git a/notifier/notifier.go b/notifier/notifier.go index 7d236755..c39c37b4 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -53,6 +53,7 @@ type Notifiers struct { AwsSns *AwsSnsNotifier `json:"awssns"` VictorOps *VictorOpsNotifier `json:"victorops"` HttpEndpoint *HttpEndpointNotifier `json:"http-endpoint"` + Prometheus *PrometheusNotifier `json:"prometheus"` ILert *ILertNotifier `json:"ilert"` Custom []string `json:"custom"` } @@ -83,6 +84,8 @@ func (n Notifiers) GetNotifier(name string) (Notifier, bool) { return n.VictorOps, true case n.HttpEndpoint != nil && n.HttpEndpoint.NotifierName() == name: return n.HttpEndpoint, true + case n.Prometheus != nil && n.Prometheus.NotifierName() == name: + return n.Prometheus, true case n.ILert != nil && n.ILert.NotifierName() == name: return n.ILert, true diff --git a/notifier/prometheus-notifier.go b/notifier/prometheus-notifier.go new file mode 100644 index 00000000..46d04f33 --- /dev/null +++ b/notifier/prometheus-notifier.go @@ -0,0 +1,134 @@ +package notifier + +import ( + "fmt" + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "html/template" + + log "github.com/AcalephStorage/consul-alerts/Godeps/_workspace/src/github.com/Sirupsen/logrus" +) + +type PrometheusNotifier struct { + Enabled bool + ClusterName string `json:"cluster-name"` + BaseURLs []string `json:"base-urls"` + Endpoint string `json:"endpoint"` + Payload map[string]string `json:"payload"` +} + +type TemplatePayloadData struct { + Node string + Service string + Check string + Status string + Output string + Notes string + Timestamp string +} + +// NotifierName provides name for notifier selection +func (notifier *PrometheusNotifier) NotifierName() string { + return "prometheus" +} + +func (notifier *PrometheusNotifier) Copy() Notifier { + n := *notifier + return &n +} + +func renderPayload(t TemplatePayloadData, templateFile string, defaultTemplate string) (string, error) { + var tmpl *template.Template + var err error + if templateFile == "" { + tmpl, err = template.New("base").Parse(defaultTemplate) + } else { + tmpl, err = template.ParseFiles(templateFile) + } + + if err != nil { + return "", err + } + + var body bytes.Buffer + if err := tmpl.Execute(&body, t); err != nil { + return "", err + } + + return body.String(), nil +} + +//Notify sends messages to the endpoint notifier +func (notifier *PrometheusNotifier) Notify(messages Messages) bool { + var values []map[string]map[string]string + + for _, m := range messages { + value := map[string]string{} + t := TemplatePayloadData{ + Node: m.Node, + Service: m.Service, + Check: m.Check, + Status: m.Status, + Output: m.Output, + Notes: m.Notes, + Timestamp: m.Timestamp.Format("2006-01-02T15:04:05-0700"), + } + + for payloadKey, payloadVal := range notifier.Payload { + data, err := renderPayload(t, "", payloadVal) + if err != nil { + log.Println("Error rendering template: ", err) + return false + } + value[payloadKey] = string(data) + } + + values = append(values, map[string]map[string]string{"labels": value}) + } + + requestBody, err := json.Marshal(values) + if err != nil { + log.Println("Unable to encode POST data") + return false + } + + c := make(chan bool) + defer close(c) + for _, bu := range notifier.BaseURLs { + endpoint := fmt.Sprintf("%s%s", bu, notifier.Endpoint) + + // Channel senders. Logging the result where needed, and sending status back + go func() { + if res, err := http.Post(endpoint, "application/json", bytes.NewBuffer(requestBody)); err != nil { + log.Printf(fmt.Sprintf("Unable to send data to Prometheus server (%s): %s", endpoint, err)) + c <- false + } else { + defer res.Body.Close() + statusCode := res.StatusCode + + if statusCode != 200 { + body, _ := ioutil.ReadAll(res.Body) + log.Printf(fmt.Sprintf("Unable to notify Prometheus server (%s): %s", endpoint, string(body))) + c <- false + } else { + log.Printf(fmt.Sprintf("Notification sent to Prometheus server (%s).", endpoint)) + c <- true + } + } + }() + } + + // Channel receiver. Making sure to return the final result in bool + for i := 0; i < len(notifier.BaseURLs); i++ { + select { + case r := <- c: + if (! r) { + return false + } + } + } + + return true +}