merge: feat/alerting (#4)

feat: alert credentials api to sync configs to alertmanager

.github/workflows/test.yaml

@@ -2,18 +2,28 @@ on: [push, pull_request]
 name: Test
 jobs:
   test:
-    strategy:
-      matrix:
-        go-version: [1.15.x]
-        os: [ubuntu-latest]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:12
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
-      - name: Install Go
+      - name: Set up Go 1.x
         uses: actions/setup-go@v2
         with:
-          go-version: ${{ matrix.go-version }}
-      - name: Checkout code
+          go-version: ^1.15
+        id: go
+      - name: Check out code into the Go module directory
         uses: actions/checkout@v2
       - name: Test
+        env:
+          POSTGRES_PASSWORD: postgres
         run: make test
-

alert/alertmanager/configsync.go

@@ -0,0 +1,127 @@
+package alertmanager
+
+import (
+	"bytes"
+	"context"
+	"github.com/grafana/cortex-tools/pkg/client"
+	"github.com/markbates/pkger"
+	"github.com/odpf/siren/domain"
+	"github.com/prometheus/alertmanager/config"
+	"text/template"
+)
+
+type SlackCredential struct {
+	Webhook  string
+	Channel  string
+	Username string
+}
+
+type SlackConfig struct {
+	Critical SlackCredential
+	Warning  SlackCredential
+}
+
+type TeamCredentials struct {
+	PagerdutyCredential string
+	Slackcredentials    SlackConfig
+	Name                string
+}
+
+type EntityCredentials struct {
+	Entity string
+	Teams  map[string]TeamCredentials
+}
+
+type Client interface {
+	SyncConfig(credentials EntityCredentials) error
+}
+
+type AlertmanagerClient struct {
+	CortextClient client.CortexClient
+	vartmplStr    string
+	detmplStr     string
+}
+
+func NewClient(c domain.AlertmanagerConfig) (AlertmanagerClient, error) {
+	config := client.Config{
+		Address: c.Address,
+	}
+	amClient, err := client.New(config)
+	if err != nil {
+		return AlertmanagerClient{}, err
+	}
+
+	deTemplatePath := "/alert/alertmanagerde.tmpl"
+	deTmplateString, err := readTemplateString(err, deTemplatePath)
+	if err != nil {
+		return AlertmanagerClient{}, err
+	}
+	varTmplPath := "/alert/alertmanagervar.tmpl"
+	varTmplateString, err := readTemplateString(err, varTmplPath)
+	if err != nil {
+		return AlertmanagerClient{}, err
+	}
+	return AlertmanagerClient{
+		CortextClient: *amClient,
+		detmplStr:     deTmplateString,
+		vartmplStr:    varTmplateString,
+	}, nil
+}
+
+func (am AlertmanagerClient) SyncConfig(credentials EntityCredentials) error {
+	cfg, err := generateAlertmanagerConfig(credentials)
+	if err != nil {
+		return err
+	}
+	templates := map[string]string{
+		"var.tmpl": am.vartmplStr,
+		"de.tmpl":  am.detmplStr,
+	}
+
+	ctx := client.NewContextWithTenantId(context.Background(), credentials.Entity)
+	err = am.CortextClient.CreateAlertmanagerConfig(ctx, cfg, templates)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func generateAlertmanagerConfig(credentials EntityCredentials) (string, error) {
+	configYaml, err := pkger.Open("/alert/alertmanagerconfig.goyaml")
+	if err != nil {
+		return "", err
+	}
+	defer configYaml.Close()
+	configYamlBuf := new(bytes.Buffer)
+	configYamlBuf.ReadFrom(configYaml)
+	delims := template.New("alertmanagerConfigTemplate").Delims("[[", "]]")
+	parse, err := delims.Parse(configYamlBuf.String())
+	if err != nil {
+		return "", err
+	}
+	var tpl bytes.Buffer
+	err = parse.Execute(&tpl, credentials)
+	if err != nil {
+		return "", err
+	}
+	configStr := tpl.String()
+	_, err = config.Load(configStr)
+	if err != nil {
+		return "", err
+	}
+	return configStr, nil
+}
+
+func readTemplateString(err error, templatePath string) (string, error) {
+	tmpl, err := pkger.Open(templatePath)
+	if err != nil {
+		return "", err
+	}
+	defer tmpl.Close()
+	detmplBuf := new(bytes.Buffer)
+	_, err = detmplBuf.ReadFrom(tmpl)
+	if err != nil {
+		return "", err
+	}
+	return detmplBuf.String(), nil
+}

alert/alertmanager/configsync_test.go

@@ -0,0 +1,218 @@
+package alertmanager
+
+import (
+	"bytes"
+	"github.com/odpf/siren/domain"
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestGenerateAlertmanagerConfig(t *testing.T) {
+	credentials := EntityCredentials{
+		Entity: "de-infra",
+		Teams: map[string]TeamCredentials{
+			"eureka": {
+				Name: "eureka",
+				Slackcredentials: SlackConfig{
+					Critical: SlackCredential{
+						Webhook:  "http://eurcritical.com",
+						Channel:  "dss",
+						Username: "ss",
+					},
+				},
+			},
+			"wonder-woman": {
+				Name: "wonder-woman",
+				Slackcredentials: SlackConfig{
+					Warning: SlackCredential{
+						Webhook:  "http://eurcritical.com",
+						Channel:  "dss",
+						Username: "ss",
+					},
+				},
+				PagerdutyCredential: "abc",
+			},
+		},
+	}
+	expectedConfigStr :=
+		`  templates:
+    - 'de.tmpl'
+    - 'var.tmpl'
+  global:
+    pagerduty_url: https://events.pagerduty.com/v2/enqueue
+    resolve_timeout: 5m
+  receivers:
+    - name: default
+    - name: slack-critical-eureka
+      slack_configs:
+        - channel: 'dss'
+          api_url: 'http://eurcritical.com'
+          username: 'ss'
+          icon_emoji: ':eagle:'
+          link_names: false
+          send_resolved: true
+          color: '{{ template "slack.color" . }}'
+          title: ''
+          pretext: '{{template "slack.pretext" . }}'
+          text: '{{ template "slack.body" . }}'
+          actions:
+            - type: button
+              text: 'Runbook :books:'
+              url: '{{template "slack.runbook" . }}'
+            - type: button
+              text: 'Dashboard :bar_chart:'
+              url: '{{template "slack.dashboard" . }}'
+  
+    - name: slack-warning-wonder-woman
+      slack_configs:
+        - channel: 'dss'
+          api_url: 'http://eurcritical.com'
+          username: 'ss'
+          icon_emoji: ':eagle:'
+          link_names: false
+          send_resolved: true
+          color: '{{ template "slack.color" . }}'
+          title: ''
+          pretext: '{{template "slack.pretext" . }}'
+          text: '{{ template "slack.body" . }}'
+          actions:
+            - type: button
+              text: 'Runbook :books:'
+              url: '{{template "slack.runbook" . }}'
+            - type: button
+              text: 'Dashboard :bar_chart:'
+              url: '{{template "slack.dashboard" . }}'
+    - name: pagerduty-wonder-woman
+      pagerduty_configs:
+        - service_key: 'abc'
+  route:
+    group_by:
+      - alertname
+      - severity
+      - owner
+      - service_name
+      - time_stamp
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 4h
+    receiver: default
+    routes:
+      - match:
+          team: 'eureka'
+        routes:
+          - match:
+              severity: "CRITICAL"
+              environment: "production"
+            receiver: default
+            continue: true
+          - match:
+              severity: "CRITICAL"
+            receiver: slack-critical-eureka
+          - match:
+              severity: "WARNING"
+            receiver:  default 
+
+      - match:
+          team: 'wonder-woman'
+        routes:
+          - match:
+              severity: "CRITICAL"
+              environment: "production"
+            receiver: pagerduty-wonder-woman
+            continue: true
+          - match:
+              severity: "CRITICAL"
+            receiver:  default 
+          - match:
+              severity: "WARNING"
+            receiver: slack-warning-wonder-woman
+
+`
+	configStr, err := generateAlertmanagerConfig(credentials)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, expectedConfigStr, configStr)
+
+}
+
+type ConfigCompat struct {
+	TemplateFiles      map[string]string `yaml:"template_files"`
+	AlertmanagerConfig string            `yaml:"alertmanager_config"`
+}
+
+func TestSyncConfig(t *testing.T) {
+	credentials := EntityCredentials{
+		Entity: "greek",
+		Teams: map[string]TeamCredentials{
+			"eureka": {
+				Name: "eureka",
+				Slackcredentials: SlackConfig{
+					Critical: SlackCredential{
+						Webhook:  "http://eurcritical.com",
+						Channel:  "dss",
+						Username: "ss",
+					},
+				},
+			},
+			"wonder": {
+				Name: "wonder",
+				Slackcredentials: SlackConfig{
+					Warning: SlackCredential{
+						Webhook:  "http://eurcritical.com",
+						Channel:  "dss",
+						Username: "ss",
+					},
+				},
+				PagerdutyCredential: "abc",
+			},
+		},
+	}
+	t.Run("should return error if alertmanager response code is non-2xx", func(t *testing.T) {
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(500)
+		}))
+		defer ts.Close()
+		client, err := NewClient(domain.AlertmanagerConfig{
+			Address: ts.URL,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = client.SyncConfig(credentials)
+		assert.Error(t, err)
+
+	})
+	t.Run("should return nil on successful sync", func(t *testing.T) {
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			tenant := r.Header.Get("X-Scope-Orgid")
+			assert.Equal(t, "greek", tenant)
+			requestBody := ConfigCompat{}
+			buf := new(bytes.Buffer)
+			buf.ReadFrom(r.Body)
+			err := yaml.Unmarshal(buf.Bytes(), &requestBody)
+			if err != nil {
+				t.Fatal(err)
+			}
+			assert.NotEmpty(t, requestBody.AlertmanagerConfig)
+			vartmpl := requestBody.TemplateFiles["var.tmpl"]
+			detmpl := requestBody.TemplateFiles["de.tmpl"]
+			assert.NotEmpty(t, vartmpl)
+			assert.NotEmpty(t, detmpl)
+		}))
+		defer ts.Close()
+		client, err := NewClient(domain.AlertmanagerConfig{
+			Address: ts.URL,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = client.SyncConfig(credentials)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}