Merge pull request #14 from stefanprodan/slack

 Add details to Slack messages

.codecov.yml

@@ -3,6 +3,6 @@ coverage:
  project:
  default:
  target: auto
- threshold: 0.50
+ threshold: 50
  base: auto
- patch: off
+ patch: off

README.md

@@ -367,7 +367,25 @@ helm upgrade -i flagger flagger/flagger \
 Once configured with a Slack incoming webhook, Flagger will post messages when a canary deployment has been initialized,
 when a new revision has been detected and if the canary analysis failed or succeeded.
 
-![flagger-slack](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-notifications.png)
+![flagger-slack](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-canary-success.png)
+
+A canary deployment will be rolled back if the progress deadline exceeded or if the analysis 
+reached the maximum number of failed checks:
+
+![flagger-slack-errors](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-canary-failed.png)
+
+Besides Slack, you can use Alertmanager to trigger alerts when a canary deployment failed:
+
+```yaml
+ - alert: canary_rollback
+ expr: flagger_canary_status > 1
+ for: 1m
+ labels:
+ severity: warning
+ annotations:
+ summary: "Canary failed"
+ description: "Workload {{ $labels.name }} namespace {{ $labels.namespace }}"
+```
 
 ### Roadmap
 

pkg/controller/controller.go

@@ -260,12 +260,36 @@ func (c *Controller) recordEventWarningf(r *flaggerv1.Canary, template string, a
  c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...))
 }
 
-func (c *Controller) sendNotification(workload string, namespace string, message string, warn bool) {
+func (c *Controller) sendNotification(cd *flaggerv1.Canary, message string, metadata bool, warn bool) {
  if c.notifier == nil {
  return
  }
 
- err := c.notifier.Post(workload, namespace, message, warn)
+ var fields []notifier.SlackField
+
+ if metadata {
+ fields = append(fields,
+ notifier.SlackField{
+ Title: "Target",
+ Value: fmt.Sprintf("%s/%s.%s", cd.Spec.TargetRef.Kind, cd.Spec.TargetRef.Name, cd.Namespace),
+ },
+ notifier.SlackField{
+ Title: "Traffic routing",
+ Value: fmt.Sprintf("Weight step: %v max: %v",
+ cd.Spec.CanaryAnalysis.StepWeight,
+ cd.Spec.CanaryAnalysis.MaxWeight),
+ },
+ notifier.SlackField{
+ Title: "Failed checks threshold",
+ Value: fmt.Sprintf("%v", cd.Spec.CanaryAnalysis.Threshold),
+ },
+ notifier.SlackField{
+ Title: "Progress deadline",
+ Value: fmt.Sprintf("%vs", cd.GetProgressDeadlineSeconds()),
+ },
+ )
+ }
+ err := c.notifier.Post(cd.Name, cd.Namespace, message, fields, warn)
  if err != nil {
  c.logger.Error(err)
  }

pkg/controller/scheduler.go

@@ -95,11 +95,15 @@ func (c *Controller) advanceCanary(name string, namespace string) {
  if cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold {
  c.recordEventWarningf(cd, "Rolling back %s.%s failed checks threshold reached %v",
  cd.Name, cd.Namespace, cd.Status.FailedChecks)
+ c.sendNotification(cd, fmt.Sprintf("Failed checks threshold reached %v", cd.Status.FailedChecks),
+ false, true)
  }
 
  if !retriable {
  c.recordEventWarningf(cd, "Rolling back %s.%s progress deadline exceeded %v",
  cd.Name, cd.Namespace, err)
+ c.sendNotification(cd, fmt.Sprintf("Progress deadline exceeded %v", err),
+ false, true)
  }
 
  // route all traffic back to primary
@@ -112,7 +116,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
 
  c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight)
  c.recordEventWarningf(cd, "Canary failed! Scaling down %s.%s",
- cd.Spec.TargetRef.Name, cd.Namespace)
+ cd.Name, cd.Namespace)
 
  // shutdown canary
  if err := c.deployer.Scale(cd, 0); err != nil {
@@ -127,8 +131,6 @@ func (c *Controller) advanceCanary(name string, namespace string) {
  }
 
  c.recorder.SetStatus(cd)
- c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace,
- "Canary analysis failed, rollback finished.", true)
  return
  }
 
@@ -199,8 +201,8 @@ func (c *Controller) advanceCanary(name string, namespace string) {
  return
  }
  c.recorder.SetStatus(cd)
- c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace,
- "Canary analysis completed successfully, promotion finished.", false)
+ c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
+ false, false)
  }
 }
 
@@ -217,15 +219,15 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl
  }
  c.recorder.SetStatus(cd)
  c.recordEventInfof(cd, "Initialization done! %s.%s", cd.Name, cd.Namespace)
- c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace,
- "New deployment detected, initialization completed.", false)
+ c.sendNotification(cd, "New deployment detected, initialization completed.",
+ true, false)
  return false
  }
 
  if diff, err := deployer.IsNewSpec(cd); diff {
  c.recordEventInfof(cd, "New revision detected! Scaling up %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
- c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace,
- "New revision detected, starting canary analysis.", false)
+ c.sendNotification(cd, "New revision detected, starting canary analysis.",
+ true, false)
  if err = deployer.Scale(cd, 1); err != nil {
  c.recordEventErrorf(cd, "%v", err)
  return false

pkg/notifier/slack.go

@@ -30,10 +30,17 @@ type SlackPayload struct {
 
 // SlackAttachment holds the markdown message body
 type SlackAttachment struct {
- Color string `json:"color"`
- AuthorName string `json:"author_name"`
- Text string `json:"text"`
- MrkdwnIn []string `json:"mrkdwn_in"`
+ Color string `json:"color"`
+ AuthorName string `json:"author_name"`
+ Text string `json:"text"`
+ MrkdwnIn []string `json:"mrkdwn_in"`
+ Fields []SlackField `json:"fields"`
+}
+
+type SlackField struct {
+ Title string `json:"title"`
+ Value string `json:"value"`
+ Short bool `json:"short"`
 }
 
 // NewSlack validates the Slack URL and returns a Slack object
@@ -60,7 +67,7 @@ func NewSlack(hookURL string, username string, channel string) (*Slack, error) {
 }
 
 // Post Slack message
-func (s *Slack) Post(workload string, namespace string, message string, warn bool) error {
+func (s *Slack) Post(workload string, namespace string, message string, fields []SlackField, warn bool) error {
  payload := SlackPayload{
  Channel: s.Channel,
  Username: s.Username,
@@ -76,6 +83,7 @@ func (s *Slack) Post(workload string, namespace string, message string, warn boo
  AuthorName: fmt.Sprintf("%s.%s", workload, namespace),
  Text: message,
  MrkdwnIn: []string{"text"},
+ Fields: fields,
  }
 
  payload.Attachments = []SlackAttachment{a}