diff --git a/.codecov.yml b/.codecov.yml index 2cdd00882..ec49f9c47 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -3,6 +3,6 @@ coverage: project: default: target: auto - threshold: 0.50 + threshold: 50 base: auto - patch: off \ No newline at end of file + patch: off diff --git a/README.md b/README.md index 3cb7a080e..59641d22e 100644 --- a/README.md +++ b/README.md @@ -367,7 +367,25 @@ helm upgrade -i flagger flagger/flagger \ Once configured with a Slack incoming webhook, Flagger will post messages when a canary deployment has been initialized, when a new revision has been detected and if the canary analysis failed or succeeded. -![flagger-slack](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-notifications.png) +![flagger-slack](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-canary-success.png) + +A canary deployment will be rolled back if the progress deadline exceeded or if the analysis +reached the maximum number of failed checks: + +![flagger-slack-errors](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-canary-failed.png) + +Besides Slack, you can use Alertmanager to trigger alerts when a canary deployment failed: + +```yaml + - alert: canary_rollback + expr: flagger_canary_status > 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Canary failed" + description: "Workload {{ $labels.name }} namespace {{ $labels.namespace }}" +``` ### Roadmap diff --git a/docs/screens/slack-canary-failed.png b/docs/screens/slack-canary-failed.png new file mode 100644 index 000000000..b150f2801 Binary files /dev/null and b/docs/screens/slack-canary-failed.png differ diff --git a/docs/screens/slack-canary-success.png b/docs/screens/slack-canary-success.png new file mode 100644 index 000000000..bdb332eff Binary files /dev/null and b/docs/screens/slack-canary-success.png differ diff --git a/docs/screens/slack-notifications.png b/docs/screens/slack-notifications.png deleted file mode 100644 index 98e2c83da..000000000 Binary files a/docs/screens/slack-notifications.png and /dev/null differ diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index b17e63051..96c9f28d4 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -260,12 +260,36 @@ func (c *Controller) recordEventWarningf(r *flaggerv1.Canary, template string, a c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...)) } -func (c *Controller) sendNotification(workload string, namespace string, message string, warn bool) { +func (c *Controller) sendNotification(cd *flaggerv1.Canary, message string, metadata bool, warn bool) { if c.notifier == nil { return } - err := c.notifier.Post(workload, namespace, message, warn) + var fields []notifier.SlackField + + if metadata { + fields = append(fields, + notifier.SlackField{ + Title: "Target", + Value: fmt.Sprintf("%s/%s.%s", cd.Spec.TargetRef.Kind, cd.Spec.TargetRef.Name, cd.Namespace), + }, + notifier.SlackField{ + Title: "Traffic routing", + Value: fmt.Sprintf("Weight step: %v max: %v", + cd.Spec.CanaryAnalysis.StepWeight, + cd.Spec.CanaryAnalysis.MaxWeight), + }, + notifier.SlackField{ + Title: "Failed checks threshold", + Value: fmt.Sprintf("%v", cd.Spec.CanaryAnalysis.Threshold), + }, + notifier.SlackField{ + Title: "Progress deadline", + Value: fmt.Sprintf("%vs", cd.GetProgressDeadlineSeconds()), + }, + ) + } + err := c.notifier.Post(cd.Name, cd.Namespace, message, fields, warn) if err != nil { c.logger.Error(err) } diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index f2f2a6096..bc1d57f7f 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -95,11 +95,15 @@ func (c *Controller) advanceCanary(name string, namespace string) { if cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold { c.recordEventWarningf(cd, "Rolling back %s.%s failed checks threshold reached %v", cd.Name, cd.Namespace, cd.Status.FailedChecks) + c.sendNotification(cd, fmt.Sprintf("Failed checks threshold reached %v", cd.Status.FailedChecks), + false, true) } if !retriable { c.recordEventWarningf(cd, "Rolling back %s.%s progress deadline exceeded %v", cd.Name, cd.Namespace, err) + c.sendNotification(cd, fmt.Sprintf("Progress deadline exceeded %v", err), + false, true) } // route all traffic back to primary @@ -112,7 +116,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight) c.recordEventWarningf(cd, "Canary failed! Scaling down %s.%s", - cd.Spec.TargetRef.Name, cd.Namespace) + cd.Name, cd.Namespace) // shutdown canary if err := c.deployer.Scale(cd, 0); err != nil { @@ -127,8 +131,6 @@ func (c *Controller) advanceCanary(name string, namespace string) { } c.recorder.SetStatus(cd) - c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace, - "Canary analysis failed, rollback finished.", true) return } @@ -199,8 +201,8 @@ func (c *Controller) advanceCanary(name string, namespace string) { return } c.recorder.SetStatus(cd) - c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace, - "Canary analysis completed successfully, promotion finished.", false) + c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.", + false, false) } } @@ -217,15 +219,15 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl } c.recorder.SetStatus(cd) c.recordEventInfof(cd, "Initialization done! %s.%s", cd.Name, cd.Namespace) - c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace, - "New deployment detected, initialization completed.", false) + c.sendNotification(cd, "New deployment detected, initialization completed.", + true, false) return false } if diff, err := deployer.IsNewSpec(cd); diff { c.recordEventInfof(cd, "New revision detected! Scaling up %s.%s", cd.Spec.TargetRef.Name, cd.Namespace) - c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace, - "New revision detected, starting canary analysis.", false) + c.sendNotification(cd, "New revision detected, starting canary analysis.", + true, false) if err = deployer.Scale(cd, 1); err != nil { c.recordEventErrorf(cd, "%v", err) return false diff --git a/pkg/notifier/slack.go b/pkg/notifier/slack.go index 389c6b18a..a97fc0adf 100644 --- a/pkg/notifier/slack.go +++ b/pkg/notifier/slack.go @@ -30,10 +30,17 @@ type SlackPayload struct { // SlackAttachment holds the markdown message body type SlackAttachment struct { - Color string `json:"color"` - AuthorName string `json:"author_name"` - Text string `json:"text"` - MrkdwnIn []string `json:"mrkdwn_in"` + Color string `json:"color"` + AuthorName string `json:"author_name"` + Text string `json:"text"` + MrkdwnIn []string `json:"mrkdwn_in"` + Fields []SlackField `json:"fields"` +} + +type SlackField struct { + Title string `json:"title"` + Value string `json:"value"` + Short bool `json:"short"` } // NewSlack validates the Slack URL and returns a Slack object @@ -60,7 +67,7 @@ func NewSlack(hookURL string, username string, channel string) (*Slack, error) { } // Post Slack message -func (s *Slack) Post(workload string, namespace string, message string, warn bool) error { +func (s *Slack) Post(workload string, namespace string, message string, fields []SlackField, warn bool) error { payload := SlackPayload{ Channel: s.Channel, Username: s.Username, @@ -76,6 +83,7 @@ func (s *Slack) Post(workload string, namespace string, message string, warn boo AuthorName: fmt.Sprintf("%s.%s", workload, namespace), Text: message, MrkdwnIn: []string{"text"}, + Fields: fields, } payload.Attachments = []SlackAttachment{a}