diff --git a/go.mod b/go.mod index 91b3b203916..4061a2565a0 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.0.0-00010101000000-000000000000 github.com/prometheus/alertmanager v0.20.0 github.com/prometheus/client_golang v1.6.0 + github.com/prometheus/common v0.10.0 github.com/prometheus/prometheus v1.8.2-0.20200609102542-5d7e3e970602 github.com/stretchr/testify v1.5.1 github.com/thanos-io/thanos v0.11.0 diff --git a/go.sum b/go.sum index 9719cd661ec..7a0cf22cb3f 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,7 @@ github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRY github.com/Azure/go-autorest/autorest/validation v0.2.1-0.20191028180845-3492b2aff503/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -1247,6 +1248,7 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A= diff --git a/pkg/alertmanager/amcfg.go b/pkg/alertmanager/amcfg.go new file mode 100644 index 00000000000..45c7c8d5d97 --- /dev/null +++ b/pkg/alertmanager/amcfg.go @@ -0,0 +1,412 @@ +// Copyright 2020 The prometheus-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alertmanager + +import ( + "fmt" + "sort" + "strings" + + "github.com/go-kit/kit/log" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/prometheus/alertmanager/config" + commoncfg "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +// Customization of Config type from alertmanager repo: +// https://github.com/prometheus/alertmanager/blob/master/config/config.go +// +// Custom global type to get around obfuscation of secret values when +// marshalling. See the following issue for details: +// https://github.com/prometheus/alertmanager/issues/1985 +type alertmanagerConfig struct { + Global *globalConfig `yaml:"global,omitempty" json:"global,omitempty"` + Route *route `yaml:"route,omitempty" json:"route,omitempty"` + InhibitRules []*inhibitRule `yaml:"inhibit_rules,omitempty" json:"inhibit_rules,omitempty"` + Receivers []*receiver `yaml:"receivers,omitempty" json:"receivers,omitempty"` + Templates []string `yaml:"templates" json:"templates"` +} + +type globalConfig struct { + // ResolveTimeout is the time after which an alert is declared resolved + // if it has not been updated. + ResolveTimeout model.Duration `yaml:"resolve_timeout" json:"resolve_timeout"` + + HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + + SMTPFrom string `yaml:"smtp_from,omitempty" json:"smtp_from,omitempty"` + SMTPHello string `yaml:"smtp_hello,omitempty" json:"smtp_hello,omitempty"` + SMTPSmarthost config.HostPort `yaml:"smtp_smarthost,omitempty" json:"smtp_smarthost,omitempty"` + SMTPAuthUsername string `yaml:"smtp_auth_username,omitempty" json:"smtp_auth_username,omitempty"` + SMTPAuthPassword string `yaml:"smtp_auth_password,omitempty" json:"smtp_auth_password,omitempty"` + SMTPAuthSecret string `yaml:"smtp_auth_secret,omitempty" json:"smtp_auth_secret,omitempty"` + SMTPAuthIdentity string `yaml:"smtp_auth_identity,omitempty" json:"smtp_auth_identity,omitempty"` + SMTPRequireTLS bool `yaml:"smtp_require_tls,omitempty" json:"smtp_require_tls,omitempty"` + SlackAPIURL *config.URL `yaml:"slack_api_url,omitempty" json:"slack_api_url,omitempty"` + PagerdutyURL *config.URL `yaml:"pagerduty_url,omitempty" json:"pagerduty_url,omitempty"` + HipchatAPIURL *config.URL `yaml:"hipchat_api_url,omitempty" json:"hipchat_api_url,omitempty"` + HipchatAuthToken string `yaml:"hipchat_auth_token,omitempty" json:"hipchat_auth_token,omitempty"` + OpsGenieAPIURL *config.URL `yaml:"opsgenie_api_url,omitempty" json:"opsgenie_api_url,omitempty"` + OpsGenieAPIKey string `yaml:"opsgenie_api_key,omitempty" json:"opsgenie_api_key,omitempty"` + WeChatAPIURL *config.URL `yaml:"wechat_api_url,omitempty" json:"wechat_api_url,omitempty"` + WeChatAPISecret string `yaml:"wechat_api_secret,omitempty" json:"wechat_api_secret,omitempty"` + WeChatAPICorpID string `yaml:"wechat_api_corp_id,omitempty" json:"wechat_api_corp_id,omitempty"` + VictorOpsAPIURL *config.URL `yaml:"victorops_api_url,omitempty" json:"victorops_api_url,omitempty"` + VictorOpsAPIKey string `yaml:"victorops_api_key,omitempty" json:"victorops_api_key,omitempty"` +} + +type route struct { + Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty"` + GroupByStr []string `yaml:"group_by,omitempty" json:"group_by,omitempty"` + Match map[string]string `yaml:"match,omitempty" json:"match,omitempty"` + MatchRE map[string]string `yaml:"match_re,omitempty" json:"match_re,omitempty"` + Continue bool `yaml:"continue,omitempty" json:"continue,omitempty"` + Routes []*route `yaml:"routes,omitempty" json:"routes,omitempty"` + GroupWait string `yaml:"group_wait,omitempty" json:"group_wait,omitempty"` + GroupInterval string `yaml:"group_interval,omitempty" json:"group_interval,omitempty"` + RepeatInterval string `yaml:"repeat_interval,omitempty" json:"repeat_interval,omitempty"` +} + +type inhibitRule struct { + TargetMatch map[string]string `yaml:"target_match,omitempty" json:"target_match,omitempty"` + TargetMatchRE map[string]string `yaml:"target_match_re,omitempty" json:"target_match_re,omitempty"` + SourceMatch map[string]string `yaml:"source_match,omitempty" json:"source_match,omitempty"` + SourceMatchRE map[string]string `yaml:"source_match_re,omitempty" json:"source_match_re,omitempty"` + Equal []string `yaml:"equal,omitempty" json:"equal,omitempty"` +} + +type receiver struct { + Name string `yaml:"name" json:"name"` + // EmailConfigs []*EmailConfig `yaml:"email_configs,omitempty" json:"email_configs,omitempty"` + PagerdutyConfigs []*pagerdutyConfig `yaml:"pagerduty_configs,omitempty" json:"pagerduty_configs,omitempty"` + // SlackConfigs []*SlackConfig `yaml:"slack_configs,omitempty" json:"slack_configs,omitempty"` + // WebhookConfigs []*WebhookConfig `yaml:"webhook_configs,omitempty" json:"webhook_configs,omitempty"` + // OpsGenieConfigs []*OpsGenieConfig `yaml:"opsgenie_configs,omitempty" json:"opsgenie_configs,omitempty"` + // WechatConfigs []*WechatConfig `yaml:"wechat_configs,omitempty" json:"wechat_configs,omitempty"` + // PushoverConfigs []*PushoverConfig `yaml:"pushover_configs,omitempty" json:"pushover_configs,omitempty"` + // VictorOpsConfigs []*VictorOpsConfig `yaml:"victorops_configs,omitempty" json:"victorops_configs,omitempty"` +} + +type pagerdutyConfig struct { + VSendResolved bool `yaml:"send_resolved" json:"send_resolved"` + HTTPConfig *httpClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"` + ServiceKey string `yaml:"service_key,omitempty" json:"service_key,omitempty"` + RoutingKey string `yaml:"routing_key,omitempty" json:"routing_key,omitempty"` + URL string `yaml:"url,omitempty" json:"url,omitempty"` + Client string `yaml:"client,omitempty" json:"client,omitempty"` + ClientURL string `yaml:"client_url,omitempty" json:"client_url,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Details map[string]string `yaml:"details,omitempty" json:"details,omitempty"` + Images []pagerdutyImage `yaml:"images,omitempty" json:"images,omitempty"` + Links []pagerdutyLink `yaml:"links,omitempty" json:"links,omitempty"` + Severity string `yaml:"severity,omitempty" json:"severity,omitempty"` + Class string `yaml:"class,omitempty" json:"class,omitempty"` + Component string `yaml:"component,omitempty" json:"component,omitempty"` + Group string `yaml:"group,omitempty" json:"group,omitempty"` +} + +type httpClientConfig struct { + BasicAuth *basicAuth `yaml:"basic_auth,omitempty"` + BearerToken string `yaml:"bearer_token,omitempty"` + BearerTokenFile string `yaml:"bearer_token_file,omitempty"` + ProxyURL string `yaml:"proxy_url,omitempty"` + TLSConfig commoncfg.TLSConfig `yaml:"tls_config,omitempty"` +} + +type basicAuth struct { + Username string `yaml:"username"` + Password string `yaml:"password,omitempty"` + PasswordFile string `yaml:"password_file,omitempty"` +} + +type pagerdutyLink struct { + Href string `yaml:"href,omitempty" json:"href,omitempty"` + Text string `yaml:"text,omitempty" json:"text,omitempty"` +} + +type pagerdutyImage struct { + Src string `yaml:"src,omitempty" json:"src,omitempty"` + Alt string `yaml:"alt,omitempty" json:"alt,omitempty"` + Href string `yaml:"href,omitempty" json:"href,omitempty"` +} + +func loadCfg(s string) (*alertmanagerConfig, error) { + // Run upstream Load function to get any validation checks that it runs. + _, err := config.Load(s) + if err != nil { + return nil, err + } + + cfg := &alertmanagerConfig{} + err = yaml.UnmarshalStrict([]byte(s), cfg) + + return cfg, nil +} + +func (c alertmanagerConfig) String() string { + b, err := yaml.Marshal(c) + if err != nil { + return fmt.Sprintf("", err) + } + return string(b) +} + +func prefixRouteReceivers(r *monitoringv1.Route, prefix string) { + if !strings.HasPrefix(r.Receiver, prefix) { + r.Receiver = prefix + r.Receiver + } + + if r.Routes != nil { + for i := range r.Routes { + prefixRouteReceivers(&r.Routes[i], prefix) + } + } +} + +type configGenerator struct { + logger log.Logger + kclient kubernetes.Interface + nsSecretCache map[string]*corev1.Secret +} + +func newConfigGenerator(logger log.Logger, kclient kubernetes.Interface) *configGenerator { + cg := &configGenerator{ + logger: logger, + kclient: kclient, + nsSecretCache: make(map[string]*corev1.Secret), + } + return cg +} + +func (cg *configGenerator) generateConfig( + baseConfig alertmanagerConfig, + amConfigs map[string]*monitoringv1.AlertmanagerConfig, +) ([]byte, error) { + + // amConfigIdentifiers is a sorted slice of keys from + // amConfigs map, used to always generate the config in the + // same order + amConfigIdentifiers := make([]string, len(amConfigs)) + i := 0 + for k := range amConfigs { + amConfigIdentifiers[i] = k + i++ + } + sort.Strings(amConfigIdentifiers) + + subRoutes := []*route{} + for _, amConfigIdentifier := range amConfigIdentifiers { + crKey := types.NamespacedName{ + Name: amConfigs[amConfigIdentifier].Name, + Namespace: amConfigs[amConfigIdentifier].Namespace, + } + + // add routes to subRoutes + subRoutes = append(subRoutes, convertRoute(amConfigs[amConfigIdentifier].Spec.Route, crKey, true)) + + // add receivers to baseConfig.Receivers + for _, receiver := range amConfigs[amConfigIdentifier].Spec.Receivers { + baseConfig.Receivers = append(baseConfig.Receivers, cg.convertReceiver(&receiver, crKey)) + } + + // add inhibitRules to baseConfig.InhibitRules + for _, inhibitRule := range amConfigs[amConfigIdentifier].Spec.InhibitRules { + baseConfig.InhibitRules = append(baseConfig.InhibitRules, convertInhibitRule(&inhibitRule)) + } + } + + // Append subroutes from base to the end, then replace with the new slice + subRoutes = append(subRoutes, baseConfig.Route.Routes...) + baseConfig.Route.Routes = subRoutes + + return yaml.Marshal(baseConfig) +} + +func convertRoute(in *monitoringv1.Route, crKey types.NamespacedName, firstLevelRoute bool) *route { + + // Enforce continue to be true for main Route in a CR + cont := in.Continue + if firstLevelRoute { + cont = true + } + + match := map[string]string{} + matchRE := map[string]string{} + if firstLevelRoute { + match["namespace"] = crKey.Namespace + } else { + for _, matcher := range in.Matchers { + if *matcher.Regex { + matchRE[matcher.Name] = matcher.Value + } else { + match[matcher.Name] = matcher.Value + } + } + } + if len(match) == 0 { + match = nil + } + if len(matchRE) == 0 { + matchRE = nil + } + + var routes []*route = nil + if len(in.Routes) > 0 { + routes := make([]*route, len(in.Routes)) + for i := range in.Routes { + routes[i] = convertRoute(&in.Routes[i], crKey, true) + } + } + + receiver := prefixReceiverName(in.Receiver, crKey) + + return &route{ + Receiver: receiver, + GroupByStr: in.GroupBy, + GroupWait: in.GroupWait, + GroupInterval: in.GroupInterval, + RepeatInterval: in.RepeatInterval, + Continue: cont, + Match: match, + MatchRE: matchRE, + Routes: routes, + } +} + +func (cg *configGenerator) convertReceiver(in *monitoringv1.Receiver, crKey types.NamespacedName) *receiver { + + var pagerdutyConfigs []*pagerdutyConfig + if l := len(in.PagerDutyConfigs); l > 0 { + pagerdutyConfigs = make([]*pagerdutyConfig, l) + for i := range in.PagerDutyConfigs { + pagerdutyConfigs[i] = cg.convertPagerdutyConfig(in.PagerDutyConfigs[i], crKey) + } + } + + return &receiver{ + Name: prefixReceiverName(in.Name, crKey), + PagerdutyConfigs: pagerdutyConfigs, + } +} + +func (cg *configGenerator) convertPagerdutyConfig(in monitoringv1.PagerDutyConfig, crKey types.NamespacedName) *pagerdutyConfig { + + out := &pagerdutyConfig{} + + if in.SendResolved != nil { + out.VSendResolved = *in.SendResolved + } + + //TODO routingkey and servicekey should be secretkeyselectors + if in.RoutingKey != nil { + out.RoutingKey = *in.RoutingKey + } + + if in.ServiceKey != nil { + out.ServiceKey = *in.ServiceKey + } + + if in.URL != nil { + out.URL = *in.URL + } + + if in.Client != nil { + out.Client = *in.Client + } + + if in.ClientURL != nil { + out.ClientURL = *in.ClientURL + } + + if in.Description != nil { + out.Description = *in.Description + } + + if in.Severity != nil { + out.Severity = *in.Severity + } + + var details map[string]string + if l := len(in.Details); l > 0 { + details = make(map[string]string, l) + for _, d := range in.Details { + details[d.Key] = d.Value + } + } + out.Details = details + + //TODO: convert HTTPConfig + + return out +} + +func convertInhibitRule(in *monitoringv1.InhibitRule) *inhibitRule { + sourceMatch := map[string]string{} + sourceMatchRE := map[string]string{} + for _, sm := range in.SourceMatch { + if sm.Regex != nil && *sm.Regex { + sourceMatchRE[sm.Name] = sm.Value + } else { + sourceMatch[sm.Name] = sm.Value + } + } + if len(sourceMatch) == 0 { + sourceMatch = nil + } + if len(sourceMatchRE) == 0 { + sourceMatchRE = nil + } + + targetMatch := map[string]string{} + targetMatchRE := map[string]string{} + for _, tm := range in.TargetMatch { + if tm.Regex != nil && *tm.Regex { + targetMatchRE[tm.Name] = tm.Value + } else { + targetMatch[tm.Name] = tm.Value + } + } + if len(targetMatch) == 0 { + targetMatch = nil + } + if len(targetMatchRE) == 0 { + targetMatchRE = nil + } + + equal := in.Equal + if len(equal) == 0 { + equal = nil + } + + return &inhibitRule{ + SourceMatch: sourceMatch, + SourceMatchRE: sourceMatchRE, + TargetMatch: targetMatch, + TargetMatchRE: targetMatchRE, + Equal: equal, + } +} + +func prefixReceiverName(receiverName string, crKey types.NamespacedName) string { + if receiverName == "" { + return "" + } + return crKey.Namespace + "-" + crKey.Name + "-" + receiverName +} diff --git a/pkg/alertmanager/operator.go b/pkg/alertmanager/operator.go index c3577894eb7..388d0816ccf 100644 --- a/pkg/alertmanager/operator.go +++ b/pkg/alertmanager/operator.go @@ -28,12 +28,10 @@ import ( "github.com/prometheus-operator/prometheus-operator/pkg/listwatch" "github.com/prometheus-operator/prometheus-operator/pkg/operator" prometheusoperator "github.com/prometheus-operator/prometheus-operator/pkg/prometheus" - "gopkg.in/yaml.v2" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" - "github.com/prometheus/alertmanager/config" "github.com/prometheus/client_golang/prometheus" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -44,6 +42,7 @@ import ( "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" ) @@ -652,54 +651,28 @@ func (c *Operator) provisionAlertmanagerConfiguration(ctx context.Context, am *m return errors.Wrap(err, "get base configuration secret") } - var baseConfigYaml yaml.MapSlice - baseConfig, _ := secret.Data["alertmanager.yaml"] - _, err = config.Load(string(baseConfig)) + rawBaseConfig, _ := secret.Data["alertmanager.yaml"] + baseConfig, err := loadCfg(string(rawBaseConfig)) if err != nil { level.Warn(c.logger).Log("msg", "base configuration loaded from Secret could not be parsed", "secret", secretName) - baseConfig = []byte{} - } - - err = yaml.Unmarshal(baseConfig, &baseConfigYaml) - if err != nil { - return errors.Wrap(err, "unmarshalling base config") - } - - foundRoute := false - foundReceivers := false - foundInhibitRules := false - for _, i := range baseConfigYaml { - if k, ok := i.Key.(string); ok { - if k == "route" { - foundRoute = true - } - if k == "receivers" { - foundReceivers = true - } - if k == "route" { - foundInhibitRules = true - } - } + rawBaseConfig = []byte{} } + foundRoute := baseConfig.Route != nil if !foundRoute { return errors.New("default route must be configured") } - amcs, err := c.selectAlertManagerConfigs(am) + amConfigs, err := c.selectAlertManagerConfigs(am) if err != nil { return errors.Wrap(err, "selecting AlertmanagerConfigs failed") } - level.Info(c.logger).Log("msg", "amcs", "amcs", amcs, "namespace", am.Namespace, "alertmanager", am.Name) - - if foundReceivers { - // just append receivrs from AlertmanagerConfig CRs, otherwise must first create receivers field - } - - if foundInhibitRules { - // just append inhibition rules from AlertmanagerConfig CRs, otherwise must first create inhibit rules field + conf, err := newConfigGenerator(c.logger, c.kclient).generateConfig(*baseConfig, amConfigs) + if err != nil { + return errors.Wrap(err, "generating Alertmanager config yaml failed") } + fmt.Printf("%v\n", conf) return nil }