feat: make silence feature optional (#41)

goto · Jan 29, 2024 · 25f1775 · 25f1775
1 parent 4447a08
commit 25f1775
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 29 deletions.
diff --git a/cli/deps.go b/cli/deps.go
@@ -187,6 +187,7 @@ func InitDeps(
 			SilenceService:        silenceService,
 			AlertService:          alertService,
 		},
+		cfg.Service.EnableSilenceFeature,
 	)
 
 	return &api.Deps{

diff --git a/core/notification/dispatch_subscriber_service.go b/core/notification/dispatch_subscriber_service.go
@@ -12,22 +12,26 @@ import (
 )
 
 type DispatchSubscriberService struct {
-	logger              saltlog.Logger
-	subscriptionService SubscriptionService
-	silenceService      SilenceService
-	notifierPlugins     map[string]Notifier
+	logger               saltlog.Logger
+	subscriptionService  SubscriptionService
+	silenceService       SilenceService
+	notifierPlugins      map[string]Notifier
+	enableSilenceFeature bool
 }
 
 func NewDispatchSubscriberService(
 	logger saltlog.Logger,
 	subscriptionService SubscriptionService,
 	silenceService SilenceService,
-	notifierPlugins map[string]Notifier) *DispatchSubscriberService {
+	notifierPlugins map[string]Notifier,
+	enableSilenceFeature bool,
+) *DispatchSubscriberService {
 	return &DispatchSubscriberService{
-		logger:              logger,
-		subscriptionService: subscriptionService,
-		silenceService:      silenceService,
-		notifierPlugins:     notifierPlugins,
+		logger:               logger,
+		subscriptionService:  subscriptionService,
+		silenceService:       silenceService,
+		notifierPlugins:      notifierPlugins,
+		enableSilenceFeature: enableSilenceFeature,
 	}
 }
 
@@ -64,13 +68,18 @@ func (s *DispatchSubscriberService) PrepareMessage(ctx context.Context, n Notifi
 			continue
 		}
 
-		// try silencing by labels
-		silences, err := s.silenceService.List(ctx, silence.Filter{
-			NamespaceID:       n.NamespaceID,
-			SubscriptionMatch: sub.Match,
-		})
-		if err != nil {
-			return nil, nil, false, err
+		var silences []silence.Silence
+
+		// Reliability of silence feature need to be tested more
+		if s.enableSilenceFeature {
+			// try silencing by labels
+			silences, err = s.silenceService.List(ctx, silence.Filter{
+				NamespaceID:       n.NamespaceID,
+				SubscriptionMatch: sub.Match,
+			})
+			if err != nil {
+				return nil, nil, false, err
+			}
 		}
 
 		if len(silences) != 0 {
@@ -93,13 +102,16 @@ func (s *DispatchSubscriberService) PrepareMessage(ctx context.Context, n Notifi
 			continue
 		}
 
-		// subscription not being silenced by label
-		silences, err = s.silenceService.List(ctx, silence.Filter{
-			NamespaceID:    n.NamespaceID,
-			SubscriptionID: sub.ID,
-		})
-		if err != nil {
-			return nil, nil, false, err
+		// Reliability of silence feature need to be tested more
+		if s.enableSilenceFeature {
+			// subscription not being silenced by label
+			silences, err = s.silenceService.List(ctx, silence.Filter{
+				NamespaceID:    n.NamespaceID,
+				SubscriptionID: sub.ID,
+			})
+			if err != nil {
+				return nil, nil, false, err
+			}
 		}
 
 		silencedReceiversMap, validReceivers, err := sub.SilenceReceivers(silences)

diff --git a/core/notification/dispatch_subscriber_service_test.go b/core/notification/dispatch_subscriber_service_test.go
@@ -315,7 +315,9 @@ func TestDispatchSubscriberService_PrepareMessage(t *testing.T) {
 				mockSubscriptionService,
 				mockSilenceService, map[string]notification.Notifier{
 					testPluginType: mockNotifier,
-				})
+				},
+				true,
+			)
 
 			if tt.setup != nil {
 				tt.setup(mockSubscriptionService, mockSilenceService, mockNotifier)

diff --git a/core/notification/service.go b/core/notification/service.go
@@ -63,6 +63,7 @@ type Service struct {
 	notifierPlugins       map[string]Notifier
 	dispatcher            map[string]Dispatcher
 	messagingTracer       *telemetry.MessagingTracer
+	enableSilenceFeature  bool
 }
 
 type Deps struct {
@@ -84,6 +85,7 @@ func NewService(
 	q Queuer,
 	notifierPlugins map[string]Notifier,
 	deps Deps,
+	enableSilenceFeature bool,
 ) *Service {
 	var (
 		dispatchReceiverService   = deps.DispatchReceiverService
@@ -93,7 +95,7 @@ func NewService(
 		dispatchReceiverService = NewDispatchReceiverService(deps.ReceiverService, notifierPlugins)
 	}
 	if deps.DispatchSubscriberService == nil {
-		dispatchSubscriberService = NewDispatchSubscriberService(logger, deps.SubscriptionService, deps.SilenceService, notifierPlugins)
+		dispatchSubscriberService = NewDispatchSubscriberService(logger, deps.SubscriptionService, deps.SilenceService, notifierPlugins, enableSilenceFeature)
 	}
 
 	ns := &Service{
@@ -111,7 +113,8 @@ func NewService(
 			TypeReceiver:   dispatchReceiverService,
 			TypeSubscriber: dispatchSubscriberService,
 		},
-		notifierPlugins: notifierPlugins,
+		notifierPlugins:      notifierPlugins,
+		enableSilenceFeature: enableSilenceFeature,
 	}
 
 	ns.messagingTracer = telemetry.NewMessagingTracer("default")
@@ -165,8 +168,11 @@ func (s *Service) Dispatch(ctx context.Context, n Notification) error {
 		return fmt.Errorf("failed logging notifications: %w", err)
 	}
 
-	if err := s.alertService.UpdateSilenceStatus(ctx, n.AlertIDs, hasSilenced, len(messages) != 0); err != nil {
-		return fmt.Errorf("failed updating silence status: %w", err)
+	// Reliability of silence feature need to be tested more
+	if s.enableSilenceFeature {
+		if err := s.alertService.UpdateSilenceStatus(ctx, n.AlertIDs, hasSilenced, len(messages) != 0); err != nil {
+			return fmt.Errorf("failed updating silence status: %w", err)
+		}
 	}
 
 	if len(messages) == 0 {

diff --git a/core/notification/service_test.go b/core/notification/service_test.go
@@ -70,7 +70,7 @@ func TestService_CheckAndInsertIdempotency(t *testing.T) {
 				tc.setup(mockIdempotencyRepository)
 			}
 
-			ns := notification.NewService(saltlog.NewNoop(), notification.Config{}, nil, nil, nil, notification.Deps{IdempotencyRepository: mockIdempotencyRepository})
+			ns := notification.NewService(saltlog.NewNoop(), notification.Config{}, nil, nil, nil, notification.Deps{IdempotencyRepository: mockIdempotencyRepository}, false)
 
 			_, err := ns.CheckAndInsertIdempotency(context.TODO(), tc.scope, tc.key)
 
@@ -228,6 +228,7 @@ func TestService_Dispatch(t *testing.T) {
 					DispatchReceiverService:   mockDispatcher,
 					DispatchSubscriberService: mockDispatcher,
 				},
+				true,
 			)
 			if err := s.Dispatch(context.TODO(), tt.n); (err != nil) != tt.wantErr {
 				t.Errorf("Service.Dispatch() error = %v, wantErr %v", err, tt.wantErr)
@@ -341,6 +342,7 @@ func TestService_BuildFromAlerts(t *testing.T) {
 				nil,
 				nil,
 				notification.Deps{},
+				false,
 			)
 			got, err := s.BuildFromAlerts(tt.alerts, tt.firingLen, time.Time{})
 			if (err != nil) && (err.Error() != tt.errString) {

diff --git a/internal/server/server.go b/internal/server/server.go
@@ -44,6 +44,7 @@ type Config struct {
 	EncryptionKey         string            `mapstructure:"encryption_key" yaml:"encryption_key" default:"_ENCRYPTIONKEY_OF_32_CHARACTERS_"`
 	APIHeaders            api.HeadersConfig `mapstructure:"api_headers" yaml:"api_headers"`
 	UseGlobalSubscription bool              `mapstructure:"use_global_subscription" yaml:"use_global_subscription" default:"false"`
+	EnableSilenceFeature  bool              `mapstructure:"enable_silence_feature" yaml:"enable_silence_feature" default:"false"`
 	DebugRequest          bool              `mapstructure:"debug_request" yaml:"debug_request" default:"false"`
 	GRPC                  GRPCConfig        `mapstructure:"grpc"`
 }

diff --git a/test/e2e_test/cortex_webhook_test.go b/test/e2e_test/cortex_webhook_test.go
@@ -70,6 +70,7 @@ func (s *CortexWebhookTestSuite) SetupTest() {
 	s.appConfig.Telemetry.Debug = ""
 	s.appConfig.Telemetry.EnableNewrelic = false
 	s.appConfig.Telemetry.EnableOtelAgent = false
+	s.appConfig.Service.EnableSilenceFeature = true
 
 	s.testBench, err = InitCortexEnvironment(s.appConfig)
 	s.Require().NoError(err)