From a455dd6efb785d89f7e59b07b6471e76341608ef Mon Sep 17 00:00:00 2001 From: Muhammad Abduh Date: Thu, 29 Sep 2022 16:16:39 +0700 Subject: [PATCH] feat(notification): add postgres queue --- cli/deps.go | 148 +++++++++ cli/job.go | 108 +++++++ cli/root.go | 2 + cli/server.go | 163 +++------- cli/worker.go | 210 +++++++++++++ config/config.go | 19 +- config/config.yaml | 14 +- core/namespace/crypto.go | 6 +- core/namespace/mocks/encryptor.go | 42 +-- core/namespace/namespace.go | 2 +- core/namespace/service.go | 9 +- core/namespace/service_test.go | 95 +++--- core/notification/config.go | 20 ++ core/notification/handler.go | 93 +++--- core/notification/handler_option.go | 18 +- core/notification/handler_test.go | 40 ++- core/notification/message.go | 8 + core/notification/message_test.go | 6 +- core/notification/mocks/notifier.go | 127 +++++--- core/notification/mocks/queuer.go | 76 +++-- core/notification/notification.go | 9 +- core/notification/routing.go | 12 + core/notification/service.go | 37 ++- core/notification/service_test.go | 40 ++- core/provider/service_test.go | 18 +- core/subscription/mocks/receiver_service.go | 39 --- core/subscription/service.go | 2 + internal/api/v1beta1/alert.go | 10 +- internal/api/v1beta1/alert_test.go | 30 +- internal/server/server.go | 5 +- internal/store/model/namespace.go | 45 +-- internal/store/model/provider.go | 64 +--- internal/store/model/receiver.go | 19 +- internal/store/model/subscription.go | 15 +- ...create_index_subscription_matcher.down.sql | 1 + ...2_create_index_subscription_matcher.up.sql | 1 + internal/store/postgres/namespace.go | 4 +- internal/store/postgres/namespace_test.go | 24 +- internal/store/postgres/postgres_test.go | 18 +- internal/store/postgres/subscription.go | 18 +- pkg/pgtype/map.go | 49 +++ pkg/secret/secret.go | 10 +- pkg/secret/string.go | 13 + pkg/secret/string_test.go | 25 ++ pkg/telemetry/telemetry.go | 6 +- pkg/worker/option.go | 13 + pkg/worker/ticker.go | 64 ++++ plugins/providers/cortex/config.go | 2 +- plugins/queues/config.go | 21 ++ plugins/queues/inmemory/queue.go | 18 +- plugins/queues/postgresq/cleanup.go | 76 +++++ ...000001_create_message_queue_table.down.sql | 1 + .../000001_create_message_queue_table.up.sql | 14 + .../queues/postgresq/migrations/migrations.go | 8 + plugins/queues/postgresq/model.go | 76 +++++ plugins/queues/postgresq/option.go | 9 + plugins/queues/postgresq/queue.go | 233 +++++++++++++++ plugins/queues/postgresq/queue_test.go | 281 ++++++++++++++++++ plugins/receivers/config.go | 6 +- .../httpreceiver/notification_service.go | 8 + plugins/receivers/pagerduty/config.go | 3 +- ...fault_cortex_alert_template_body_v1.goyaml | 14 +- plugins/receivers/pagerduty/config_test.go | 5 +- plugins/receivers/pagerduty/messagev1.go | 4 +- .../pagerduty/notification_service.go | 8 + .../pagerduty/receiver_service_test.go | 5 +- plugins/receivers/slack/client.go | 15 +- plugins/receivers/slack/client_test.go | 9 +- plugins/receivers/slack/config.go | 14 +- .../default_cortex_alert_template_body.goyaml | 18 +- plugins/receivers/slack/config_test.go | 5 +- plugins/receivers/slack/mocks/encryptor.go | 41 +-- plugins/receivers/slack/mocks/slack_caller.go | 16 +- .../receivers/slack/notification_service.go | 49 ++- .../slack/notification_service_test.go | 148 ++++++++- .../receivers/slack/receiver_service_test.go | 15 +- plugins/receivers/slack/slack.go | 12 +- test/e2e_test/cortex_rule_test.go | 4 +- test/e2e_test/cortex_subscription_test.go | 4 +- 79 files changed, 2310 insertions(+), 629 deletions(-) create mode 100644 cli/deps.go create mode 100644 cli/job.go create mode 100644 cli/worker.go create mode 100644 core/notification/config.go create mode 100644 core/notification/routing.go create mode 100644 internal/store/postgres/migrations/000002_create_index_subscription_matcher.down.sql create mode 100644 internal/store/postgres/migrations/000002_create_index_subscription_matcher.up.sql create mode 100644 pkg/pgtype/map.go create mode 100644 pkg/secret/string.go create mode 100644 pkg/secret/string_test.go create mode 100644 pkg/worker/option.go create mode 100644 pkg/worker/ticker.go create mode 100644 plugins/queues/config.go create mode 100644 plugins/queues/postgresq/cleanup.go create mode 100644 plugins/queues/postgresq/migrations/000001_create_message_queue_table.down.sql create mode 100644 plugins/queues/postgresq/migrations/000001_create_message_queue_table.up.sql create mode 100644 plugins/queues/postgresq/migrations/migrations.go create mode 100644 plugins/queues/postgresq/model.go create mode 100644 plugins/queues/postgresq/option.go create mode 100644 plugins/queues/postgresq/queue.go create mode 100644 plugins/queues/postgresq/queue_test.go diff --git a/cli/deps.go b/cli/deps.go new file mode 100644 index 00000000..76ffa541 --- /dev/null +++ b/cli/deps.go @@ -0,0 +1,148 @@ +package cli + +import ( + "fmt" + + "github.com/odpf/salt/log" + "github.com/odpf/siren/config" + "github.com/odpf/siren/core/alert" + "github.com/odpf/siren/core/namespace" + "github.com/odpf/siren/core/notification" + "github.com/odpf/siren/core/provider" + "github.com/odpf/siren/core/receiver" + "github.com/odpf/siren/core/rule" + "github.com/odpf/siren/core/subscription" + "github.com/odpf/siren/core/template" + "github.com/odpf/siren/internal/api" + "github.com/odpf/siren/internal/store/postgres" + "github.com/odpf/siren/pkg/httpclient" + "github.com/odpf/siren/pkg/retry" + "github.com/odpf/siren/pkg/secret" + "github.com/odpf/siren/plugins/providers/cortex" + "github.com/odpf/siren/plugins/receivers/httpreceiver" + "github.com/odpf/siren/plugins/receivers/pagerduty" + "github.com/odpf/siren/plugins/receivers/slack" +) + +type ReceiverClient struct { + SlackClient *slack.Client + PagerDutyClient *pagerduty.Client + HTTPReceiverClient *httpreceiver.Client +} + +type ProviderClient struct { + CortexClient *cortex.Client +} + +func InitAPIDeps( + logger log.Logger, + cfg config.Config, + pgClient *postgres.Client, + encryptor *secret.Crypto, + queue notification.Queuer, +) (*api.Deps, *ReceiverClient, *ProviderClient, map[string]notification.Notifier, error) { + templateRepository := postgres.NewTemplateRepository(pgClient) + templateService := template.NewService(templateRepository) + + alertRepository := postgres.NewAlertRepository(pgClient) + alertHistoryService := alert.NewService(alertRepository) + + providerRepository := postgres.NewProviderRepository(pgClient) + providerService := provider.NewService(providerRepository) + + namespaceRepository := postgres.NewNamespaceRepository(pgClient) + namespaceService := namespace.NewService(encryptor, namespaceRepository) + + cortexClient, err := cortex.NewClient(cortex.Config{Address: cfg.Cortex.Address}) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("failed to init cortex client: %w", err) + } + cortexProviderService := cortex.NewProviderService(cortexClient) + + ruleRepository := postgres.NewRuleRepository(pgClient) + ruleService := rule.NewService( + ruleRepository, + templateService, + namespaceService, + map[string]rule.RuleUploader{ + provider.TypeCortex: cortexProviderService, + }, + ) + + // plugin receiver services + slackHTTPClient := httpclient.New(cfg.Receivers.Slack.HTTPClient) + slackRetrier := retry.New(cfg.Receivers.Slack.Retry) + slackClient := slack.NewClient( + cfg.Receivers.Slack, + slack.ClientWithHTTPClient(slackHTTPClient), + slack.ClientWithRetrier(slackRetrier), + ) + pagerdutyHTTPClient := httpclient.New(cfg.Receivers.Pagerduty.HTTPClient) + pagerdutyRetrier := retry.New(cfg.Receivers.Slack.Retry) + pagerdutyClient := pagerduty.NewClient( + cfg.Receivers.Pagerduty, + pagerduty.ClientWithHTTPClient(pagerdutyHTTPClient), + pagerduty.ClientWithRetrier(pagerdutyRetrier), + ) + httpreceiverHTTPClient := httpclient.New(cfg.Receivers.HTTPReceiver.HTTPClient) + httpreceiverRetrier := retry.New(cfg.Receivers.Slack.Retry) + httpreceiverClient := httpreceiver.NewClient( + logger, + cfg.Receivers.HTTPReceiver, + httpreceiver.ClientWithHTTPClient(httpreceiverHTTPClient), + httpreceiver.ClientWithRetrier(httpreceiverRetrier), + ) + + slackReceiverService := slack.NewReceiverService(slackClient, encryptor) + httpReceiverService := httpreceiver.NewReceiverService() + pagerDutyReceiverService := pagerduty.NewReceiverService() + + receiverRepository := postgres.NewReceiverRepository(pgClient) + receiverService := receiver.NewService( + receiverRepository, + map[string]receiver.ConfigResolver{ + receiver.TypeSlack: slackReceiverService, + receiver.TypeHTTP: httpReceiverService, + receiver.TypePagerDuty: pagerDutyReceiverService, + }, + ) + + subscriptionRepository := postgres.NewSubscriptionRepository(pgClient) + subscriptionService := subscription.NewService( + subscriptionRepository, + namespaceService, + receiverService, + subscription.RegisterProviderPlugin(provider.TypeCortex, cortexProviderService), + ) + + // notification + slackNotificationService := slack.NewNotificationService(slackClient, encryptor) + pagerdutyNotificationService := pagerduty.NewNotificationService(pagerdutyClient) + httpreceiverNotificationService := httpreceiver.NewNotificationService(httpreceiverClient) + + notifierRegistry := map[string]notification.Notifier{ + receiver.TypeSlack: slackNotificationService, + receiver.TypePagerDuty: pagerdutyNotificationService, + receiver.TypeHTTP: httpreceiverNotificationService, + } + + notificationService := notification.NewService(logger, queue, receiverService, subscriptionService, notifierRegistry) + + return &api.Deps{ + TemplateService: templateService, + RuleService: ruleService, + AlertService: alertHistoryService, + ProviderService: providerService, + NamespaceService: namespaceService, + ReceiverService: receiverService, + SubscriptionService: subscriptionService, + NotificationService: notificationService, + }, &ReceiverClient{ + SlackClient: slackClient, + PagerDutyClient: pagerdutyClient, + HTTPReceiverClient: httpreceiverClient, + }, &ProviderClient{ + CortexClient: cortexClient, + }, notifierRegistry, + nil +} diff --git a/cli/job.go b/cli/job.go new file mode 100644 index 00000000..9bea5795 --- /dev/null +++ b/cli/job.go @@ -0,0 +1,108 @@ +package cli + +import ( + "fmt" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/cmdx" + "github.com/odpf/salt/log" + "github.com/odpf/salt/printer" + "github.com/odpf/siren/config" + "github.com/odpf/siren/core/notification" + "github.com/odpf/siren/plugins/queues" + "github.com/odpf/siren/plugins/queues/postgresq" + "github.com/spf13/cobra" +) + +func jobCmd(cmdxConfig *cmdx.Config) *cobra.Command { + cmd := &cobra.Command{ + Use: "job ", + Aliases: []string{"jobs"}, + Short: "Manage siren jobs", + Long: "Jobs management commands.", + Example: heredoc.Doc(` + $ siren job run cleanup_queue + `), + } + + cmd.AddCommand( + jobRunCommand(cmdxConfig), + ) + + return cmd +} + +func jobRunCommand(cmdxConfig *cmdx.Config) *cobra.Command { + cmd := &cobra.Command{ + Use: "run", + Short: "Trigger a job", + Long: heredoc.Doc(` + Trigger a job + `), + Args: cobra.ExactValidArgs(1), + ValidArgs: []string{ + "cleanup_queue", + }, + Example: heredoc.Doc(` + $ siren job run cleanup_queue + `), + } + + cmd.AddCommand( + jobRunCleanupQueueCommand(), + ) + + return cmd +} + +func jobRunCleanupQueueCommand() *cobra.Command { + var configFile string + + cmd := &cobra.Command{ + Use: "cleanup_queue", + Short: "Cleanup stale messages in queue", + Long: heredoc.Doc(` + Cleaning up all published messages in queue with last updated + more than specific threshold (default 7 days) from now() and + (Optional) cleaning up all pending messages in queue with last updated + more than specific threshold (default 7 days) from now(). + `), + Example: heredoc.Doc(` + $ siren job run cleanup_queue + `), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load(configFile) + if err != nil { + return err + } + + var queue notification.Queuer + switch cfg.Notification.Queue.Kind { + case queues.KindPostgres: + queue, err = postgresq.New(log.NewZap(), cfg.DB) + if err != nil { + return err + } + default: + printer.Info("Cleanup queue job only works for postgres queue") + return nil + } + + spinner := printer.Spin("") + defer spinner.Stop() + printer.Info("Running job cleanup_queue(%s)", cfg.Notification.Queue.Kind.String()) + if err := queue.Cleanup(cmd.Context(), queues.FilterCleanup{}); err != nil { + return err + } + spinner.Stop() + printer.Success(fmt.Sprintf("Job cleanup_queue(%s) finished", cfg.Notification.Queue.Kind.String())) + printer.Space() + printer.SuccessIcon() + + return nil + }, + } + + cmd.Flags().StringVarP(&configFile, "config", "c", "config.yaml", "Config file path") + return cmd +} diff --git a/cli/root.go b/cli/root.go index cef39b68..5d2b44cb 100644 --- a/cli/root.go +++ b/cli/root.go @@ -42,6 +42,8 @@ func New() *cobra.Command { rootCmd.AddCommand(templatesCmd(cmdxConfig)) rootCmd.AddCommand(rulesCmd(cmdxConfig)) rootCmd.AddCommand(alertsCmd(cmdxConfig)) + rootCmd.AddCommand(jobCmd(cmdxConfig)) + rootCmd.AddCommand(workerCmd()) // Help topics cmdx.SetHelp(rootCmd) diff --git a/cli/server.go b/cli/server.go index 72e02e72..bd14d3ba 100644 --- a/cli/server.go +++ b/cli/server.go @@ -11,27 +11,16 @@ import ( "github.com/odpf/salt/log" "github.com/odpf/salt/printer" "github.com/odpf/siren/config" - "github.com/odpf/siren/core/alert" - "github.com/odpf/siren/core/namespace" "github.com/odpf/siren/core/notification" - "github.com/odpf/siren/core/provider" - "github.com/odpf/siren/core/receiver" - "github.com/odpf/siren/core/rule" - "github.com/odpf/siren/core/subscription" - "github.com/odpf/siren/core/template" - "github.com/odpf/siren/internal/api" "github.com/odpf/siren/internal/server" "github.com/odpf/siren/internal/store/postgres" - "github.com/odpf/siren/pkg/httpclient" - "github.com/odpf/siren/pkg/retry" "github.com/odpf/siren/pkg/secret" "github.com/odpf/siren/pkg/telemetry" + "github.com/odpf/siren/pkg/worker" "github.com/odpf/siren/pkg/zaputil" - "github.com/odpf/siren/plugins/providers/cortex" + "github.com/odpf/siren/plugins/queues" "github.com/odpf/siren/plugins/queues/inmemory" - "github.com/odpf/siren/plugins/receivers/httpreceiver" - "github.com/odpf/siren/plugins/receivers/pagerduty" - "github.com/odpf/siren/plugins/receivers/slack" + "github.com/odpf/siren/plugins/queues/postgresq" "github.com/spf13/cobra" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -152,118 +141,61 @@ func StartServer(ctx context.Context, cfg config.Config) error { return err } - encryptor, err := secret.New(cfg.EncryptionKey) + encryptor, err := secret.New(cfg.Service.EncryptionKey) if err != nil { return fmt.Errorf("cannot initialize encryptor: %w", err) } - templateRepository := postgres.NewTemplateRepository(pgClient) - templateService := template.NewService(templateRepository) - - alertRepository := postgres.NewAlertRepository(pgClient) - alertHistoryService := alert.NewService(alertRepository) - - providerRepository := postgres.NewProviderRepository(pgClient) - providerService := provider.NewService(providerRepository) - - namespaceRepository := postgres.NewNamespaceRepository(pgClient) - namespaceService := namespace.NewService(encryptor, namespaceRepository) + var queue, dlq notification.Queuer + switch cfg.Notification.Queue.Kind { + case queues.KindPostgres: + queue, err = postgresq.New(logger, cfg.DB) + if err != nil { + return err + } + dlq, err = postgresq.New(logger, cfg.DB, postgresq.WithStrategy(postgresq.StrategyDLQ)) + if err != nil { + return err + } + default: + queue = inmemory.New(logger, 50) + dlq = inmemory.New(logger, 10) + } - cortexClient, err := cortex.NewClient(cortex.Config{Address: cfg.Cortex.Address}) + apiDeps, _, _, notifierRegistry, err := InitAPIDeps(logger, cfg, pgClient, encryptor, queue) if err != nil { - return fmt.Errorf("failed to init cortex client: %w", err) + return err } - cortexProviderService := cortex.NewProviderService(cortexClient) - - ruleRepository := postgres.NewRuleRepository(pgClient) - ruleService := rule.NewService( - ruleRepository, - templateService, - namespaceService, - map[string]rule.RuleUploader{ - provider.TypeCortex: cortexProviderService, - }, - ) - - // plugin receiver services - slackHTTPClient := httpclient.New(cfg.Receivers.Slack.HTTPClient) - slackRetrier := retry.New(cfg.Receivers.Slack.Retry) - slackClient := slack.NewClient( - cfg.Receivers.Slack, - slack.ClientWithHTTPClient(slackHTTPClient), - slack.ClientWithRetrier(slackRetrier), - ) - pagerdutyHTTPClient := httpclient.New(cfg.Receivers.Pagerduty.HTTPClient) - pagerdutyRetrier := retry.New(cfg.Receivers.Slack.Retry) - pagerdutyClient := pagerduty.NewClient( - cfg.Receivers.Pagerduty, - pagerduty.ClientWithHTTPClient(pagerdutyHTTPClient), - pagerduty.ClientWithRetrier(pagerdutyRetrier), - ) - httpreceiverHTTPClient := httpclient.New(cfg.Receivers.HTTPReceiver.HTTPClient) - httpreceiverRetrier := retry.New(cfg.Receivers.Slack.Retry) - httpreceiverClient := httpreceiver.NewClient( - logger, - cfg.Receivers.HTTPReceiver, - httpreceiver.ClientWithHTTPClient(httpreceiverHTTPClient), - httpreceiver.ClientWithRetrier(httpreceiverRetrier), - ) - - slackReceiverService := slack.NewReceiverService(slackClient, encryptor) - httpReceiverService := httpreceiver.NewReceiverService() - pagerDutyReceiverService := pagerduty.NewReceiverService() - - receiverRepository := postgres.NewReceiverRepository(pgClient) - receiverService := receiver.NewService( - receiverRepository, - map[string]receiver.ConfigResolver{ - receiver.TypeSlack: slackReceiverService, - receiver.TypeHTTP: httpReceiverService, - receiver.TypePagerDuty: pagerDutyReceiverService, - }, - ) - - subscriptionRepository := postgres.NewSubscriptionRepository(pgClient) - subscriptionService := subscription.NewService( - subscriptionRepository, - namespaceService, - receiverService, - subscription.RegisterProviderPlugin(provider.TypeCortex, cortexProviderService), - ) // notification - slackNotificationService := slack.NewNotificationService(slackClient) - pagerdutyNotificationService := pagerduty.NewNotificationService(pagerdutyClient) - httpreceiverNotificationService := httpreceiver.NewNotificationService(httpreceiverClient) - - notifierRegistry := map[string]notification.Notifier{ - receiver.TypeSlack: slackNotificationService, - receiver.TypePagerDuty: pagerdutyNotificationService, - receiver.TypeHTTP: httpreceiverNotificationService, - } - - // for dev purpose only, should not be used in production - queue := inmemory.New(logger, 50) - notificationService := notification.NewService(logger, queue, receiverService, subscriptionService, notifierRegistry) - - apiDeps := &api.Deps{ - TemplateService: templateService, - RuleService: ruleService, - AlertService: alertHistoryService, - ProviderService: providerService, - NamespaceService: namespaceService, - ReceiverService: receiverService, - SubscriptionService: subscriptionService, - NotificationService: notificationService, - } - // run worker - notificationHandler := notification.NewHandler(logger, queue, notifierRegistry) - cancelWorkerChan := make(chan struct{}) wg := &sync.WaitGroup{} - wg.Add(1) - go notificationHandler.RunHandler(ctx, wg, cancelWorkerChan) + + if cfg.Notification.MessageHandler.Enabled { + workerTicker := worker.NewTicker(logger, worker.WithTickerDuration(cfg.Notification.MessageHandler.PollDuration)) + notificationHandler := notification.NewHandler(cfg.Notification.MessageHandler, logger, queue, notifierRegistry, + notification.HandlerWithIdentifier(workerTicker.GetID())) + wg.Add(1) + go func() { + defer wg.Done() + workerTicker.Run(ctx, cancelWorkerChan, func(ctx context.Context, runningAt time.Time) error { + return notificationHandler.Process(ctx, runningAt) + }) + }() + } + if cfg.Notification.DLQHandler.Enabled { + workerDLQTicker := worker.NewTicker(logger, worker.WithTickerDuration(cfg.Notification.DLQHandler.PollDuration)) + notificationDLQHandler := notification.NewHandler(cfg.Notification.DLQHandler, logger, dlq, notifierRegistry, + notification.HandlerWithIdentifier("dlq"+workerDLQTicker.GetID())) + wg.Add(1) + go func() { + defer wg.Done() + workerDLQTicker.Run(ctx, cancelWorkerChan, func(ctx context.Context, runningAt time.Time) error { + return notificationDLQHandler.Process(ctx, runningAt) + }) + }() + } err = server.RunServer( ctx, @@ -287,6 +219,9 @@ func StartServer(ctx context.Context, cfg config.Config) error { if err := queue.Stop(timeoutCtx); err != nil { logger.Error("error stopping queue", "error", err) } + if err := dlq.Stop(timeoutCtx); err != nil { + logger.Error("error stopping dlq", "error", err) + } return err } diff --git a/cli/worker.go b/cli/worker.go new file mode 100644 index 00000000..2ff7a837 --- /dev/null +++ b/cli/worker.go @@ -0,0 +1,210 @@ +package cli + +import ( + "context" + "fmt" + "time" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/db" + "github.com/odpf/siren/config" + "github.com/odpf/siren/core/notification" + "github.com/odpf/siren/internal/store/postgres" + "github.com/odpf/siren/pkg/secret" + "github.com/odpf/siren/pkg/worker" + "github.com/odpf/siren/plugins/queues" + "github.com/odpf/siren/plugins/queues/postgresq" + "github.com/spf13/cobra" +) + +func workerCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "worker ", + Aliases: []string{"w"}, + Short: "Siren's worker command", + Long: "Worker management commands.", + Example: heredoc.Doc(` + $ siren worker start notification_handler + $ siren worker start notification_handler -c ./config.yaml + `), + } + + cmd.AddCommand( + workerStartCommand(), + ) + + return cmd +} + +func workerStartCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "start ", + Aliases: []string{"w"}, + Short: "Start a siren worker", + Long: "Command to start a siren worker.", + Example: heredoc.Doc(` + $ siren worker start notification_handler + $ siren server start notification_handler -c ./config.yaml + `), + } + + cmd.AddCommand( + workerStartNotificationHandlerCommand(), + workerStartNotificationDLQHandlerCommand(), + ) + + return cmd +} + +func workerStartNotificationHandlerCommand() *cobra.Command { + var configFile string + + c := &cobra.Command{ + Use: "notification_handler", + Short: "A notification handler", + Long: "Start a handler to dequeue and publish notification messages.", + Example: heredoc.Doc(` + $ siren worker start notification_handler + $ siren worker start notification_handler -c ./config.yaml + `), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + cfg, err := config.Load(configFile) + if err != nil { + return err + } + + logger := initLogger(cfg.Log) + + dbClient, err := db.New(cfg.DB) + if err != nil { + return err + } + + pgClient, err := postgres.NewClient(logger, dbClient) + if err != nil { + return err + } + + encryptor, err := secret.New(cfg.Service.EncryptionKey) + if err != nil { + return fmt.Errorf("cannot initialize encryptor: %w", err) + } + + _, _, _, notifierRegistry, err := InitAPIDeps(logger, cfg, pgClient, encryptor, nil) + if err != nil { + return err + } + + cancelWorkerChan := make(chan struct{}) + + var queue notification.Queuer + switch cfg.Notification.Queue.Kind { + case queues.KindPostgres: + queue, err = postgresq.New(logger, cfg.DB) + if err != nil { + return err + } + default: + return fmt.Errorf(heredoc.Docf(` + unsupported kind of queue for worker: %s + supported queue kind are: + - postgres + `, cfg.Notification.Queue.Kind.String())) + } + workerTicker := worker.NewTicker(logger, worker.WithTickerDuration(cfg.Notification.MessageHandler.PollDuration)) + notificationHandler := notification.NewHandler(cfg.Notification.MessageHandler, logger, queue, notifierRegistry, + notification.HandlerWithIdentifier(workerTicker.GetID())) + go func() { + workerTicker.Run(ctx, cancelWorkerChan, func(ctx context.Context, runningAt time.Time) error { + return notificationHandler.Process(ctx, runningAt) + }) + }() + + <-ctx.Done() + close(cancelWorkerChan) + + return nil + }, + } + + c.Flags().StringVarP(&configFile, "config", "c", "config.yaml", "Config file path") + return c +} + +func workerStartNotificationDLQHandlerCommand() *cobra.Command { + var configFile string + + c := &cobra.Command{ + Use: "notification_dlq_handler", + Short: "A notification dlq handler", + Long: "Start a handler to dequeue dlq and publish notification messages.", + Example: heredoc.Doc(` + $ siren worker start notification_dlq_handler + $ siren worker start notification_dlq_handler -c ./config.yaml + `), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + cfg, err := config.Load(configFile) + if err != nil { + return err + } + + logger := initLogger(cfg.Log) + + dbClient, err := db.New(cfg.DB) + if err != nil { + return err + } + + pgClient, err := postgres.NewClient(logger, dbClient) + if err != nil { + return err + } + + encryptor, err := secret.New(cfg.Service.EncryptionKey) + if err != nil { + return fmt.Errorf("cannot initialize encryptor: %w", err) + } + + _, _, _, notifierRegistry, err := InitAPIDeps(logger, cfg, pgClient, encryptor, nil) + if err != nil { + return err + } + + cancelWorkerChan := make(chan struct{}) + + var queue notification.Queuer + switch cfg.Notification.Queue.Kind { + case queues.KindPostgres: + queue, err = postgresq.New(logger, cfg.DB, postgresq.WithStrategy(postgresq.StrategyDLQ)) + if err != nil { + return err + } + default: + return fmt.Errorf(heredoc.Docf(` + unsupported kind of queue for worker: %s + supported queue kind are: + - postgres + `, string(cfg.Notification.Queue.Kind))) + } + + workerTicker := worker.NewTicker(logger, worker.WithTickerDuration(cfg.Notification.DLQHandler.PollDuration)) + notificationHandler := notification.NewHandler(cfg.Notification.DLQHandler, logger, queue, notifierRegistry, + notification.HandlerWithIdentifier("dlq-"+workerTicker.GetID())) + go func() { + workerTicker.Run(ctx, cancelWorkerChan, func(ctx context.Context, runningAt time.Time) error { + return notificationHandler.Process(ctx, runningAt) + }) + }() + + <-ctx.Done() + close(cancelWorkerChan) + + return nil + }, + } + + c.Flags().StringVarP(&configFile, "config", "c", "config.yaml", "Config file path") + return c +} diff --git a/config/config.go b/config/config.go index d2708a22..ea883966 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "github.com/odpf/salt/config" "github.com/odpf/salt/db" + "github.com/odpf/siren/core/notification" "github.com/odpf/siren/internal/server" "github.com/odpf/siren/pkg/errors" "github.com/odpf/siren/pkg/telemetry" @@ -30,17 +31,17 @@ func Load(configFile string) (Config, error) { } type Log struct { - Level string `yaml:"level" mapstructure:"level" default:"info"` - GCPCompatible bool `yaml:"gcp_compatible" mapstructure:"gcp_compatible" default:"true"` + Level string `mapstructure:"level" default:"info"` + GCPCompatible bool `mapstructure:"gcp_compatible" default:"true"` } // Config contains the application configuration type Config struct { - DB db.Config `mapstructure:"db"` - Cortex cortex.Config `mapstructure:"cortex"` - NewRelic telemetry.NewRelicConfig `mapstructure:"newrelic"` - Service server.Config `mapstructure:"service"` - Log Log `mapstructure:"log"` - EncryptionKey string `mapstructure:"encryption_key"` - Receivers receivers.Config `mapstructure:"receivers"` + DB db.Config `mapstructure:"db"` + Cortex cortex.Config `mapstructure:"cortex"` + NewRelic telemetry.NewRelicConfig `mapstructure:"newrelic"` + Service server.Config `mapstructure:"service"` + Log Log `mapstructure:"log"` + Receivers receivers.Config `mapstructure:"receivers"` + Notification notification.Config `mapstructure:"notification"` } diff --git a/config/config.yaml b/config/config.yaml index 4d3839e4..8fcc3088 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -24,7 +24,7 @@ newrelic: service: host: localhost port: 8080 -encryption_key: ____STRING_OF_32_CHARACTERS_____ + encryption_key: ____STRING_OF_32_CHARACTERS_____ receivers: slack: api_host: @@ -32,4 +32,14 @@ receivers: max_retry: 4 wait_duration: 20ms http_client: - timeout_ms: 500 \ No newline at end of file + timeout_ms: 500 +notification: + queue: + kind: postgres + message_handler: + enabled: true + receiver_types: slack,http,pagerduty + poll_duration: 5s + batch_size: 1 + dlq_handler: + enabled: true diff --git a/core/namespace/crypto.go b/core/namespace/crypto.go index d44bd219..51743a96 100644 --- a/core/namespace/crypto.go +++ b/core/namespace/crypto.go @@ -1,7 +1,9 @@ package namespace +import "github.com/odpf/siren/pkg/secret" + //go:generate mockery --name=Encryptor -r --case underscore --with-expecter --structname Encryptor --filename encryptor.go --output=./mocks type Encryptor interface { - Encrypt(str string) (string, error) - Decrypt(str string) (string, error) + Encrypt(str secret.MaskableString) (secret.MaskableString, error) + Decrypt(str secret.MaskableString) (secret.MaskableString, error) } diff --git a/core/namespace/mocks/encryptor.go b/core/namespace/mocks/encryptor.go index 6659fae9..04e4d2f3 100644 --- a/core/namespace/mocks/encryptor.go +++ b/core/namespace/mocks/encryptor.go @@ -2,7 +2,11 @@ package mocks -import mock "github.com/stretchr/testify/mock" +import ( + mock "github.com/stretchr/testify/mock" + + secret "github.com/odpf/siren/pkg/secret" +) // Encryptor is an autogenerated mock type for the Encryptor type type Encryptor struct { @@ -18,18 +22,18 @@ func (_m *Encryptor) EXPECT() *Encryptor_Expecter { } // Decrypt provides a mock function with given fields: str -func (_m *Encryptor) Decrypt(str string) (string, error) { +func (_m *Encryptor) Decrypt(str secret.MaskableString) (secret.MaskableString, error) { ret := _m.Called(str) - var r0 string - if rf, ok := ret.Get(0).(func(string) string); ok { + var r0 secret.MaskableString + if rf, ok := ret.Get(0).(func(secret.MaskableString) secret.MaskableString); ok { r0 = rf(str) } else { - r0 = ret.Get(0).(string) + r0 = ret.Get(0).(secret.MaskableString) } var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { + if rf, ok := ret.Get(1).(func(secret.MaskableString) error); ok { r1 = rf(str) } else { r1 = ret.Error(1) @@ -44,36 +48,36 @@ type Encryptor_Decrypt_Call struct { } // Decrypt is a helper method to define mock.On call -// - str string +// - str secret.MaskableString func (_e *Encryptor_Expecter) Decrypt(str interface{}) *Encryptor_Decrypt_Call { return &Encryptor_Decrypt_Call{Call: _e.mock.On("Decrypt", str)} } -func (_c *Encryptor_Decrypt_Call) Run(run func(str string)) *Encryptor_Decrypt_Call { +func (_c *Encryptor_Decrypt_Call) Run(run func(str secret.MaskableString)) *Encryptor_Decrypt_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) + run(args[0].(secret.MaskableString)) }) return _c } -func (_c *Encryptor_Decrypt_Call) Return(_a0 string, _a1 error) *Encryptor_Decrypt_Call { +func (_c *Encryptor_Decrypt_Call) Return(_a0 secret.MaskableString, _a1 error) *Encryptor_Decrypt_Call { _c.Call.Return(_a0, _a1) return _c } // Encrypt provides a mock function with given fields: str -func (_m *Encryptor) Encrypt(str string) (string, error) { +func (_m *Encryptor) Encrypt(str secret.MaskableString) (secret.MaskableString, error) { ret := _m.Called(str) - var r0 string - if rf, ok := ret.Get(0).(func(string) string); ok { + var r0 secret.MaskableString + if rf, ok := ret.Get(0).(func(secret.MaskableString) secret.MaskableString); ok { r0 = rf(str) } else { - r0 = ret.Get(0).(string) + r0 = ret.Get(0).(secret.MaskableString) } var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { + if rf, ok := ret.Get(1).(func(secret.MaskableString) error); ok { r1 = rf(str) } else { r1 = ret.Error(1) @@ -88,19 +92,19 @@ type Encryptor_Encrypt_Call struct { } // Encrypt is a helper method to define mock.On call -// - str string +// - str secret.MaskableString func (_e *Encryptor_Expecter) Encrypt(str interface{}) *Encryptor_Encrypt_Call { return &Encryptor_Encrypt_Call{Call: _e.mock.On("Encrypt", str)} } -func (_c *Encryptor_Encrypt_Call) Run(run func(str string)) *Encryptor_Encrypt_Call { +func (_c *Encryptor_Encrypt_Call) Run(run func(str secret.MaskableString)) *Encryptor_Encrypt_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) + run(args[0].(secret.MaskableString)) }) return _c } -func (_c *Encryptor_Encrypt_Call) Return(_a0 string, _a1 error) *Encryptor_Encrypt_Call { +func (_c *Encryptor_Encrypt_Call) Return(_a0 secret.MaskableString, _a1 error) *Encryptor_Encrypt_Call { _c.Call.Return(_a0, _a1) return _c } diff --git a/core/namespace/namespace.go b/core/namespace/namespace.go index 1397dc3e..b3e08cca 100644 --- a/core/namespace/namespace.go +++ b/core/namespace/namespace.go @@ -18,7 +18,7 @@ type Repository interface { type EncryptedNamespace struct { *Namespace - Credentials string + CredentialString string } type Namespace struct { diff --git a/core/namespace/service.go b/core/namespace/service.go index 23f38958..b5ffe224 100644 --- a/core/namespace/service.go +++ b/core/namespace/service.go @@ -5,6 +5,7 @@ import ( "encoding/json" "github.com/odpf/siren/pkg/errors" + "github.com/odpf/siren/pkg/secret" ) // Service handles business logic @@ -106,19 +107,19 @@ func (s *Service) encrypt(ns *Namespace) (*EncryptedNamespace, error) { return nil, err } - encryptedCredentials, err := s.cryptoClient.Encrypt(string(plainTextCredentials)) + encryptedCredentials, err := s.cryptoClient.Encrypt(secret.MaskableString(plainTextCredentials)) if err != nil { return nil, err } return &EncryptedNamespace{ - Namespace: ns, - Credentials: encryptedCredentials, + Namespace: ns, + CredentialString: encryptedCredentials.UnmaskedString(), }, nil } func (s *Service) decrypt(ens *EncryptedNamespace) (*Namespace, error) { - decryptedCredentialsStr, err := s.cryptoClient.Decrypt(ens.Credentials) + decryptedCredentialsStr, err := s.cryptoClient.Decrypt(secret.MaskableString(ens.CredentialString)) if err != nil { return nil, err } diff --git a/core/namespace/service_test.go b/core/namespace/service_test.go index c0c5c631..f451d2b0 100644 --- a/core/namespace/service_test.go +++ b/core/namespace/service_test.go @@ -10,6 +10,7 @@ import ( "github.com/odpf/siren/core/namespace/mocks" "github.com/odpf/siren/core/provider" "github.com/odpf/siren/pkg/errors" + "github.com/odpf/siren/pkg/secret" "github.com/stretchr/testify/assert" mock "github.com/stretchr/testify/mock" ) @@ -49,7 +50,7 @@ func TestService_ListNamespaces(t *testing.T) { CreatedAt: timeNow, UpdatedAt: timeNow, }, - Credentials: `encrypted-text-1`, + CredentialString: `encrypted-text-1`, }, { Namespace: &namespace.Namespace{ @@ -62,10 +63,10 @@ func TestService_ListNamespaces(t *testing.T) { CreatedAt: timeNow, UpdatedAt: timeNow, }, - Credentials: `encrypted-text-2`, + CredentialString: `encrypted-text-2`, }, }, nil) - e.EXPECT().Decrypt(mock.AnythingOfType("string")).Return("", errors.New("decrypt error")) + e.EXPECT().Decrypt(mock.AnythingOfType("secret.MaskableString")).Return("", errors.New("decrypt error")) }, Err: errors.New("decrypt error"), }, @@ -84,7 +85,7 @@ func TestService_ListNamespaces(t *testing.T) { CreatedAt: timeNow, UpdatedAt: timeNow, }, - Credentials: `encrypted-text-1`, + CredentialString: `encrypted-text-1`, }, { Namespace: &namespace.Namespace{ @@ -97,10 +98,10 @@ func TestService_ListNamespaces(t *testing.T) { CreatedAt: timeNow, UpdatedAt: timeNow, }, - Credentials: `encrypted-text-2`, + CredentialString: `encrypted-text-2`, }, }, nil) - e.EXPECT().Decrypt(mock.AnythingOfType("string")).Return("", nil) + e.EXPECT().Decrypt(mock.AnythingOfType("secret.MaskableString")).Return("", nil) }, Err: errors.New("unexpected end of JSON input"), }, @@ -119,7 +120,7 @@ func TestService_ListNamespaces(t *testing.T) { CreatedAt: timeNow, UpdatedAt: timeNow, }, - Credentials: `encrypted-text-1`, + CredentialString: `encrypted-text-1`, }, { Namespace: &namespace.Namespace{ @@ -132,10 +133,10 @@ func TestService_ListNamespaces(t *testing.T) { CreatedAt: timeNow, UpdatedAt: timeNow, }, - Credentials: `encrypted-text-2`, + CredentialString: `encrypted-text-2`, }, }, nil) - e.EXPECT().Decrypt(mock.AnythingOfType("string")).Return("{\"name\": \"a\"}", nil) + e.EXPECT().Decrypt(mock.AnythingOfType("secret.MaskableString")).Return("{\"name\": \"a\"}", nil) }, ExpectedNamespaces: []namespace.Namespace{ { @@ -224,7 +225,7 @@ func TestService_CreateNamespace(t *testing.T) { { Description: "should return error if encrypt return error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("", errors.New("some error")) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("", errors.New("some error")) }, NSpace: &namespace.Namespace{ Credentials: map[string]interface{}{ @@ -236,10 +237,10 @@ func TestService_CreateNamespace(t *testing.T) { { Description: "should return error if encrypt success and create repository error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Create(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(errors.New("some error")) }, NSpace: &namespace.Namespace{ @@ -252,10 +253,10 @@ func TestService_CreateNamespace(t *testing.T) { { Description: "should return error conflict if encrypt success and create repository return duplicate error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Create(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(namespace.ErrDuplicate) }, NSpace: &namespace.Namespace{ @@ -268,10 +269,10 @@ func TestService_CreateNamespace(t *testing.T) { { Description: "should return error not found if encrypt success and create repository return relation error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Create(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(namespace.ErrRelation) }, NSpace: &namespace.Namespace{ @@ -284,10 +285,10 @@ func TestService_CreateNamespace(t *testing.T) { { Description: "should return nil error if encrypt success and create repository success", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Create(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(nil) }, NSpace: &namespace.Namespace{ @@ -351,10 +352,10 @@ func TestService_GetNamespace(t *testing.T) { Description: "should return error if Get repository success and decrypt return error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { rr.EXPECT().Get(mock.AnythingOfType("*context.emptyCtx"), testID).Return(&namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }, nil) - e.EXPECT().Decrypt("some-ciphertext").Return("", errors.New("some error")) + e.EXPECT().Decrypt(secret.MaskableString("some-ciphertext")).Return("", errors.New("some error")) }, Err: errors.New("some error"), }, @@ -362,10 +363,10 @@ func TestService_GetNamespace(t *testing.T) { Description: "should return error if Get repository success and decrypted credentials is not json marshallable", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { rr.EXPECT().Get(mock.AnythingOfType("*context.emptyCtx"), testID).Return(&namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }, nil) - e.EXPECT().Decrypt("some-ciphertext").Return("", nil) + e.EXPECT().Decrypt(secret.MaskableString("some-ciphertext")).Return("", nil) }, Err: errors.New("unexpected end of JSON input"), }, @@ -373,10 +374,10 @@ func TestService_GetNamespace(t *testing.T) { Description: "should return nil error if Get repository success and decrypt success", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { rr.EXPECT().Get(mock.AnythingOfType("*context.emptyCtx"), testID).Return(&namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }, nil) - e.EXPECT().Decrypt("some-ciphertext").Return("{ \"key\": \"value\" }", nil) + e.EXPECT().Decrypt(secret.MaskableString("some-ciphertext")).Return("{ \"key\": \"value\" }", nil) }, NSpace: &namespace.Namespace{ Credentials: map[string]interface{}{ @@ -436,7 +437,7 @@ func TestService_UpdateNamespace(t *testing.T) { { Description: "should return error if encrypt return error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("", errors.New("some error")) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("", errors.New("some error")) }, NSpace: &namespace.Namespace{ Credentials: map[string]interface{}{ @@ -448,10 +449,10 @@ func TestService_UpdateNamespace(t *testing.T) { { Description: "should return error if encrypt success and update repository error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Update(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(errors.New("some error")) }, NSpace: &namespace.Namespace{ @@ -464,10 +465,10 @@ func TestService_UpdateNamespace(t *testing.T) { { Description: "should return error not found if encrypt success and update repository return not found error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Update(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(namespace.NotFoundError{}) }, NSpace: &namespace.Namespace{ @@ -480,10 +481,10 @@ func TestService_UpdateNamespace(t *testing.T) { { Description: "should return error not found if encrypt success and update repository return relation error", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Update(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(namespace.ErrRelation) }, NSpace: &namespace.Namespace{ @@ -496,10 +497,10 @@ func TestService_UpdateNamespace(t *testing.T) { { Description: "should return error conflict if encrypt success and update repository return error duplicate", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Update(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(namespace.ErrDuplicate) }, NSpace: &namespace.Namespace{ @@ -512,10 +513,10 @@ func TestService_UpdateNamespace(t *testing.T) { { Description: "should return nil error if encrypt success and update repository success", Setup: func(rr *mocks.NamespaceRepository, e *mocks.Encryptor, tc testCase) { - e.EXPECT().Encrypt(mock.AnythingOfType("string")).Return("some-ciphertext", nil) + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("some-ciphertext", nil) rr.EXPECT().Update(mock.AnythingOfType("*context.emptyCtx"), &namespace.EncryptedNamespace{ - Namespace: tc.NSpace, - Credentials: "some-ciphertext", + Namespace: tc.NSpace, + CredentialString: "some-ciphertext", }).Return(nil) }, NSpace: &namespace.Namespace{ diff --git a/core/notification/config.go b/core/notification/config.go new file mode 100644 index 00000000..980fd18e --- /dev/null +++ b/core/notification/config.go @@ -0,0 +1,20 @@ +package notification + +import ( + "time" + + "github.com/odpf/siren/plugins/queues" +) + +type Config struct { + Queue queues.Config `mapstructure:"queue"` + MessageHandler HandlerConfig `mapstructure:"message_handler"` + DLQHandler HandlerConfig `mapstructure:"dlq_handler"` +} + +type HandlerConfig struct { + Enabled bool `mapstructure:"enabled"` + PollDuration time.Duration `mapstructure:"poll_duration"` + ReceiverTypes []string `mapstructure:"receiver_types"` + BatchSize int `mapstructure:"batch_size"` +} diff --git a/core/notification/handler.go b/core/notification/handler.go index 4b7e6d8d..6952624f 100644 --- a/core/notification/handler.go +++ b/core/notification/handler.go @@ -2,48 +2,63 @@ package notification import ( "context" - "sync" + "fmt" "time" - "github.com/google/uuid" "github.com/odpf/salt/log" "github.com/odpf/siren/pkg/errors" ) const ( - defaultPollDuration = 5 * time.Second - defaultBatchSize = 1 + defaultBatchSize = 1 ) // Handler is a process to handle message publishing type Handler struct { - id string logger log.Logger q Queuer + identifier string notifierRegistry map[string]Notifier supportedReceiverTypes []string - batchSize int - pollDuration time.Duration + batchSize int } // NewHandler creates a new handler with some supported type of Notifiers -func NewHandler(logger log.Logger, q Queuer, registry map[string]Notifier, opts ...HandlerOption) *Handler { +func NewHandler(cfg HandlerConfig, logger log.Logger, q Queuer, registry map[string]Notifier, opts ...HandlerOption) *Handler { h := &Handler{ - id: uuid.NewString(), - batchSize: defaultBatchSize, - pollDuration: defaultPollDuration, + batchSize: defaultBatchSize, logger: logger, notifierRegistry: registry, q: q, } - keys := make([]string, 0, len(h.notifierRegistry)) + if cfg.BatchSize != 0 { + h.batchSize = cfg.BatchSize + } + registeredReceivers := make([]string, 0, len(h.notifierRegistry)) for k := range h.notifierRegistry { - keys = append(keys, k) + registeredReceivers = append(registeredReceivers, k) + } + h.supportedReceiverTypes = registeredReceivers + + if len(cfg.ReceiverTypes) != 0 { + newSupportedReceiverTypes := []string{} + for _, rt := range cfg.ReceiverTypes { + found := false + for _, k := range registeredReceivers { + if rt == k { + found = true + break + } + } + if found { + newSupportedReceiverTypes = append(newSupportedReceiverTypes, rt) + } + } + h.supportedReceiverTypes = newSupportedReceiverTypes } - h.supportedReceiverTypes = keys for _, opt := range opts { opt(h) @@ -55,37 +70,26 @@ func NewHandler(logger log.Logger, q Queuer, registry map[string]Notifier, opts func (h *Handler) getNotifierPlugin(receiverType string) (Notifier, error) { receiverPlugin, exist := h.notifierRegistry[receiverType] if !exist { - return nil, errors.ErrInvalid.WithMsgf("unsupported receiver type: %q", receiverType) + return nil, errors.ErrInvalid.WithMsgf("unsupported receiver type: %q on handler %s", receiverType, h.identifier) } return receiverPlugin, nil } -func (h *Handler) RunHandler(ctx context.Context, wg *sync.WaitGroup, cancelChan chan struct{}) { - defer wg.Done() - - ticker := time.NewTicker(h.pollDuration) - defer ticker.Stop() - - h.logger.Info("running handler", "id", h.id) - - for { - select { - case <-cancelChan: - h.logger.Info("stopping handler", "id", h.id) - return - - case t := <-ticker.C: - receiverTypes := h.supportedReceiverTypes - if len(receiverTypes) == 0 { - h.logger.Warn("no receiver type plugin registered, skipping dequeue", "scope", "notification.handler") +func (h *Handler) Process(ctx context.Context, runAt time.Time) error { + receiverTypes := h.supportedReceiverTypes + if len(receiverTypes) == 0 { + return errors.New("no receiver type plugin registered, skipping dequeue") + } else { + h.logger.Debug("dequeueing and publishing messages", "scope", "notification.handler", "receivers", receiverTypes, "batch size", h.batchSize, "running_at", runAt, "id", h.identifier) + if err := h.q.Dequeue(ctx, receiverTypes, h.batchSize, h.MessageHandler); err != nil { + if errors.Is(err, ErrNoMessage) { + h.logger.Debug(err.Error(), "id", h.identifier) } else { - h.logger.Debug("dequeueing and publishing messages", "scope", "notification.handler", "receivers", receiverTypes, "batch size", h.batchSize, "running_at", t) - if err := h.q.Dequeue(ctx, receiverTypes, h.batchSize, h.MessageHandler); err != nil && err != ErrNoMessage { - h.logger.Error("dequeue failed", "scope", "notification.handler", "error", err) - } + return fmt.Errorf("dequeue failed on handler with id %s: %w", h.identifier, err) } } } + return nil } // MessageHandler is a function to handler dequeued message @@ -98,11 +102,22 @@ func (h *Handler) MessageHandler(ctx context.Context, messages []Message) error message.MarkPending(time.Now()) + newConfig, err := notifier.PostHookTransformConfigs(ctx, message.Configs) + if err != nil { + message.MarkFailed(time.Now(), false, err) + + if err := h.q.ErrorCallback(ctx, message); err != nil { + return err + } + return err + } + message.Configs = newConfig + if retryable, err := notifier.Publish(ctx, message); err != nil { message.MarkFailed(time.Now(), retryable, err) - if err := h.q.ErrorHandler(ctx, message); err != nil { + if err := h.q.ErrorCallback(ctx, message); err != nil { return err } return err @@ -110,7 +125,7 @@ func (h *Handler) MessageHandler(ctx context.Context, messages []Message) error message.MarkPublished(time.Now()) - if err := h.q.SuccessHandler(ctx, message); err != nil { + if err := h.q.SuccessCallback(ctx, message); err != nil { return err } diff --git a/core/notification/handler_option.go b/core/notification/handler_option.go index 7708d746..aa888f06 100644 --- a/core/notification/handler_option.go +++ b/core/notification/handler_option.go @@ -1,20 +1,18 @@ package notification -import "time" - // HandlerOption is an option to customize handler creation type HandlerOption func(*Handler) -// HandlerWithPollDuration sets created handler with the specified poll duration -func HandlerWithPollDuration(pollDuration time.Duration) HandlerOption { - return func(h *Handler) { - h.pollDuration = pollDuration +// HandlerWithBatchSize sets created handler with the specified batch size +func HandlerWithBatchSize(bs int) HandlerOption { + return func(w *Handler) { + w.batchSize = bs } } -// HandlerWithBatchSize sets created handler with the specified batch size -func HandlerWithBatchSize(bs int) HandlerOption { - return func(h *Handler) { - h.batchSize = bs +// HandlerWithIdentifier sets created handler with the specified batch size +func HandlerWithIdentifier(identifier string) HandlerOption { + return func(w *Handler) { + w.identifier = identifier } } diff --git a/core/notification/handler_test.go b/core/notification/handler_test.go index 4f238fd2..97ba2567 100644 --- a/core/notification/handler_test.go +++ b/core/notification/handler_test.go @@ -29,6 +29,32 @@ func TestHandler_MessageHandler(t *testing.T) { }, wantErr: true, }, + { + name: "return error if post hook transform config is failing and error callback success", + messages: []notification.Message{ + { + ReceiverType: testPluginType, + }, + }, + setup: func(q *mocks.Queuer, n *mocks.Notifier) { + n.EXPECT().PostHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(nil, errors.New("some error")) + q.EXPECT().ErrorCallback(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(nil) + }, + wantErr: true, + }, + { + name: "return error if post hook transform config is failing and error callback is failing", + messages: []notification.Message{ + { + ReceiverType: testPluginType, + }, + }, + setup: func(q *mocks.Queuer, n *mocks.Notifier) { + n.EXPECT().PostHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(nil, errors.New("some error")) + q.EXPECT().ErrorCallback(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(errors.New("some error")) + }, + wantErr: true, + }, { name: "return error if publish message return error and error handler queue return error", messages: []notification.Message{ @@ -37,8 +63,9 @@ func TestHandler_MessageHandler(t *testing.T) { }, }, setup: func(q *mocks.Queuer, n *mocks.Notifier) { + n.EXPECT().PostHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(map[string]interface{}{}, nil) n.EXPECT().Publish(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(false, errors.New("some error")) - q.EXPECT().ErrorHandler(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(errors.New("some error")) + q.EXPECT().ErrorCallback(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(errors.New("some error")) }, wantErr: true, }, @@ -50,8 +77,9 @@ func TestHandler_MessageHandler(t *testing.T) { }, }, setup: func(q *mocks.Queuer, n *mocks.Notifier) { + n.EXPECT().PostHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(map[string]interface{}{}, nil) n.EXPECT().Publish(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(false, errors.New("some error")) - q.EXPECT().ErrorHandler(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(nil) + q.EXPECT().ErrorCallback(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(nil) }, wantErr: true, }, @@ -63,8 +91,9 @@ func TestHandler_MessageHandler(t *testing.T) { }, }, setup: func(q *mocks.Queuer, n *mocks.Notifier) { + n.EXPECT().PostHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(map[string]interface{}{}, nil) n.EXPECT().Publish(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(false, nil) - q.EXPECT().SuccessHandler(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(errors.New("some error")) + q.EXPECT().SuccessCallback(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(errors.New("some error")) }, wantErr: true, }, @@ -76,8 +105,9 @@ func TestHandler_MessageHandler(t *testing.T) { }, }, setup: func(q *mocks.Queuer, n *mocks.Notifier) { + n.EXPECT().PostHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(map[string]interface{}{}, nil) n.EXPECT().Publish(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(false, nil) - q.EXPECT().SuccessHandler(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(nil) + q.EXPECT().SuccessCallback(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(nil) }, wantErr: false, }, @@ -93,7 +123,7 @@ func TestHandler_MessageHandler(t *testing.T) { tc.setup(mockQueue, mockNotifier) } - h := notification.NewHandler(log.NewNoop(), mockQueue, map[string]notification.Notifier{ + h := notification.NewHandler(notification.HandlerConfig{}, log.NewNoop(), mockQueue, map[string]notification.Notifier{ testReceiverType: mockNotifier, }) if err := h.MessageHandler(context.TODO(), tc.messages); (err != nil) != tc.wantErr { diff --git a/core/notification/message.go b/core/notification/message.go index cb2b43c4..7bcaee33 100644 --- a/core/notification/message.go +++ b/core/notification/message.go @@ -12,6 +12,9 @@ type MessageStatus string const ( DefaultMaxTries = 3 + // additional details + DetailsKeyRoutingMethod = "routing_method" + MessageStatusEnqueued MessageStatus = "enqueued" MessageStatusFailed MessageStatus = "failed" MessageStatusPending MessageStatus = "pending" @@ -130,3 +133,8 @@ func (m *Message) MarkPublished(updatedAt time.Time) { m.Status = MessageStatusPublished m.UpdatedAt = updatedAt } + +// AddDetail adds a custom kv string detail +func (m *Message) AddStringDetail(key, value string) { + m.Details[key] = value +} diff --git a/core/notification/message_test.go b/core/notification/message_test.go index 40638819..df3e2a64 100644 --- a/core/notification/message_test.go +++ b/core/notification/message_test.go @@ -25,7 +25,7 @@ func TestMessage_Initialize(t *testing.T) { wantErr bool }{ { - name: "all notification labels and variables should be merged to message detail and variable takes precedence if key conflict", + name: "all notification labels and data should be merged to message detail and data takes precedence if key conflict", n: notification.Notification{ Labels: map[string]string{ "labelkey1": "value1", @@ -54,7 +54,9 @@ func TestMessage_Initialize(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { m := ¬ification.Message{} - m.Initialize(tc.n, tc.receiverType, tc.notificationConfigs, + m.Initialize(tc.n, + tc.receiverType, + tc.notificationConfigs, notification.InitWithID(testID), notification.InitWithCreateTime(testTimeNow), notification.InitWithExpiryDuration(testExpiryDuration), diff --git a/core/notification/mocks/notifier.go b/core/notification/mocks/notifier.go index 50802fc4..786456f3 100644 --- a/core/notification/mocks/notifier.go +++ b/core/notification/mocks/notifier.go @@ -59,20 +59,22 @@ func (_c *Notifier_DefaultTemplateOfProvider_Call) Return(_a0 string) *Notifier_ return _c } -// Publish provides a mock function with given fields: ctx, message -func (_m *Notifier) Publish(ctx context.Context, message notification.Message) (bool, error) { - ret := _m.Called(ctx, message) +// PostHookTransformConfigs provides a mock function with given fields: ctx, notificationConfigMap +func (_m *Notifier) PostHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) { + ret := _m.Called(ctx, notificationConfigMap) - var r0 bool - if rf, ok := ret.Get(0).(func(context.Context, notification.Message) bool); ok { - r0 = rf(ctx, message) + var r0 map[string]interface{} + if rf, ok := ret.Get(0).(func(context.Context, map[string]interface{}) map[string]interface{}); ok { + r0 = rf(ctx, notificationConfigMap) } else { - r0 = ret.Get(0).(bool) + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]interface{}) + } } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, notification.Message) error); ok { - r1 = rf(ctx, message) + if rf, ok := ret.Get(1).(func(context.Context, map[string]interface{}) error); ok { + r1 = rf(ctx, notificationConfigMap) } else { r1 = ret.Error(1) } @@ -80,64 +82,119 @@ func (_m *Notifier) Publish(ctx context.Context, message notification.Message) ( return r0, r1 } -// Notifier_Publish_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Publish' -type Notifier_Publish_Call struct { +// Notifier_PostHookTransformConfigs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PostHookTransformConfigs' +type Notifier_PostHookTransformConfigs_Call struct { *mock.Call } -// Publish is a helper method to define mock.On call +// PostHookTransformConfigs is a helper method to define mock.On call // - ctx context.Context -// - message notification.Message -func (_e *Notifier_Expecter) Publish(ctx interface{}, message interface{}) *Notifier_Publish_Call { - return &Notifier_Publish_Call{Call: _e.mock.On("Publish", ctx, message)} +// - notificationConfigMap map[string]interface{} +func (_e *Notifier_Expecter) PostHookTransformConfigs(ctx interface{}, notificationConfigMap interface{}) *Notifier_PostHookTransformConfigs_Call { + return &Notifier_PostHookTransformConfigs_Call{Call: _e.mock.On("PostHookTransformConfigs", ctx, notificationConfigMap)} } -func (_c *Notifier_Publish_Call) Run(run func(ctx context.Context, message notification.Message)) *Notifier_Publish_Call { +func (_c *Notifier_PostHookTransformConfigs_Call) Run(run func(ctx context.Context, notificationConfigMap map[string]interface{})) *Notifier_PostHookTransformConfigs_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(notification.Message)) + run(args[0].(context.Context), args[1].(map[string]interface{})) }) return _c } -func (_c *Notifier_Publish_Call) Return(_a0 bool, _a1 error) *Notifier_Publish_Call { +func (_c *Notifier_PostHookTransformConfigs_Call) Return(_a0 map[string]interface{}, _a1 error) *Notifier_PostHookTransformConfigs_Call { _c.Call.Return(_a0, _a1) return _c } -// ValidateConfigMap provides a mock function with given fields: notificationConfigMap -func (_m *Notifier) ValidateConfigMap(notificationConfigMap map[string]interface{}) error { - ret := _m.Called(notificationConfigMap) +// PreHookTransformConfigs provides a mock function with given fields: ctx, notificationConfigMap +func (_m *Notifier) PreHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) { + ret := _m.Called(ctx, notificationConfigMap) - var r0 error - if rf, ok := ret.Get(0).(func(map[string]interface{}) error); ok { - r0 = rf(notificationConfigMap) + var r0 map[string]interface{} + if rf, ok := ret.Get(0).(func(context.Context, map[string]interface{}) map[string]interface{}); ok { + r0 = rf(ctx, notificationConfigMap) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]interface{}) + } } - return r0 + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, map[string]interface{}) error); ok { + r1 = rf(ctx, notificationConfigMap) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -// Notifier_ValidateConfigMap_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateConfigMap' -type Notifier_ValidateConfigMap_Call struct { +// Notifier_PreHookTransformConfigs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PreHookTransformConfigs' +type Notifier_PreHookTransformConfigs_Call struct { *mock.Call } -// ValidateConfigMap is a helper method to define mock.On call +// PreHookTransformConfigs is a helper method to define mock.On call +// - ctx context.Context // - notificationConfigMap map[string]interface{} -func (_e *Notifier_Expecter) ValidateConfigMap(notificationConfigMap interface{}) *Notifier_ValidateConfigMap_Call { - return &Notifier_ValidateConfigMap_Call{Call: _e.mock.On("ValidateConfigMap", notificationConfigMap)} +func (_e *Notifier_Expecter) PreHookTransformConfigs(ctx interface{}, notificationConfigMap interface{}) *Notifier_PreHookTransformConfigs_Call { + return &Notifier_PreHookTransformConfigs_Call{Call: _e.mock.On("PreHookTransformConfigs", ctx, notificationConfigMap)} } -func (_c *Notifier_ValidateConfigMap_Call) Run(run func(notificationConfigMap map[string]interface{})) *Notifier_ValidateConfigMap_Call { +func (_c *Notifier_PreHookTransformConfigs_Call) Run(run func(ctx context.Context, notificationConfigMap map[string]interface{})) *Notifier_PreHookTransformConfigs_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(map[string]interface{})) + run(args[0].(context.Context), args[1].(map[string]interface{})) }) return _c } -func (_c *Notifier_ValidateConfigMap_Call) Return(_a0 error) *Notifier_ValidateConfigMap_Call { - _c.Call.Return(_a0) +func (_c *Notifier_PreHookTransformConfigs_Call) Return(_a0 map[string]interface{}, _a1 error) *Notifier_PreHookTransformConfigs_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// Publish provides a mock function with given fields: ctx, message +func (_m *Notifier) Publish(ctx context.Context, message notification.Message) (bool, error) { + ret := _m.Called(ctx, message) + + var r0 bool + if rf, ok := ret.Get(0).(func(context.Context, notification.Message) bool); ok { + r0 = rf(ctx, message) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, notification.Message) error); ok { + r1 = rf(ctx, message) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Notifier_Publish_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Publish' +type Notifier_Publish_Call struct { + *mock.Call +} + +// Publish is a helper method to define mock.On call +// - ctx context.Context +// - message notification.Message +func (_e *Notifier_Expecter) Publish(ctx interface{}, message interface{}) *Notifier_Publish_Call { + return &Notifier_Publish_Call{Call: _e.mock.On("Publish", ctx, message)} +} + +func (_c *Notifier_Publish_Call) Run(run func(ctx context.Context, message notification.Message)) *Notifier_Publish_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(notification.Message)) + }) + return _c +} + +func (_c *Notifier_Publish_Call) Return(_a0 bool, _a1 error) *Notifier_Publish_Call { + _c.Call.Return(_a0, _a1) return _c } diff --git a/core/notification/mocks/queuer.go b/core/notification/mocks/queuer.go index b98abd7e..55ab5ad3 100644 --- a/core/notification/mocks/queuer.go +++ b/core/notification/mocks/queuer.go @@ -7,6 +7,8 @@ import ( notification "github.com/odpf/siren/core/notification" mock "github.com/stretchr/testify/mock" + + queues "github.com/odpf/siren/plugins/queues" ) // Queuer is an autogenerated mock type for the Queuer type @@ -22,6 +24,44 @@ func (_m *Queuer) EXPECT() *Queuer_Expecter { return &Queuer_Expecter{mock: &_m.Mock} } +// Cleanup provides a mock function with given fields: ctx, filter +func (_m *Queuer) Cleanup(ctx context.Context, filter queues.FilterCleanup) error { + ret := _m.Called(ctx, filter) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, queues.FilterCleanup) error); ok { + r0 = rf(ctx, filter) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Queuer_Cleanup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Cleanup' +type Queuer_Cleanup_Call struct { + *mock.Call +} + +// Cleanup is a helper method to define mock.On call +// - ctx context.Context +// - filter queues.FilterCleanup +func (_e *Queuer_Expecter) Cleanup(ctx interface{}, filter interface{}) *Queuer_Cleanup_Call { + return &Queuer_Cleanup_Call{Call: _e.mock.On("Cleanup", ctx, filter)} +} + +func (_c *Queuer_Cleanup_Call) Run(run func(ctx context.Context, filter queues.FilterCleanup)) *Queuer_Cleanup_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(queues.FilterCleanup)) + }) + return _c +} + +func (_c *Queuer_Cleanup_Call) Return(_a0 error) *Queuer_Cleanup_Call { + _c.Call.Return(_a0) + return _c +} + // Dequeue provides a mock function with given fields: ctx, receiverTypes, batchSize, handlerFn func (_m *Queuer) Dequeue(ctx context.Context, receiverTypes []string, batchSize int, handlerFn func(context.Context, []notification.Message) error) error { ret := _m.Called(ctx, receiverTypes, batchSize, handlerFn) @@ -114,8 +154,8 @@ func (_c *Queuer_Enqueue_Call) Return(_a0 error) *Queuer_Enqueue_Call { return _c } -// ErrorHandler provides a mock function with given fields: ctx, ms -func (_m *Queuer) ErrorHandler(ctx context.Context, ms notification.Message) error { +// ErrorCallback provides a mock function with given fields: ctx, ms +func (_m *Queuer) ErrorCallback(ctx context.Context, ms notification.Message) error { ret := _m.Called(ctx, ms) var r0 error @@ -128,26 +168,26 @@ func (_m *Queuer) ErrorHandler(ctx context.Context, ms notification.Message) err return r0 } -// Queuer_ErrorHandler_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ErrorHandler' -type Queuer_ErrorHandler_Call struct { +// Queuer_ErrorCallback_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ErrorCallback' +type Queuer_ErrorCallback_Call struct { *mock.Call } -// ErrorHandler is a helper method to define mock.On call +// ErrorCallback is a helper method to define mock.On call // - ctx context.Context // - ms notification.Message -func (_e *Queuer_Expecter) ErrorHandler(ctx interface{}, ms interface{}) *Queuer_ErrorHandler_Call { - return &Queuer_ErrorHandler_Call{Call: _e.mock.On("ErrorHandler", ctx, ms)} +func (_e *Queuer_Expecter) ErrorCallback(ctx interface{}, ms interface{}) *Queuer_ErrorCallback_Call { + return &Queuer_ErrorCallback_Call{Call: _e.mock.On("ErrorCallback", ctx, ms)} } -func (_c *Queuer_ErrorHandler_Call) Run(run func(ctx context.Context, ms notification.Message)) *Queuer_ErrorHandler_Call { +func (_c *Queuer_ErrorCallback_Call) Run(run func(ctx context.Context, ms notification.Message)) *Queuer_ErrorCallback_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(notification.Message)) }) return _c } -func (_c *Queuer_ErrorHandler_Call) Return(_a0 error) *Queuer_ErrorHandler_Call { +func (_c *Queuer_ErrorCallback_Call) Return(_a0 error) *Queuer_ErrorCallback_Call { _c.Call.Return(_a0) return _c } @@ -189,8 +229,8 @@ func (_c *Queuer_Stop_Call) Return(_a0 error) *Queuer_Stop_Call { return _c } -// SuccessHandler provides a mock function with given fields: ctx, ms -func (_m *Queuer) SuccessHandler(ctx context.Context, ms notification.Message) error { +// SuccessCallback provides a mock function with given fields: ctx, ms +func (_m *Queuer) SuccessCallback(ctx context.Context, ms notification.Message) error { ret := _m.Called(ctx, ms) var r0 error @@ -203,26 +243,26 @@ func (_m *Queuer) SuccessHandler(ctx context.Context, ms notification.Message) e return r0 } -// Queuer_SuccessHandler_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SuccessHandler' -type Queuer_SuccessHandler_Call struct { +// Queuer_SuccessCallback_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SuccessCallback' +type Queuer_SuccessCallback_Call struct { *mock.Call } -// SuccessHandler is a helper method to define mock.On call +// SuccessCallback is a helper method to define mock.On call // - ctx context.Context // - ms notification.Message -func (_e *Queuer_Expecter) SuccessHandler(ctx interface{}, ms interface{}) *Queuer_SuccessHandler_Call { - return &Queuer_SuccessHandler_Call{Call: _e.mock.On("SuccessHandler", ctx, ms)} +func (_e *Queuer_Expecter) SuccessCallback(ctx interface{}, ms interface{}) *Queuer_SuccessCallback_Call { + return &Queuer_SuccessCallback_Call{Call: _e.mock.On("SuccessCallback", ctx, ms)} } -func (_c *Queuer_SuccessHandler_Call) Run(run func(ctx context.Context, ms notification.Message)) *Queuer_SuccessHandler_Call { +func (_c *Queuer_SuccessCallback_Call) Run(run func(ctx context.Context, ms notification.Message)) *Queuer_SuccessCallback_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].(notification.Message)) }) return _c } -func (_c *Queuer_SuccessHandler_Call) Return(_a0 error) *Queuer_SuccessHandler_Call { +func (_c *Queuer_SuccessCallback_Call) Return(_a0 error) *Queuer_SuccessCallback_Call { _c.Call.Return(_a0) return _c } diff --git a/core/notification/notification.go b/core/notification/notification.go index bb9003a1..01c80b28 100644 --- a/core/notification/notification.go +++ b/core/notification/notification.go @@ -5,11 +5,13 @@ import ( "time" "github.com/odpf/siren/pkg/errors" + "github.com/odpf/siren/plugins/queues" ) //go:generate mockery --name=Notifier -r --case underscore --with-expecter --structname Notifier --filename notifier.go --output=./mocks type Notifier interface { - ValidateConfigMap(notificationConfigMap map[string]interface{}) error + PreHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) + PostHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) DefaultTemplateOfProvider(templateName string) string Publish(ctx context.Context, message Message) (bool, error) } @@ -18,8 +20,9 @@ type Notifier interface { type Queuer interface { Enqueue(ctx context.Context, ms ...Message) error Dequeue(ctx context.Context, receiverTypes []string, batchSize int, handlerFn func(context.Context, []Message) error) error - SuccessHandler(ctx context.Context, ms Message) error - ErrorHandler(ctx context.Context, ms Message) error + SuccessCallback(ctx context.Context, ms Message) error + ErrorCallback(ctx context.Context, ms Message) error + Cleanup(ctx context.Context, filter queues.FilterCleanup) error Stop(ctx context.Context) error } diff --git a/core/notification/routing.go b/core/notification/routing.go new file mode 100644 index 00000000..b62f9054 --- /dev/null +++ b/core/notification/routing.go @@ -0,0 +1,12 @@ +package notification + +type RoutingMethod string + +const ( + RoutingMethodReceiver RoutingMethod = "receiver" + RoutingMethodSubscribers RoutingMethod = "subscribers" +) + +func (rm RoutingMethod) String() string { + return string(rm) +} diff --git a/core/notification/service.go b/core/notification/service.go index bab50f86..bfd98508 100644 --- a/core/notification/service.go +++ b/core/notification/service.go @@ -27,7 +27,7 @@ type NotificationService struct { q Queuer receiverService ReceiverService subscriptionService SubscriptionService - receiverPlugins map[string]Notifier + notifierPlugins map[string]Notifier } // NewService creates a new notification service @@ -36,23 +36,23 @@ func NewService( q Queuer, receiverService ReceiverService, subscriptionService SubscriptionService, - receiverPlugins map[string]Notifier, + notifierPlugins map[string]Notifier, ) *NotificationService { return &NotificationService{ logger: logger, q: q, receiverService: receiverService, subscriptionService: subscriptionService, - receiverPlugins: receiverPlugins, + notifierPlugins: notifierPlugins, } } -func (ns *NotificationService) getReceiverPlugin(receiverType string) (Notifier, error) { - receiverPlugin, exist := ns.receiverPlugins[receiverType] +func (ns *NotificationService) getNotifierPlugin(receiverType string) (Notifier, error) { + notifierPlugin, exist := ns.notifierPlugins[receiverType] if !exist { return nil, errors.ErrInvalid.WithMsgf("unsupported receiver type: %q", receiverType) } - return receiverPlugin, nil + return notifierPlugin, nil } func (ns *NotificationService) DispatchToReceiver(ctx context.Context, n Notification, receiverID uint64) error { @@ -66,7 +66,20 @@ func (ns *NotificationService) DispatchToReceiver(ctx context.Context, n Notific return err } - // supported no template + notifierPlugin, err := ns.getNotifierPlugin(rcv.Type) + if err != nil { + return err + } + + newConfigs, err := notifierPlugin.PreHookTransformConfigs(ctx, message.Configs) + if err != nil { + return err + } + message.Configs = newConfigs + + message.AddStringDetail(DetailsKeyRoutingMethod, RoutingMethodReceiver.String()) + + // supported no templating for now if err := ns.q.Enqueue(ctx, *message); err != nil { return err @@ -94,14 +107,18 @@ func (ns *NotificationService) DispatchToSubscribers(ctx context.Context, n Noti return err } - receiverPlugin, err := ns.getReceiverPlugin(rcv.Type) + notifierPlugin, err := ns.getNotifierPlugin(rcv.Type) if err != nil { return err } - if err := receiverPlugin.ValidateConfigMap(message.Configs); err != nil { + newConfigs, err := notifierPlugin.PreHookTransformConfigs(ctx, message.Configs) + if err != nil { return err } + message.Configs = newConfigs + + message.AddStringDetail(DetailsKeyRoutingMethod, RoutingMethodSubscribers.String()) //TODO fetch template if any, if not exist, check provider type, if exist use the default template, if not pass as-is // if there is template, render and replace detail with the new one @@ -109,7 +126,7 @@ func (ns *NotificationService) DispatchToSubscribers(ctx context.Context, n Noti var templateBody string if template.IsReservedName(n.Template) { - templateBody = receiverPlugin.DefaultTemplateOfProvider(n.Template) + templateBody = notifierPlugin.DefaultTemplateOfProvider(n.Template) } if templateBody != "" { diff --git a/core/notification/service_test.go b/core/notification/service_test.go index d9b53b80..d9b187d9 100644 --- a/core/notification/service_test.go +++ b/core/notification/service_test.go @@ -22,6 +22,17 @@ func TestNotificationService_DispatchToReceiver(t *testing.T) { n notification.Notification wantErr bool }{ + + { + name: "should return error if failed to transform notification to messages", + setup: func(rs *mocks.ReceiverService, q *mocks.Queuer, n *mocks.Notifier) { + rs.EXPECT().Get(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("uint64")).Return(&receiver.Receiver{}, nil) + }, + n: notification.Notification{ + ValidDurationString: "xxx", + }, + wantErr: true, + }, { name: "should return error if there is an error when fetching receiver", setup: func(rs *mocks.ReceiverService, q *mocks.Queuer, n *mocks.Notifier) { @@ -30,12 +41,15 @@ func TestNotificationService_DispatchToReceiver(t *testing.T) { wantErr: true, }, { - name: "should return error if failed to transform notification to messages", + name: "should return error if prehook transform config return error", setup: func(rs *mocks.ReceiverService, q *mocks.Queuer, n *mocks.Notifier) { - rs.EXPECT().Get(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("uint64")).Return(&receiver.Receiver{}, nil) - }, - n: notification.Notification{ - ValidDurationString: "xxx", + rs.EXPECT().Get(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("uint64")).Return(&receiver.Receiver{ + Type: testPluginType, + Configurations: map[string]interface{}{ + "key": "value", + }, + }, nil) + n.EXPECT().PreHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(nil, errors.New("invalid config")) }, wantErr: true, }, @@ -48,6 +62,9 @@ func TestNotificationService_DispatchToReceiver(t *testing.T) { "key": "value", }, }, nil) + n.EXPECT().PreHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(map[string]interface{}{ + "key": "value", + }, nil) q.EXPECT().Enqueue(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(errors.New("some error")) }, wantErr: true, @@ -61,6 +78,9 @@ func TestNotificationService_DispatchToReceiver(t *testing.T) { "key": "value", }, }, nil) + n.EXPECT().PreHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(map[string]interface{}{ + "key": "value", + }, nil) q.EXPECT().Enqueue(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(nil) }, wantErr: false, @@ -169,7 +189,7 @@ func TestNotificationService_DispatchToSubscribers(t *testing.T) { }, }, }, nil) - n.EXPECT().ValidateConfigMap(mock.AnythingOfType("map[string]interface {}")).Return(errors.New("invalid config")) + n.EXPECT().PreHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(nil, errors.New("invalid config")) }, wantErr: true, }, @@ -190,7 +210,9 @@ func TestNotificationService_DispatchToSubscribers(t *testing.T) { }, }, }, nil) - n.EXPECT().ValidateConfigMap(mock.AnythingOfType("map[string]interface {}")).Return(nil) + n.EXPECT().PreHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(map[string]interface{}{ + "key": "value", + }, nil) q.EXPECT().Enqueue(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(errors.New("some error")) }, wantErr: true, @@ -212,7 +234,9 @@ func TestNotificationService_DispatchToSubscribers(t *testing.T) { }, }, }, nil) - n.EXPECT().ValidateConfigMap(mock.AnythingOfType("map[string]interface {}")).Return(nil) + n.EXPECT().PreHookTransformConfigs(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("map[string]interface {}")).Return(map[string]interface{}{ + "key": "value", + }, nil) q.EXPECT().Enqueue(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("notification.Message")).Return(nil) }, wantErr: false, diff --git a/core/provider/service_test.go b/core/provider/service_test.go index b7860056..43342114 100644 --- a/core/provider/service_test.go +++ b/core/provider/service_test.go @@ -7,17 +7,17 @@ import ( "github.com/odpf/siren/core/provider" "github.com/odpf/siren/core/provider/mocks" - "github.com/odpf/siren/internal/store/model" "github.com/odpf/siren/pkg/errors" + "github.com/odpf/siren/pkg/pgtype" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) func TestList(t *testing.T) { ctx := context.TODO() - credentials := make(model.StringInterfaceMap) + credentials := make(pgtype.StringInterfaceMap) credentials["foo"] = "bar" - labels := make(model.StringStringMap) + labels := make(pgtype.StringStringMap) labels["foo"] = "bar" t.Run("should call repository List method and return result in domain's type", func(t *testing.T) { @@ -56,9 +56,9 @@ func TestList(t *testing.T) { func TestCreate(t *testing.T) { ctx := context.TODO() - credentials := make(model.StringInterfaceMap) + credentials := make(pgtype.StringInterfaceMap) credentials["foo"] = "bar" - labels := make(model.StringStringMap) + labels := make(pgtype.StringStringMap) labels["foo"] = "bar" timenow := time.Now() dummyProviderID := uint64(10) @@ -104,9 +104,9 @@ func TestCreate(t *testing.T) { func TestGetProvider(t *testing.T) { ctx := context.TODO() dummyProviderID := uint64(10) - credentials := make(model.StringInterfaceMap) + credentials := make(pgtype.StringInterfaceMap) credentials["foo"] = "bar" - labels := make(model.StringStringMap) + labels := make(pgtype.StringStringMap) labels["foo"] = "bar" timenow := time.Now() dummyProvider := &provider.Provider{ @@ -155,9 +155,9 @@ func TestUpdateProvider(t *testing.T) { ctx := context.TODO() dummyProviderID := uint64(10) timenow := time.Now() - credentials := make(model.StringInterfaceMap) + credentials := make(pgtype.StringInterfaceMap) credentials["foo"] = "bar" - labels := make(model.StringStringMap) + labels := make(pgtype.StringStringMap) labels["foo"] = "bar" dummyProvider := &provider.Provider{ ID: dummyProviderID, diff --git a/core/subscription/mocks/receiver_service.go b/core/subscription/mocks/receiver_service.go index f3174c7d..6f40f7d1 100644 --- a/core/subscription/mocks/receiver_service.go +++ b/core/subscription/mocks/receiver_service.go @@ -239,45 +239,6 @@ func (_c *ReceiverService_List_Call) Return(_a0 []receiver.Receiver, _a1 error) return _c } -// Notify provides a mock function with given fields: ctx, id, payloadMessage -func (_m *ReceiverService) Notify(ctx context.Context, id uint64, payloadMessage map[string]interface{}) error { - ret := _m.Called(ctx, id, payloadMessage) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, uint64, map[string]interface{}) error); ok { - r0 = rf(ctx, id, payloadMessage) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// ReceiverService_Notify_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Notify' -type ReceiverService_Notify_Call struct { - *mock.Call -} - -// Notify is a helper method to define mock.On call -// - ctx context.Context -// - id uint64 -// - payloadMessage map[string]interface{} -func (_e *ReceiverService_Expecter) Notify(ctx interface{}, id interface{}, payloadMessage interface{}) *ReceiverService_Notify_Call { - return &ReceiverService_Notify_Call{Call: _e.mock.On("Notify", ctx, id, payloadMessage)} -} - -func (_c *ReceiverService_Notify_Call) Run(run func(ctx context.Context, id uint64, payloadMessage map[string]interface{})) *ReceiverService_Notify_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(uint64), args[2].(map[string]interface{})) - }) - return _c -} - -func (_c *ReceiverService_Notify_Call) Return(_a0 error) *ReceiverService_Notify_Call { - _c.Call.Return(_a0) - return _c -} - // Update provides a mock function with given fields: ctx, rcv func (_m *ReceiverService) Update(ctx context.Context, rcv *receiver.Receiver) error { ret := _m.Called(ctx, rcv) diff --git a/core/subscription/service.go b/core/subscription/service.go index e5bd33a0..8129cd6c 100644 --- a/core/subscription/service.go +++ b/core/subscription/service.go @@ -221,6 +221,8 @@ func (s *Service) Delete(ctx context.Context, id uint64) error { return nil } +// TODO we might want to add filter by namespace id too here +// to filter by tenant func (s *Service) MatchByLabels(ctx context.Context, labels map[string]string) ([]Subscription, error) { // fetch all subscriptions by matching labels. subscriptionsByLabels, err := s.repository.List(ctx, Filter{ diff --git a/internal/api/v1beta1/alert.go b/internal/api/v1beta1/alert.go index e79002cc..d5feeb5a 100644 --- a/internal/api/v1beta1/alert.go +++ b/internal/api/v1beta1/alert.go @@ -63,8 +63,8 @@ func (s *GRPCServer) CreateCortexAlerts(ctx context.Context, req *sirenv1beta1.C alrt := &alert.Alert{ ProviderID: req.GetProviderId(), ResourceName: fmt.Sprintf("%v", item.GetAnnotations()["resource"]), - MetricName: fmt.Sprintf("%v", item.GetAnnotations()["metric_name"]), - MetricValue: fmt.Sprintf("%v", item.GetAnnotations()["metric_value"]), + MetricName: fmt.Sprintf("%v", item.GetAnnotations()["metricName"]), + MetricValue: fmt.Sprintf("%v", item.GetAnnotations()["metricValue"]), Severity: severity, Rule: fmt.Sprintf("%v", item.GetAnnotations()["template"]), TriggeredAt: item.GetStartsAt().AsTime(), @@ -138,9 +138,9 @@ func CortexAlertPBToNotification( } data["status"] = a.GetStatus() - data["generator_url"] = a.GetGeneratorUrl() - data["num_alerts_firing"] = firingLen - data["group_key"] = groupKey + data["generatorUrl"] = a.GetGeneratorUrl() + data["numAlertsFiring"] = firingLen + data["groupKey"] = groupKey return notification.Notification{ ID: "cortex-" + a.GetFingerprint(), diff --git a/internal/api/v1beta1/alert_test.go b/internal/api/v1beta1/alert_test.go index 9b852211..0527302e 100644 --- a/internal/api/v1beta1/alert_test.go +++ b/internal/api/v1beta1/alert_test.go @@ -97,10 +97,10 @@ func TestGRPCServer_CreateAlertHistory(t *testing.T) { "severity": "CRITICAL", }, Annotations: map[string]string{ - "resource": "foo", - "template": "random", - "metric_name": "bar", - "metric_value": "30", + "resource": "foo", + "template": "random", + "metricName": "bar", + "metricValue": "30", }, StartsAt: timenow, }, @@ -152,10 +152,10 @@ func TestGRPCServer_CreateAlertHistory(t *testing.T) { "severity": "CRITICAL", }, Annotations: map[string]string{ - "resource": "foo", - "template": "random", - "metric_name": "bar", - "metric_value": "30", + "resource": "foo", + "template": "random", + "metricName": "bar", + "metricValue": "30", }, StartsAt: timenow, }, @@ -226,9 +226,9 @@ func TestGRPCServer_CreateAlertHistory(t *testing.T) { "severity": "CRITICAL", }, Annotations: map[string]string{ - "resource": "foo", - "metric_name": "bar", - "metric_value": "30", + "resource": "foo", + "metricName": "bar", + "metricValue": "30", }, StartsAt: timenow, }, @@ -238,10 +238,10 @@ func TestGRPCServer_CreateAlertHistory(t *testing.T) { "severity": "CRITICAL", }, Annotations: map[string]string{ - "resource": "foo", - "template": "random", - "metric_name": "bar", - "metric_value": "30", + "resource": "foo", + "template": "random", + "metricName": "bar", + "metricValue": "30", }, StartsAt: timenow, }, diff --git a/internal/server/server.go b/internal/server/server.go index 4679938d..d3007689 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -34,8 +34,9 @@ import ( const defaultGracePeriod = 5 * time.Second type Config struct { - Host string `mapstructure:"host" default:"localhost"` - Port int `mapstructure:"port" default:"8080"` + Host string `mapstructure:"host" default:"localhost"` + Port int `mapstructure:"port" default:"8080"` + EncryptionKey string `mapstructure:"encryption_key"` } func (cfg Config) addr() string { return fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) } diff --git a/internal/store/model/namespace.go b/internal/store/model/namespace.go index e2081b8d..83247a30 100644 --- a/internal/store/model/namespace.go +++ b/internal/store/model/namespace.go @@ -5,17 +5,18 @@ import ( "github.com/odpf/siren/core/namespace" "github.com/odpf/siren/core/provider" + "github.com/odpf/siren/pkg/pgtype" ) type Namespace struct { - ID uint64 `db:"id"` - ProviderID uint64 `db:"provider_id"` - URN string `db:"urn"` - Name string `db:"name"` - Credentials string `db:"credentials"` - Labels StringStringMap `db:"labels"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID uint64 `db:"id"` + ProviderID uint64 `db:"provider_id"` + URN string `db:"urn"` + Name string `db:"name"` + CredentialString string `db:"credentials"` + Labels pgtype.StringStringMap `db:"labels"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } func (ns *Namespace) FromDomain(n namespace.EncryptedNamespace) { @@ -23,8 +24,8 @@ func (ns *Namespace) FromDomain(n namespace.EncryptedNamespace) { ns.URN = n.URN ns.Name = n.Name ns.ProviderID = n.Provider.ID - ns.Credentials = n.Credentials - ns.Labels = StringStringMap(n.Labels) + ns.CredentialString = n.CredentialString + ns.Labels = pgtype.StringStringMap(n.Labels) ns.CreatedAt = n.CreatedAt ns.UpdatedAt = n.UpdatedAt } @@ -42,19 +43,19 @@ func (ns *Namespace) ToDomain() *namespace.EncryptedNamespace { CreatedAt: ns.CreatedAt, UpdatedAt: ns.UpdatedAt, }, - Credentials: ns.Credentials, + CredentialString: ns.CredentialString, } } type NamespaceDetail struct { - ID uint64 `db:"id"` - Provider Provider `db:"provider"` - URN string `db:"urn"` - Name string `db:"name"` - Credentials string `db:"credentials"` - Labels StringStringMap `db:"labels"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID uint64 `db:"id"` + Provider Provider `db:"provider"` + URN string `db:"urn"` + Name string `db:"name"` + CredentialString string `db:"credentials"` + Labels pgtype.StringStringMap `db:"labels"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } func (ns *NamespaceDetail) FromDomain(n namespace.EncryptedNamespace) { @@ -62,8 +63,8 @@ func (ns *NamespaceDetail) FromDomain(n namespace.EncryptedNamespace) { ns.URN = n.URN ns.Name = n.Name ns.Provider.FromDomain(n.Provider) - ns.Credentials = n.Credentials - ns.Labels = StringStringMap(n.Labels) + ns.CredentialString = n.CredentialString + ns.Labels = pgtype.StringStringMap(n.Labels) ns.CreatedAt = n.CreatedAt ns.UpdatedAt = n.UpdatedAt } @@ -79,6 +80,6 @@ func (ns *NamespaceDetail) ToDomain() *namespace.EncryptedNamespace { CreatedAt: ns.CreatedAt, UpdatedAt: ns.UpdatedAt, }, - Credentials: ns.Credentials, + CredentialString: ns.CredentialString, } } diff --git a/internal/store/model/provider.go b/internal/store/model/provider.go index 4ab27d9b..fdcb8ac5 100644 --- a/internal/store/model/provider.go +++ b/internal/store/model/provider.go @@ -1,66 +1,22 @@ package model import ( - "database/sql/driver" - "encoding/json" "time" "github.com/odpf/siren/core/provider" - "github.com/odpf/siren/pkg/errors" + "github.com/odpf/siren/pkg/pgtype" ) -type StringInterfaceMap map[string]interface{} - -type StringStringMap map[string]string - -func (m *StringInterfaceMap) Scan(value interface{}) error { - if value == nil { - m = new(StringInterfaceMap) - return nil - } - b, ok := value.([]byte) - if !ok { - return errors.New("failed type assertion to []byte") - } - return json.Unmarshal(b, &m) -} - -func (a StringInterfaceMap) Value() (driver.Value, error) { - if len(a) == 0 { - return nil, nil - } - return json.Marshal(a) -} - -func (m *StringStringMap) Scan(value interface{}) error { - if value == nil { - m = new(StringStringMap) - return nil - } - b, ok := value.([]byte) - if !ok { - return errors.New("failed type assertion to []byte") - } - return json.Unmarshal(b, &m) -} - -func (a StringStringMap) Value() (driver.Value, error) { - if len(a) == 0 { - return nil, nil - } - return json.Marshal(a) -} - type Provider struct { - ID uint64 `db:"id"` - Host string `db:"host"` - URN string `db:"urn"` - Name string `db:"name"` - Type string `db:"type"` - Credentials StringInterfaceMap `db:"credentials"` - Labels StringStringMap `db:"labels"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID uint64 `db:"id"` + Host string `db:"host"` + URN string `db:"urn"` + Name string `db:"name"` + Type string `db:"type"` + Credentials pgtype.StringInterfaceMap `db:"credentials"` + Labels pgtype.StringStringMap `db:"labels"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } func (p *Provider) FromDomain(t provider.Provider) { diff --git a/internal/store/model/receiver.go b/internal/store/model/receiver.go index 3db90910..16d169c5 100644 --- a/internal/store/model/receiver.go +++ b/internal/store/model/receiver.go @@ -4,17 +4,18 @@ import ( "time" "github.com/odpf/siren/core/receiver" + "github.com/odpf/siren/pkg/pgtype" ) type Receiver struct { - ID uint64 `db:"id"` - Name string `db:"name"` - Type string `db:"type"` - Labels StringStringMap `db:"labels"` - Configurations StringInterfaceMap `db:"configurations"` - Data StringInterfaceMap `db:"-"` //TODO do we need this? - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID uint64 `db:"id"` + Name string `db:"name"` + Type string `db:"type"` + Labels pgtype.StringStringMap `db:"labels"` + Configurations pgtype.StringInterfaceMap `db:"configurations"` + Data pgtype.StringInterfaceMap `db:"-"` //TODO do we need this? + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } func (rcv *Receiver) FromDomain(t receiver.Receiver) { @@ -22,7 +23,7 @@ func (rcv *Receiver) FromDomain(t receiver.Receiver) { rcv.Name = t.Name rcv.Type = t.Type rcv.Labels = t.Labels - rcv.Configurations = StringInterfaceMap(t.Configurations) + rcv.Configurations = pgtype.StringInterfaceMap(t.Configurations) rcv.Data = t.Data rcv.CreatedAt = t.CreatedAt rcv.UpdatedAt = t.UpdatedAt diff --git a/internal/store/model/subscription.go b/internal/store/model/subscription.go index 9c68aa58..1a70a7cd 100644 --- a/internal/store/model/subscription.go +++ b/internal/store/model/subscription.go @@ -6,6 +6,7 @@ import ( "time" "github.com/odpf/siren/core/subscription" + "github.com/odpf/siren/pkg/pgtype" ) type SubscriptionReceiver struct { @@ -25,13 +26,13 @@ func (list SubscriptionReceivers) Value() (driver.Value, error) { } type Subscription struct { - ID uint64 `db:"id"` - NamespaceID uint64 `db:"namespace_id"` - URN string `db:"urn"` - Receiver SubscriptionReceivers `db:"receiver"` - Match StringStringMap `db:"match"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID uint64 `db:"id"` + NamespaceID uint64 `db:"namespace_id"` + URN string `db:"urn"` + Receiver SubscriptionReceivers `db:"receiver"` + Match pgtype.StringStringMap `db:"match"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } func (s *Subscription) FromDomain(sub subscription.Subscription) { diff --git a/internal/store/postgres/migrations/000002_create_index_subscription_matcher.down.sql b/internal/store/postgres/migrations/000002_create_index_subscription_matcher.down.sql new file mode 100644 index 00000000..dd0430e9 --- /dev/null +++ b/internal/store/postgres/migrations/000002_create_index_subscription_matcher.down.sql @@ -0,0 +1 @@ +CREATE INDEX subscriptions_idx_match ON subscriptions USING GIN(match jsonb_path_ops); \ No newline at end of file diff --git a/internal/store/postgres/migrations/000002_create_index_subscription_matcher.up.sql b/internal/store/postgres/migrations/000002_create_index_subscription_matcher.up.sql new file mode 100644 index 00000000..04d731a3 --- /dev/null +++ b/internal/store/postgres/migrations/000002_create_index_subscription_matcher.up.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS subscriptions_idx_match; \ No newline at end of file diff --git a/internal/store/postgres/namespace.go b/internal/store/postgres/namespace.go index 04fa5948..f43c98dd 100644 --- a/internal/store/postgres/namespace.go +++ b/internal/store/postgres/namespace.go @@ -93,7 +93,7 @@ func (r NamespaceRepository) Create(ctx context.Context, ns *namespace.Encrypted nsModel.ProviderID, nsModel.URN, nsModel.Name, - nsModel.Credentials, + nsModel.CredentialString, nsModel.Labels, ).StructScan(&createdNamespace); err != nil { err = checkPostgresError(err) @@ -142,7 +142,7 @@ func (r NamespaceRepository) Update(ctx context.Context, ns *namespace.Encrypted namespaceModel.ProviderID, namespaceModel.URN, namespaceModel.Name, - namespaceModel.Credentials, + namespaceModel.CredentialString, namespaceModel.Labels, ).StructScan(&updatedNamespace); err != nil { err := checkPostgresError(err) diff --git a/internal/store/postgres/namespace_test.go b/internal/store/postgres/namespace_test.go index c606c803..ec13e482 100644 --- a/internal/store/postgres/namespace_test.go +++ b/internal/store/postgres/namespace_test.go @@ -115,7 +115,7 @@ func (s *NamespaceRepositoryTestSuite) TestList() { }, Labels: map[string]string{}, }, - Credentials: "map[secret_key:odpf-secret-key-1]", + CredentialString: "map[secret_key:odpf-secret-key-1]", }, { Namespace: &namespace.Namespace{ @@ -133,7 +133,7 @@ func (s *NamespaceRepositoryTestSuite) TestList() { }, Labels: map[string]string{}, }, - Credentials: "map[secret_key:odpf-secret-key-2]", + CredentialString: "map[secret_key:odpf-secret-key-2]", }, { Namespace: &namespace.Namespace{ @@ -151,7 +151,7 @@ func (s *NamespaceRepositoryTestSuite) TestList() { }, Labels: map[string]string{}, }, - Credentials: "map[service_key:instance-1-service-key]", + CredentialString: "map[service_key:instance-1-service-key]", }, }, }, @@ -202,7 +202,7 @@ func (s *NamespaceRepositoryTestSuite) TestGet() { }, Labels: map[string]string{}, }, - Credentials: "map[service_key:instance-1-service-key]", + CredentialString: "map[service_key:instance-1-service-key]", }, }, { @@ -246,7 +246,7 @@ func (s *NamespaceRepositoryTestSuite) TestCreate() { ID: 2, }, }, - Credentials: "xxx", + CredentialString: "xxx", }, ExpectedID: uint64(4), // autoincrement in db side }, @@ -260,7 +260,7 @@ func (s *NamespaceRepositoryTestSuite) TestCreate() { ID: 1000, }, }, - Credentials: "xxx", + CredentialString: "xxx", }, ErrString: "provider id does not exist", }, @@ -274,7 +274,7 @@ func (s *NamespaceRepositoryTestSuite) TestCreate() { ID: 2, }, }, - Credentials: "xxx", + CredentialString: "xxx", }, ErrString: "urn and provider pair already exist", }, @@ -316,7 +316,7 @@ func (s *NamespaceRepositoryTestSuite) TestUpdate() { ID: 2, }, }, - Credentials: "xxx", + CredentialString: "xxx", }, ExpectedID: uint64(1), }, @@ -331,7 +331,7 @@ func (s *NamespaceRepositoryTestSuite) TestUpdate() { ID: 2, }, }, - Credentials: "xxx", + CredentialString: "xxx", }, ErrString: "urn and provider pair already exist", }, @@ -346,7 +346,7 @@ func (s *NamespaceRepositoryTestSuite) TestUpdate() { ID: 2, }, }, - Credentials: "xxx", + CredentialString: "xxx", }, ErrString: "namespace with id 1000 not found", }, @@ -361,7 +361,7 @@ func (s *NamespaceRepositoryTestSuite) TestUpdate() { ID: 1000, }, }, - Credentials: "xxx", + CredentialString: "xxx", }, ErrString: "provider id does not exist", }, @@ -376,7 +376,7 @@ func (s *NamespaceRepositoryTestSuite) TestUpdate() { ID: 2, }, }, - Credentials: "xxx", + CredentialString: "xxx", }, ErrString: "urn and provider pair already exist", }, diff --git a/internal/store/postgres/postgres_test.go b/internal/store/postgres/postgres_test.go index 644232f8..d8786370 100644 --- a/internal/store/postgres/postgres_test.go +++ b/internal/store/postgres/postgres_test.go @@ -111,23 +111,11 @@ func bootstrapNamespace(client *postgres.Client) ([]namespace.EncryptedNamespace return nil, err } - // encryptService, err := secret.New(testEncryptionKey) - // if err != nil { - // return nil, err - // } - for _, d := range data { - // plainTextCredentials, err := json.Marshal(d.Credentials) - // if err != nil { - // return nil, err - // } - // cipherTextCredentials, err := encryptService.Encrypt(string(plainTextCredentials)) - // if err != nil { - // return nil, err - // } + encryptedNS := namespace.EncryptedNamespace{ - Namespace: &d, - Credentials: fmt.Sprintf("%+v", d.Credentials), + Namespace: &d, + CredentialString: fmt.Sprintf("%+v", d.Credentials), } if err := repo.Create(context.Background(), &encryptedNS); err != nil { return nil, err diff --git a/internal/store/postgres/subscription.go b/internal/store/postgres/subscription.go index aa465597..3cb01d26 100644 --- a/internal/store/postgres/subscription.go +++ b/internal/store/postgres/subscription.go @@ -53,16 +53,18 @@ func NewSubscriptionRepository(client *Client) *SubscriptionRepository { func (r *SubscriptionRepository) List(ctx context.Context, flt subscription.Filter) ([]subscription.Subscription, error) { var queryBuilder = subscriptionListQueryBuilder - if flt.NamespaceID != 0 { - queryBuilder = queryBuilder.Where("namespace_id = ?", flt.NamespaceID) - } - + // If filter by Labels and namespace ID exist, filter by namespace should be done in app + // to make use of search by labels with GIN index if len(flt.Labels) != 0 { labelsJSON, err := json.Marshal(flt.Labels) if err != nil { return nil, errors.ErrInvalid.WithCausef("problem marshalling json to string with err: %s", err.Error()) } queryBuilder = queryBuilder.Where(fmt.Sprintf("match <@ '%s'::jsonb", string(json.RawMessage(labelsJSON)))) + } else { + if flt.NamespaceID != 0 { + queryBuilder = queryBuilder.Where("namespace_id = ?", flt.NamespaceID) + } } query, args, err := queryBuilder.PlaceholderFormat(sq.Dollar).ToSql() @@ -83,6 +85,14 @@ func (r *SubscriptionRepository) List(ctx context.Context, flt subscription.Filt return nil, err } + // If filter by Labels and namespace ID exist, filter by namespace should be done in app + // to make use of search by labels with GIN index + if len(flt.Labels) != 0 && flt.NamespaceID != 0 { + if subscriptionModel.NamespaceID != flt.NamespaceID { + continue + } + } + subscriptionsDomain = append(subscriptionsDomain, *subscriptionModel.ToDomain()) } diff --git a/pkg/pgtype/map.go b/pkg/pgtype/map.go new file mode 100644 index 00000000..46152615 --- /dev/null +++ b/pkg/pgtype/map.go @@ -0,0 +1,49 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/json" + "errors" +) + +type StringInterfaceMap map[string]interface{} + +type StringStringMap map[string]string + +func (m *StringInterfaceMap) Scan(value interface{}) error { + if value == nil { + m = new(StringInterfaceMap) + return nil + } + b, ok := value.([]byte) + if !ok { + return errors.New("failed type assertion to []byte") + } + return json.Unmarshal(b, &m) +} + +func (a StringInterfaceMap) Value() (driver.Value, error) { + if len(a) == 0 { + return nil, nil + } + return json.Marshal(a) +} + +func (m *StringStringMap) Scan(value interface{}) error { + if value == nil { + m = new(StringStringMap) + return nil + } + b, ok := value.([]byte) + if !ok { + return errors.New("failed type assertion to []byte") + } + return json.Unmarshal(b, &m) +} + +func (a StringStringMap) Value() (driver.Value, error) { + if len(a) == 0 { + return nil, nil + } + return json.Marshal(a) +} diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go index 613709f0..f9c1a993 100644 --- a/pkg/secret/secret.go +++ b/pkg/secret/secret.go @@ -28,16 +28,16 @@ func New(encryptionKey string) (*Crypto, error) { }, nil } -func (sec *Crypto) Encrypt(str string) (string, error) { +func (sec *Crypto) Encrypt(str MaskableString) (MaskableString, error) { cipher, err := cryptopasta.Encrypt([]byte(str), sec.encryptionKey) if err != nil { return "", err } - return base64.StdEncoding.EncodeToString(cipher), nil + return MaskableString(base64.StdEncoding.EncodeToString(cipher)), nil } -func (sec *Crypto) Decrypt(str string) (string, error) { - encrypted, err := base64.StdEncoding.DecodeString(str) +func (sec *Crypto) Decrypt(str MaskableString) (MaskableString, error) { + encrypted, err := base64.StdEncoding.DecodeString(str.UnmaskedString()) if err != nil { return "", err } @@ -45,5 +45,5 @@ func (sec *Crypto) Decrypt(str string) (string, error) { if err != nil { return "", err } - return string(decryptedToken), nil + return MaskableString(decryptedToken), nil } diff --git a/pkg/secret/string.go b/pkg/secret/string.go new file mode 100644 index 00000000..ee8c386c --- /dev/null +++ b/pkg/secret/string.go @@ -0,0 +1,13 @@ +package secret + +import "strings" + +type MaskableString string + +func (m MaskableString) UnmaskedString() string { + return string(m) +} + +func (m MaskableString) String() string { + return strings.Repeat("*", len(m)) +} diff --git a/pkg/secret/string_test.go b/pkg/secret/string_test.go new file mode 100644 index 00000000..e64a4814 --- /dev/null +++ b/pkg/secret/string_test.go @@ -0,0 +1,25 @@ +package secret + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestString(t *testing.T) { + testString := "testtest" + + t.Run("should print masked string by default", func(t *testing.T) { + maskedString := MaskableString(testString) + result := fmt.Sprintf("%v", maskedString) + assert.Equal(t, result, strings.Repeat("*", len(testString))) + }) + + t.Run("should print unmasked string with unmasked function", func(t *testing.T) { + maskedString := MaskableString(testString) + result := fmt.Sprintf("%v", maskedString.UnmaskedString()) + assert.Equal(t, result, testString) + }) +} diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index 7cc40c54..c4714638 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -6,9 +6,9 @@ import ( // NewRelic contains the New Relic go-agent configuration type NewRelicConfig struct { - Enabled bool `yaml:"enabled" mapstructure:"enabled" default:"false"` - AppName string `yaml:"appname" mapstructure:"appname" default:"siren"` - License string `yaml:"license" mapstructure:"license"` + Enabled bool `mapstructure:"enabled" default:"false"` + AppName string `mapstructure:"appname" default:"siren"` + License string `mapstructure:"license"` } func New(c NewRelicConfig) (*newrelic.Application, error) { diff --git a/pkg/worker/option.go b/pkg/worker/option.go new file mode 100644 index 00000000..2742ad0d --- /dev/null +++ b/pkg/worker/option.go @@ -0,0 +1,13 @@ +package worker + +import "time" + +// TickerOption is an option to customize worker ticker creation +type TickerOption func(*Ticker) + +// WithTickerDuration sets created handler with the specified poll duration +func WithTickerDuration(pollDuration time.Duration) TickerOption { + return func(wt *Ticker) { + wt.pollDuration = pollDuration + } +} diff --git a/pkg/worker/ticker.go b/pkg/worker/ticker.go new file mode 100644 index 00000000..3d101ce3 --- /dev/null +++ b/pkg/worker/ticker.go @@ -0,0 +1,64 @@ +package worker + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/odpf/salt/log" +) + +const ( + defaultPollDuration = 5 * time.Second +) + +// Ticker is a worker that runs periodically +type Ticker struct { + id string + logger log.Logger + pollDuration time.Duration +} + +// NewTicker creates a new worker that does an action periodically +func NewTicker(logger log.Logger, opts ...TickerOption) *Ticker { + wt := &Ticker{ + id: uuid.NewString(), + logger: logger, + } + + for _, opt := range opts { + opt(wt) + } + + if wt.pollDuration == 0 { + wt.pollDuration = defaultPollDuration + } + + return wt +} + +// Run starts worker that handle a task periodically +func (wt *Ticker) Run(ctx context.Context, cancelChan chan struct{}, handlerFn func(ctx context.Context, runningAt time.Time) error) { + ticker := time.NewTicker(wt.pollDuration) + defer ticker.Stop() + + wt.logger.Info("running worker", "id", wt.id) + + for { + select { + case <-cancelChan: + wt.logger.Info("stopping worker", "id", wt.id) + return + + case t := <-ticker.C: + if err := handlerFn(ctx, t); err != nil { + wt.logger.Error("error running worker", "error", err, "id", wt.id) + } + } + } +} + +// GetID fetch identifier of a worker +func (wt *Ticker) GetID() string { + return wt.id +} diff --git a/plugins/providers/cortex/config.go b/plugins/providers/cortex/config.go index e7313265..c088d234 100644 --- a/plugins/providers/cortex/config.go +++ b/plugins/providers/cortex/config.go @@ -11,5 +11,5 @@ var ( // Config is a cortex provider config type Config struct { - Address string `yaml:"address" mapstructure:"address" default:"http://localhost:8080"` + Address string `mapstructure:"address" default:"http://localhost:8080"` } diff --git a/plugins/queues/config.go b/plugins/queues/config.go new file mode 100644 index 00000000..e1720913 --- /dev/null +++ b/plugins/queues/config.go @@ -0,0 +1,21 @@ +package queues + +type Kind string + +const ( + KindInMemory Kind = "inmemory" + KindPostgres Kind = "postgres" +) + +func (k Kind) String() string { + return string(k) +} + +type Config struct { + Kind Kind `mapstructure:"kind" default:"inmemory"` +} + +type FilterCleanup struct { + MessagePendingTimeThreshold string + MessagePublishedTimeThreshold string +} diff --git a/plugins/queues/inmemory/queue.go b/plugins/queues/inmemory/queue.go index 37790b77..6522ea3c 100644 --- a/plugins/queues/inmemory/queue.go +++ b/plugins/queues/inmemory/queue.go @@ -7,6 +7,8 @@ import ( "github.com/odpf/salt/log" "github.com/odpf/siren/core/notification" + "github.com/odpf/siren/plugins" + "github.com/odpf/siren/plugins/queues" ) // Queue simulates queue inmemory, this is for testing only @@ -71,18 +73,22 @@ func (q *Queue) Enqueue(ctx context.Context, ms ...notification.Message) error { return nil } -// SuccessHandler is a callback that will be called once the message is succesfully handled by handlerFn -func (q *Queue) SuccessHandler(ctx context.Context, ms notification.Message) error { - q.logger.Debug("successfully sending message", "scope", "queues.inmemory.success_handler", "type", ms.ReceiverType, "configs", ms.Configs, "details", ms.Details) +// SuccessCallback is a callback that will be called once the message is succesfully handled by handlerFn +func (q *Queue) SuccessCallback(ctx context.Context, ms notification.Message) error { + q.logger.Debug("successfully sending message", "scope", "queues.inmemory.success_callback", "type", ms.ReceiverType, "configs", ms.Configs, "details", ms.Details) return nil } -// ErrorHandler is a callback that will be called once the message is failed to be handled by handlerFn -func (q *Queue) ErrorHandler(ctx context.Context, ms notification.Message) error { - q.logger.Error("failed sending message", "scope", "queues.inmemory.error_handler", "type", ms.ReceiverType, "configs", ms.Configs, "details", ms.Details, "last_error", ms.LastError) +// ErrorCallback is a callback that will be called once the message is failed to be handled by handlerFn +func (q *Queue) ErrorCallback(ctx context.Context, ms notification.Message) error { + q.logger.Error("failed sending message", "scope", "queues.inmemory.error_callback", "type", ms.ReceiverType, "configs", ms.Configs, "details", ms.Details, "last_error", ms.LastError) return nil } +func (q *Queue) Cleanup(ctx context.Context, filter queues.FilterCleanup) error { + return plugins.ErrNotImplemented +} + // Stop is a inmemmory queue function // this will close the channel to simulate queue func (q *Queue) Stop(ctx context.Context) error { diff --git a/plugins/queues/postgresq/cleanup.go b/plugins/queues/postgresq/cleanup.go new file mode 100644 index 00000000..3192b420 --- /dev/null +++ b/plugins/queues/postgresq/cleanup.go @@ -0,0 +1,76 @@ +package postgresq + +import ( + "context" + "errors" + "fmt" + "time" + + sq "github.com/Masterminds/squirrel" + "github.com/odpf/siren/plugins/queues" +) + +const defaultPublishedTimeThreshold = time.Duration(7) * time.Hour + +func (q *Queue) Cleanup(ctx context.Context, filter queues.FilterCleanup) error { + + // validate filter + var ( + publishedTimeThreshold int + pendingTimeThreshold int + ) + if filter.MessagePendingTimeThreshold == "" { + publishedTimeThreshold = int(defaultPublishedTimeThreshold.Seconds()) + } else { + dur, err := time.ParseDuration(filter.MessagePublishedTimeThreshold) + if err != nil { + return err + } + publishedTimeThreshold = int(dur.Seconds()) + } + + if filter.MessagePendingTimeThreshold != "" { + dur, err := time.ParseDuration(filter.MessagePendingTimeThreshold) + if err != nil { + return err + } + pendingTimeThreshold = int(dur.Seconds()) + } + + var filterExpr sq.Sqlizer + messagePublishedExpr := sq.And{ + sq.Expr("status = 'published'"), + sq.Expr(fmt.Sprintf("now() - interval '%d seconds' > updated_at", publishedTimeThreshold)), + } + + if pendingTimeThreshold != 0 { + messagePendingExpr := sq.And{ + sq.Expr("status = 'pending'"), + sq.Expr(fmt.Sprintf("now() - interval '%d seconds' > updated_at", pendingTimeThreshold)), + } + filterExpr = sq.Or{ + messagePublishedExpr, + messagePendingExpr, + } + } else { + filterExpr = messagePublishedExpr + } + + query, args, err := sq.Delete(MESSAGE_QUEUE_TABLE_NAME).Where(filterExpr).ToSql() + if err != nil { + return err + } + + res, err := q.dbc.ExecContext(ctx, query, args...) + if err != nil { + return err + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return errors.New("no rows affected when cleanup messages") + } + return nil +} diff --git a/plugins/queues/postgresq/migrations/000001_create_message_queue_table.down.sql b/plugins/queues/postgresq/migrations/000001_create_message_queue_table.down.sql new file mode 100644 index 00000000..ef4988a2 --- /dev/null +++ b/plugins/queues/postgresq/migrations/000001_create_message_queue_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS message_queue; \ No newline at end of file diff --git a/plugins/queues/postgresq/migrations/000001_create_message_queue_table.up.sql b/plugins/queues/postgresq/migrations/000001_create_message_queue_table.up.sql new file mode 100644 index 00000000..531f40db --- /dev/null +++ b/plugins/queues/postgresq/migrations/000001_create_message_queue_table.up.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS message_queue ( + id text NOT NULL PRIMARY KEY, + status text NOT NULL, -- ENQUEUED/RUNNING/FAILED/DONE + receiver_type text NOT NULL, + configs jsonb, + details jsonb, + last_error text, + max_tries integer, + try_count integer, + retryable boolean NOT NULL DEFAULT false, + expired_at timestamptz, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); \ No newline at end of file diff --git a/plugins/queues/postgresq/migrations/migrations.go b/plugins/queues/postgresq/migrations/migrations.go new file mode 100644 index 00000000..8b080bcb --- /dev/null +++ b/plugins/queues/postgresq/migrations/migrations.go @@ -0,0 +1,8 @@ +package migrations + +import "embed" + +//go:embed *.sql +var FS embed.FS + +const ResourcePath = "." diff --git a/plugins/queues/postgresq/model.go b/plugins/queues/postgresq/model.go new file mode 100644 index 00000000..1d1163bc --- /dev/null +++ b/plugins/queues/postgresq/model.go @@ -0,0 +1,76 @@ +package postgresq + +import ( + "database/sql" + "time" + + "github.com/odpf/siren/core/notification" + "github.com/odpf/siren/pkg/pgtype" +) + +type NotificationMessage struct { + ID string `db:"id"` + Status string `db:"status"` + + ReceiverType string `db:"receiver_type"` + Configs pgtype.StringInterfaceMap `db:"configs"` + Details pgtype.StringInterfaceMap `db:"details"` + Metadata pgtype.StringInterfaceMap `db:"metadata"` + LastError sql.NullString `db:"last_error"` + + MaxTries int `db:"max_tries"` + TryCount int `db:"try_count"` + Retryable bool `db:"retryable"` + + ExpiredAt sql.NullTime `db:"expired_at"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} + +func (nm *NotificationMessage) FromDomain(domainMessage notification.Message) { + nm.ID = domainMessage.ID + nm.Status = string(domainMessage.Status) + nm.ReceiverType = domainMessage.ReceiverType + nm.Configs = domainMessage.Configs + nm.Details = domainMessage.Details + + nm.LastError = sql.NullString{String: domainMessage.LastError, Valid: func() bool { + if domainMessage.LastError == "" { + return false + } else { + return true + } + }()} + nm.MaxTries = domainMessage.MaxTries + nm.TryCount = domainMessage.TryCount + nm.Retryable = domainMessage.Retryable + nm.ExpiredAt = sql.NullTime{Time: domainMessage.ExpiredAt, Valid: func() bool { + if domainMessage.ExpiredAt.IsZero() { + return false + } else { + return true + } + }()} + nm.CreatedAt = domainMessage.CreatedAt + nm.UpdatedAt = domainMessage.UpdatedAt +} + +func (nm *NotificationMessage) ToDomain() notification.Message { + return notification.Message{ + ID: nm.ID, + Status: notification.MessageStatus(nm.Status), + + ReceiverType: nm.ReceiverType, + Configs: nm.Configs, + Details: nm.Details, + LastError: nm.LastError.String, + + MaxTries: nm.MaxTries, + TryCount: nm.TryCount, + Retryable: nm.Retryable, + + ExpiredAt: nm.ExpiredAt.Time, + CreatedAt: nm.CreatedAt, + UpdatedAt: nm.UpdatedAt, + } +} diff --git a/plugins/queues/postgresq/option.go b/plugins/queues/postgresq/option.go new file mode 100644 index 00000000..f677a0fe --- /dev/null +++ b/plugins/queues/postgresq/option.go @@ -0,0 +1,9 @@ +package postgresq + +type QueueOption func(*Queue) + +func WithStrategy(s Strategy) QueueOption { + return func(q *Queue) { + q.strategy = s + } +} diff --git a/plugins/queues/postgresq/queue.go b/plugins/queues/postgresq/queue.go new file mode 100644 index 00000000..910aaed0 --- /dev/null +++ b/plugins/queues/postgresq/queue.go @@ -0,0 +1,233 @@ +package postgresq + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/odpf/salt/db" + "github.com/odpf/salt/log" + "github.com/odpf/siren/core/notification" + "github.com/odpf/siren/plugins/queues/postgresq/migrations" +) + +const ( + MESSAGE_QUEUE_SCHEMA_NAME = "notification" + MESSAGE_QUEUE_TABLE_NAME = MESSAGE_QUEUE_SCHEMA_NAME + ".message_queue" +) + +type Strategy string + +const ( + StrategyDefault Strategy = "default" + StrategyDLQ Strategy = "dlq" +) + +type Queue struct { + logger log.Logger + dbc *db.Client + strategy Strategy +} + +var ( + successCallbackQuery = fmt.Sprintf(` +UPDATE %s +SET updated_at = $1, status = $2, try_count = $3 +WHERE id = $4 +`, MESSAGE_QUEUE_TABLE_NAME) + + errorCallbackQuery = fmt.Sprintf(` +UPDATE %s +SET updated_at = $1, status = $2, try_count = $3, last_error = $4, retryable = $5 +WHERE id = $6 +`, MESSAGE_QUEUE_TABLE_NAME) + + queueEnqueueNamedQuery = fmt.Sprintf(` +INSERT INTO %s + (id, status, receiver_type, configs, details, last_error, max_tries, try_count, retryable, + expired_at, created_at, updated_at) + VALUES (:id,:status,:receiver_type,:configs,:details,:last_error,:max_tries,:try_count,:retryable,:expired_at,:created_at,:updated_at) +`, MESSAGE_QUEUE_TABLE_NAME) +) + +func getQueueDequeueQuery(batchSize int, receiverTypesList string) string { + return fmt.Sprintf(` +UPDATE %s +SET status = '%s', updated_at = now() +WHERE id IN ( + SELECT id + FROM %s + WHERE status = '%s' AND (expired_at < now() OR expired_at IS NULL) AND try_count < max_tries %s + ORDER BY expired_at + FOR UPDATE SKIP LOCKED + LIMIT %d +) +RETURNING * +`, MESSAGE_QUEUE_TABLE_NAME, notification.MessageStatusPending, MESSAGE_QUEUE_TABLE_NAME, notification.MessageStatusEnqueued, receiverTypesList, batchSize) +} + +func getDLQDequeueQuery(batchSize int, receiverTypesList string) string { + return fmt.Sprintf(` +UPDATE %s +SET status = '%s', updated_at = now() +WHERE id IN ( + SELECT id + FROM %s + WHERE status = '%s' AND (expired_at < now() OR expired_at IS NULL) AND try_count < max_tries AND retryable IS TRUE %s + ORDER BY expired_at + FOR UPDATE SKIP LOCKED + LIMIT %d +) +RETURNING * +`, MESSAGE_QUEUE_TABLE_NAME, notification.MessageStatusPending, MESSAGE_QUEUE_TABLE_NAME, notification.MessageStatusFailed, receiverTypesList, batchSize) +} + +// New creates a new queue instance +func New(logger log.Logger, dbConfig db.Config, opts ...QueueOption) (*Queue, error) { + q := &Queue{ + logger: logger, + strategy: StrategyDefault, + } + dbClient, err := db.New(dbConfig) + if err != nil { + return nil, fmt.Errorf("error creating postgres queue client: %w", err) + } + q.dbc = dbClient + + // create schema if not exist + _, err = q.dbc.ExecContext(context.Background(), fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", MESSAGE_QUEUE_SCHEMA_NAME)) + if err != nil { + return nil, err + } + + dbConfig.URL = dbConfig.URL + fmt.Sprintf("&search_path=%s", MESSAGE_QUEUE_SCHEMA_NAME) + + if err := db.RunMigrations(dbConfig, migrations.FS, migrations.ResourcePath); err != nil { + return nil, fmt.Errorf("error migrating postgres queue: %w", err) + } + + for _, opt := range opts { + opt(q) + } + + return q, nil +} + +// Dequeue pop the queue based on specific filters (receiver types or batch size) and process the messages with handlerFn +// message left in pending state that has expired or been updated long time ago means there was a failure when transforming row into a struct +func (q *Queue) Dequeue(ctx context.Context, receiverTypes []string, batchSize int, handlerFn func(context.Context, []notification.Message) error) error { + messages := []notification.Message{} + + receiverTypesQuery := getFilterReceiverTypes(receiverTypes) + + var dequeueQuery string + if q.strategy == StrategyDLQ { + dequeueQuery = getDLQDequeueQuery(batchSize, receiverTypesQuery) + } else { + dequeueQuery = getQueueDequeueQuery(batchSize, receiverTypesQuery) + } + rows, err := q.dbc.QueryxContext(ctx, dequeueQuery) + if err != nil { + return err + } + for rows.Next() { + msg := NotificationMessage{} + if err := rows.StructScan(&msg); err != nil { + q.logger.Error("failed to transform message row into struct", "strategy", q.strategy, "error", err) + continue + } + messages = append(messages, msg.ToDomain()) + } + + if len(messages) == 0 { + return notification.ErrNoMessage + } else { + q.logger.Debug(fmt.Sprintf("dequeued %d messages with batch size %d", len(messages), batchSize), "strategy", q.strategy) + if err := handlerFn(ctx, messages); err != nil { + return fmt.Errorf("error processing dequeued message: %w", err) + } + } + + return nil +} + +// Enqueue pushes messages to the queue +func (q *Queue) Enqueue(ctx context.Context, ms ...notification.Message) error { + messages := []NotificationMessage{} + for _, m := range ms { + message := &NotificationMessage{} + message.FromDomain(m) + messages = append(messages, *message) + } + + res, err := q.dbc.NamedExecContext(ctx, queueEnqueueNamedQuery, messages) + if err != nil { + return err + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return errors.New("no rows affected when enqueueing messages") + } + return nil +} + +// SuccessCallback is a callback that will be called once the message is succesfully handled by handlerFn +func (q *Queue) SuccessCallback(ctx context.Context, ms notification.Message) error { + q.logger.Debug("marking a message as published", "strategy", q.strategy, "id", ms.ID) + res, err := q.dbc.ExecContext(ctx, successCallbackQuery, ms.UpdatedAt, ms.Status, ms.TryCount, ms.ID) + if err != nil { + return err + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return errors.New("no rows affected when marking row as published") + } + q.logger.Debug("marked a message as published", "strategy", q.strategy, "id", ms.ID) + return nil +} + +// ErrorCallback is a callback that will be called once the message is failed to be handled by handlerFn +func (q *Queue) ErrorCallback(ctx context.Context, ms notification.Message) error { + q.logger.Debug("marking a message as failed with", "strategy", q.strategy, "id", ms.ID) + res, err := q.dbc.ExecContext(ctx, errorCallbackQuery, ms.UpdatedAt, ms.Status, ms.TryCount, ms.LastError, ms.Retryable, ms.ID) + if err != nil { + return err + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return errors.New("no rows affected when marking row as failed") + } + q.logger.Debug("marked a message as failed with", "strategy", q.strategy, "id", ms.ID) + return nil +} + +// Stop will close the db +func (q *Queue) Stop(ctx context.Context) error { + return q.dbc.Close() +} + +func getFilterReceiverTypes(receiverTypes []string) string { + var receiverTypesQuery = "" + if len(receiverTypes) > 0 { + receiverTypesQuery = "AND receiver_type IN (" + for _, rs := range receiverTypes { + receiverTypesQuery += "'" + receiverTypesQuery += rs + receiverTypesQuery += "'" + receiverTypesQuery += "," + } + receiverTypesQuery = strings.TrimSuffix(receiverTypesQuery, ",") + receiverTypesQuery += ")" + } + return receiverTypesQuery +} diff --git a/plugins/queues/postgresq/queue_test.go b/plugins/queues/postgresq/queue_test.go new file mode 100644 index 00000000..2e1ff015 --- /dev/null +++ b/plugins/queues/postgresq/queue_test.go @@ -0,0 +1,281 @@ +package postgresq_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/google/uuid" + "github.com/odpf/salt/db" + "github.com/odpf/salt/dockertest" + "github.com/odpf/salt/log" + "github.com/odpf/siren/core/notification" + "github.com/odpf/siren/core/receiver" + "github.com/odpf/siren/plugins/queues/postgresq" + "github.com/odpf/siren/plugins/queues/postgresq/migrations" + "github.com/stretchr/testify/suite" +) + +type QueueTestSuite struct { + suite.Suite + logger log.Logger + ctx context.Context + dbc *db.Client + pool *dockertest.Pool + resource *dockertest.Resource + q *postgresq.Queue + dlq *postgresq.Queue +} + +func (s *QueueTestSuite) SetupSuite() { + var ( + err error + pgUser = "test_user" + pgPass = "test_pass" + pgDBName = "test_db" + ) + + s.logger = log.NewZap() + dpg, err := dockertest.CreatePostgres( + dockertest.PostgresWithDetail( + pgUser, pgPass, pgDBName, + ), + ) + if err != nil { + s.T().Fatal(err) + } + + s.pool = dpg.GetPool() + s.resource = dpg.GetResource() + + dbConfig := db.Config{ + Driver: "postgres", + } + dbConfig.URL = dpg.GetExternalConnString() + s.dbc, err = db.New(dbConfig) + if err != nil { + s.T().Fatal(err) + } + + s.ctx = context.TODO() + err = db.RunMigrations(dbConfig, migrations.FS, migrations.ResourcePath) + if err != nil { + s.T().Fatal(err) + } + + s.q, err = postgresq.New(s.logger, dbConfig) + if err != nil { + s.T().Fatal(err) + } + + s.dlq, err = postgresq.New(s.logger, dbConfig, postgresq.WithStrategy(postgresq.StrategyDLQ)) + if err != nil { + s.T().Fatal(err) + } +} + +func (s *QueueTestSuite) TearDownSuite() { + s.q.Stop(s.ctx) + // Clean tests + if err := s.pool.Purge(s.resource); err != nil { + s.T().Fatal(err) + } +} + +func (s *QueueTestSuite) cleanup() error { + _, err := s.dbc.Exec(fmt.Sprintf("TRUNCATE TABLE %s RESTART IDENTITY CASCADE", postgresq.MESSAGE_QUEUE_TABLE_NAME)) + if err != nil { + return err + } + return nil +} + +func (s *QueueTestSuite) TestSimpleEnqueueDequeue() { + timeNow := time.Now() + ns := []notification.Notification{ + { + ID: uuid.NewString(), + Data: map[string]interface{}{}, + Labels: map[string]string{}, + CreatedAt: timeNow, + }, + { + ID: uuid.NewString(), + Data: map[string]interface{}{}, + Labels: map[string]string{}, + CreatedAt: timeNow, + }, + { + ID: uuid.NewString(), + Data: map[string]interface{}{}, + Labels: map[string]string{}, + CreatedAt: timeNow, + }, + { + ID: uuid.NewString(), + Data: map[string]interface{}{}, + Labels: map[string]string{}, + CreatedAt: timeNow, + }, + } + + messages := []notification.Message{} + for _, n := range ns { + msg, err := n.ToMessage(receiver.TypeSlack, map[string]interface{}{}) + s.Require().NoError(err) + messages = append(messages, *msg) + } + + s.Run("should return no error if all messages are successfully processed", func() { + handlerFn := func(ctx context.Context, messages []notification.Message) error { + s.Assert().Len(messages, 1) + return nil + } + + err := s.q.Enqueue(s.ctx, messages...) + s.Require().NoError(err) + + for i := 0; i < len(messages); i++ { + _ = s.q.Dequeue(s.ctx, nil, 1, handlerFn) + } + + err = s.cleanup() + s.Require().NoError(err) + }) + + s.Run("should return no error if all messages are successfully processed with different batch", func() { + handlerFn := func(ctx context.Context, messages []notification.Message) error { + s.Assert().Len(messages, 2) + return nil + } + + err := s.q.Enqueue(s.ctx, messages...) + s.Require().NoError(err) + + for i := 0; i < 2; i++ { + _ = s.q.Dequeue(s.ctx, nil, 2, handlerFn) + } + + err = s.cleanup() + s.Require().NoError(err) + }) + + s.Run("should return an error if a message is failed to process", func() { + handlerFn := func(ctx context.Context, messages []notification.Message) error { + return errors.New("some error") + } + + err := s.q.Enqueue(s.ctx, messages...) + s.Require().NoError(err) + + for i := 0; i < len(messages); i++ { + err := s.q.Dequeue(s.ctx, nil, 1, handlerFn) + s.Assert().Error(errors.New("error processing dequeued message: some error"), err) + } + + err = s.cleanup() + s.Require().NoError(err) + }) +} + +func (s *QueueTestSuite) TestEnqueueDequeueWithCallback() { + messages := make([]notification.Message, 5) + + for i := 0; i < len(messages); i++ { + messages[i].Initialize(notification.Notification{}, receiver.TypeSlack, map[string]interface{}{}, notification.InitWithID(fmt.Sprintf("%d", i+1))) + } + + s.Run("should update row with error for id \"5\"", func() { + var anError = errors.New("some error") + + err := s.q.Enqueue(s.ctx, messages...) + s.Require().NoError(err) + + for _, m := range messages { + if m.ID == "5" { + m.MarkFailed(time.Now(), true, anError) + err = s.q.ErrorCallback(s.ctx, m) + s.Assert().NoError(err) + } + } + + tempMessage := &postgresq.NotificationMessage{} + err = s.dbc.Get(tempMessage, fmt.Sprintf("SELECT * FROM %s WHERE id = '5'", postgresq.MESSAGE_QUEUE_TABLE_NAME)) + s.Require().NoError(err) + + s.Assert().Equal(string(notification.MessageStatusFailed), tempMessage.Status) + s.Assert().Equal(anError.Error(), tempMessage.LastError.String) + s.Assert().Equal(1, tempMessage.TryCount) + + err = s.cleanup() + s.Require().NoError(err) + }) + + s.Run("should update row with when successfully published", func() { + err := s.q.Enqueue(s.ctx, messages...) + s.Require().NoError(err) + + for _, m := range messages { + m.MarkPublished(time.Now()) + err = s.q.SuccessCallback(s.ctx, m) + s.Assert().NoError(err) + } + + tempMessage := &postgresq.NotificationMessage{} + err = s.dbc.Get(tempMessage, fmt.Sprintf("SELECT * FROM %s LIMIT 1", postgresq.MESSAGE_QUEUE_TABLE_NAME)) + s.Require().NoError(err) + + s.Assert().Equal(string(notification.MessageStatusPublished), tempMessage.Status) + s.Assert().Equal(1, tempMessage.TryCount) + + err = s.cleanup() + s.Require().NoError(err) + }) +} + +func (s *QueueTestSuite) TestEnqueueDequeueDLQ() { + messages := make([]notification.Message, 5) + + for i := 0; i < len(messages); i++ { + messages[i].Initialize(notification.Notification{}, receiver.TypeSlack, map[string]interface{}{}, notification.InitWithID(fmt.Sprintf("%d", i+1))) + } + + s.Run("failed messages should be re-processed by dlq an ignored by main queue", func() { + var anError = errors.New("some error") + + err := s.q.Enqueue(s.ctx, messages...) + s.Require().NoError(err) + + // mark failed all + for _, m := range messages { + m.MarkFailed(time.Now(), true, anError) + err = s.q.ErrorCallback(s.ctx, m) + s.Assert().NoError(err) + } + + _ = s.q.Dequeue(s.ctx, nil, 5, func(ctx context.Context, m []notification.Message) error { s.Assert().Empty(m); return nil }) + s.Assert().NoError(err) + + _ = s.dlq.Dequeue(s.ctx, nil, 5, func(ctx context.Context, m []notification.Message) error { + s.Assert().Len(m, 5) + return nil + }) + + tempMessage := &postgresq.NotificationMessage{} + err = s.dbc.Get(tempMessage, fmt.Sprintf("SELECT * FROM %s LIMIT 1", postgresq.MESSAGE_QUEUE_TABLE_NAME)) + s.Require().NoError(err) + + s.Assert().Equal(string(notification.MessageStatusPending), tempMessage.Status) + s.Assert().Equal(anError.Error(), tempMessage.LastError.String) + s.Assert().Equal(1, tempMessage.TryCount) + + err = s.cleanup() + s.Require().NoError(err) + }) +} + +func TestQueue(t *testing.T) { + suite.Run(t, new(QueueTestSuite)) +} diff --git a/plugins/receivers/config.go b/plugins/receivers/config.go index d958341d..f6eeb017 100644 --- a/plugins/receivers/config.go +++ b/plugins/receivers/config.go @@ -7,7 +7,7 @@ import ( ) type Config struct { - Slack slack.AppConfig `mapstructure:"slack" yaml:"slack"` - Pagerduty pagerduty.AppConfig `mapstructure:"pagerduty" yaml:"pagerduty"` - HTTPReceiver httpreceiver.AppConfig `mapstructure:"http" yaml:"http"` + Slack slack.AppConfig `mapstructure:"slack"` + Pagerduty pagerduty.AppConfig `mapstructure:"pagerduty"` + HTTPReceiver httpreceiver.AppConfig `mapstructure:"http"` } diff --git a/plugins/receivers/httpreceiver/notification_service.go b/plugins/receivers/httpreceiver/notification_service.go index 8764c65a..3e965dd8 100644 --- a/plugins/receivers/httpreceiver/notification_service.go +++ b/plugins/receivers/httpreceiver/notification_service.go @@ -58,6 +58,14 @@ func (h *HTTPNotificationService) Publish(ctx context.Context, notificationMessa return false, nil } +func (h *HTTPNotificationService) PreHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) { + return notificationConfigMap, nil +} + +func (h *HTTPNotificationService) PostHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) { + return notificationConfigMap, nil +} + func (h *HTTPNotificationService) DefaultTemplateOfProvider(providerType string) string { switch providerType { case provider.TypeCortex: diff --git a/plugins/receivers/pagerduty/config.go b/plugins/receivers/pagerduty/config.go index 4d02dded..12dd570d 100644 --- a/plugins/receivers/pagerduty/config.go +++ b/plugins/receivers/pagerduty/config.go @@ -6,6 +6,7 @@ import ( "github.com/odpf/siren/pkg/httpclient" "github.com/odpf/siren/pkg/retry" + "github.com/odpf/siren/pkg/secret" ) // AppConfig is a config loaded when siren is started @@ -24,7 +25,7 @@ func (c AppConfig) Validate() error { // TODO need to support versioning later v1 and v2 type ReceiverConfig struct { - ServiceKey string `mapstructure:"service_key"` + ServiceKey secret.MaskableString `mapstructure:"service_key"` } func (c *ReceiverConfig) Validate() error { diff --git a/plugins/receivers/pagerduty/config/default_cortex_alert_template_body_v1.goyaml b/plugins/receivers/pagerduty/config/default_cortex_alert_template_body_v1.goyaml index 99c38e80..9b3ad61a 100644 --- a/plugins/receivers/pagerduty/config/default_cortex_alert_template_body_v1.goyaml +++ b/plugins/receivers/pagerduty/config/default_cortex_alert_template_body_v1.goyaml @@ -1,7 +1,7 @@ [[define "pagerduty.event_type" -]] - [[if eq .Variables.status "firing" -]] + [[if eq .Data.status "firing" -]] trigger - [[- else if eq .Variables.status "resolved" -]] + [[- else if eq .Data.status "resolved" -]] resolve [[- else -]] unknown @@ -12,12 +12,12 @@ Labels:\n [[ range $index, $element := .Labels ]]- [[ $index ]] = [[ $element ]]\n [[ end ]] Annotations:\n -[[ range $index, $element := .Variables ]]- [[ $index ]] = [[ $element ]]\n -[[ end ]][[if .Variables.generator_url ]]Source: [[ .Variables.generator_url ]][[ end ]] +[[ range $index, $element := .Data ]]- [[ $index ]] = [[ $element ]]\n +[[ end ]][[if .Data.generatorUrl ]]Source: [[ .Data.generatorUrl ]][[ end ]] [[ end ]] event_type: "[[template "pagerduty.event_type" . ]]" -[[if .Variables.incident_key]]incident_key: "[[.Variables.incident_key]]"[[ end ]] -description: ([[ .Variables.status | toUpper ]][[ if .Variables.num_alerts_firing ]][[ if eq .Variables.status "firing" ]]:[[ .Variables.num_alerts_firing ]][[ end ]][[ end ]]) [[ .Labels | joinStringValues " " ]] +[[if .Data.incidentKey]]incident_key: "[[.Data.incidentKey]]"[[ end ]] +description: ([[ .Data.status | toUpper ]][[ if .Data.numAlertsFiring ]][[ if eq .Data.status "firing" ]]:[[ .Data.numAlertsFiring ]][[ end ]][[ end ]]) [[ .Labels | joinStringValues " " ]] client: "Siren" details: - [[ .Variables.status ]]: "[[template "pagerduty.details" . ]]" \ No newline at end of file + [[ .Data.status ]]: "[[template "pagerduty.details" . ]]" \ No newline at end of file diff --git a/plugins/receivers/pagerduty/config_test.go b/plugins/receivers/pagerduty/config_test.go index 9605171b..22a56bfd 100644 --- a/plugins/receivers/pagerduty/config_test.go +++ b/plugins/receivers/pagerduty/config_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/odpf/siren/pkg/secret" ) func TestReceiverConfig(t *testing.T) { @@ -68,12 +69,12 @@ func TestNotificationConfig(t *testing.T) { t.Run("AsMap", func(t *testing.T) { nc := NotificationConfig{ ReceiverConfig: ReceiverConfig{ - ServiceKey: "service_key", + ServiceKey: secret.MaskableString("service_key"), }, } if diff := cmp.Diff(map[string]interface{}{ - "service_key": "service_key", + "service_key": secret.MaskableString("service_key"), }, nc.AsMap()); diff != "" { t.Errorf("result not match\n%v", diff) } diff --git a/plugins/receivers/pagerduty/messagev1.go b/plugins/receivers/pagerduty/messagev1.go index c5f31c07..21551175 100644 --- a/plugins/receivers/pagerduty/messagev1.go +++ b/plugins/receivers/pagerduty/messagev1.go @@ -1,8 +1,10 @@ package pagerduty +import "github.com/odpf/siren/pkg/secret" + // https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTc4-send-a-v1-event type MessageV1 struct { - ServiceKey string `mapstructure:"service_key" yaml:"service_key,omitempty" json:"service_key,omitempty"` + ServiceKey secret.MaskableString `mapstructure:"service_key" yaml:"service_key,omitempty" json:"service_key,omitempty"` EventType string `mapstructure:"event_type" yaml:"event_type,omitempty" json:"event_type,omitempty"` IncidentKey string `mapstructure:"incident_key" yaml:"incident_key,omitempty" json:"incident_key,omitempty"` Description string `mapstructure:"description" yaml:"description,omitempty" json:"description,omitempty"` diff --git a/plugins/receivers/pagerduty/notification_service.go b/plugins/receivers/pagerduty/notification_service.go index d1cd2384..85a77b61 100644 --- a/plugins/receivers/pagerduty/notification_service.go +++ b/plugins/receivers/pagerduty/notification_service.go @@ -58,6 +58,14 @@ func (pd *PagerDutyNotificationService) Publish(ctx context.Context, notificatio return false, nil } +func (pd *PagerDutyNotificationService) PreHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) { + return notificationConfigMap, nil +} + +func (pd *PagerDutyNotificationService) PostHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) { + return notificationConfigMap, nil +} + func (pd *PagerDutyNotificationService) DefaultTemplateOfProvider(providerType string) string { switch providerType { case provider.TypeCortex: diff --git a/plugins/receivers/pagerduty/receiver_service_test.go b/plugins/receivers/pagerduty/receiver_service_test.go index a1694fdd..76b00c0c 100644 --- a/plugins/receivers/pagerduty/receiver_service_test.go +++ b/plugins/receivers/pagerduty/receiver_service_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/odpf/siren/pkg/secret" "github.com/odpf/siren/plugins/receivers/pagerduty" ) @@ -55,10 +56,10 @@ func TestPagerDutyService_BuildNotificationConfig(t *testing.T) { { Description: "should return configs with service_key if receiver 'service_key' exist in string", ReceiverConfigs: map[string]interface{}{ - "service_key": "service_key", + "service_key": secret.MaskableString("service_key"), }, ExpectedConfigMap: map[string]interface{}{ - "service_key": "service_key", + "service_key": secret.MaskableString("service_key"), }, }, } diff --git a/plugins/receivers/slack/client.go b/plugins/receivers/slack/client.go index aba9a884..dd0d4940 100644 --- a/plugins/receivers/slack/client.go +++ b/plugins/receivers/slack/client.go @@ -12,6 +12,7 @@ import ( "github.com/odpf/siren/pkg/errors" "github.com/odpf/siren/pkg/httpclient" "github.com/odpf/siren/pkg/retry" + "github.com/odpf/siren/pkg/secret" goslack "github.com/slack-go/slack" ) @@ -28,7 +29,7 @@ type GoSlackCaller interface { } type codeExchangeHTTPResponse struct { - AccessToken string `json:"access_token"` + AccessToken secret.MaskableString `json:"access_token"` Team struct { Name string `json:"name"` } `json:"team"` @@ -41,7 +42,7 @@ type Channel struct { } type Credential struct { - AccessToken string + AccessToken secret.MaskableString TeamName string } @@ -115,12 +116,8 @@ func (c *Client) ExchangeAuth(ctx context.Context, authCode, clientID, clientSec } // GetWorkspaceChannels fetches list of joined channel of a client -func (c *Client) GetWorkspaceChannels(ctx context.Context, token string) ([]Channel, error) { - // gsc, err := c.createGoSlackClient(ctx, opts...) - gsc := goslack.New(token, goslack.OptionAPIURL(c.cfg.APIHost)) - // if err != nil { - // return nil, fmt.Errorf("goslack client creation failure: %w", err) - // } +func (c *Client) GetWorkspaceChannels(ctx context.Context, token secret.MaskableString) ([]Channel, error) { + gsc := goslack.New(token.UnmaskedString(), goslack.OptionAPIURL(c.cfg.APIHost)) joinedChannelList, err := c.getJoinedChannelsList(ctx, gsc) if err != nil { @@ -139,7 +136,7 @@ func (c *Client) GetWorkspaceChannels(ctx context.Context, token string) ([]Chan // Notify sends message to a specific slack channel func (c *Client) Notify(ctx context.Context, conf NotificationConfig, message Message) error { - gsc := goslack.New(conf.ReceiverConfig.Token, goslack.OptionAPIURL(c.cfg.APIHost)) + gsc := goslack.New(conf.ReceiverConfig.Token.UnmaskedString(), goslack.OptionAPIURL(c.cfg.APIHost)) var channelID string switch conf.ChannelType { diff --git a/plugins/receivers/slack/client_test.go b/plugins/receivers/slack/client_test.go index 26ee2568..59ed3bff 100644 --- a/plugins/receivers/slack/client_test.go +++ b/plugins/receivers/slack/client_test.go @@ -8,13 +8,14 @@ import ( "testing" "github.com/odpf/siren/pkg/retry" + "github.com/odpf/siren/pkg/secret" "github.com/odpf/siren/plugins/receivers/slack" goslack "github.com/slack-go/slack" "github.com/stretchr/testify/assert" ) func TestClient_GetWorkspaceChannels(t *testing.T) { - var token = "test-token" + var token = secret.MaskableString("test-token") t.Run("return error when failed to fetch joined channel list", func(t *testing.T) { testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -67,7 +68,7 @@ func TestClient_GetWorkspaceChannels(t *testing.T) { } func TestClient_NotifyChannel(t *testing.T) { - var token = "test-token" + var token = secret.MaskableString("test-token") t.Run("return error when message receiver type is wrong", func(t *testing.T) { c := slack.NewClient(slack.AppConfig{}) @@ -242,7 +243,7 @@ func TestClient_NotifyChannel(t *testing.T) { } func TestClient_NotifyUser(t *testing.T) { - var token = "test-token" + var token = secret.MaskableString("test-token") t.Run("return error when failed to get user for an email", func(t *testing.T) { testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -322,7 +323,7 @@ func TestClient_NotifyUser(t *testing.T) { func TestClient_NotifyWithRetrier(t *testing.T) { var ( expectedCounter = 4 - token = "test-token" + token = secret.MaskableString("test-token") ) t.Run("when 429 is returned", func(t *testing.T) { diff --git a/plugins/receivers/slack/config.go b/plugins/receivers/slack/config.go index 3e457acc..9dd7aff5 100644 --- a/plugins/receivers/slack/config.go +++ b/plugins/receivers/slack/config.go @@ -5,6 +5,7 @@ import ( "github.com/odpf/siren/pkg/httpclient" "github.com/odpf/siren/pkg/retry" + "github.com/odpf/siren/pkg/secret" ) // AppConfig is a config loaded when siren is started @@ -31,15 +32,15 @@ func (c *SlackCredentialConfig) Validate() error { // ReceiverConfig is a stored config for a slack receiver type ReceiverConfig struct { - Token string `json:"token" mapstructure:"token"` - Workspace string `json:"workspace" mapstructure:"workspace"` + Token secret.MaskableString `json:"token" mapstructure:"token"` + Workspace string `json:"workspace" mapstructure:"workspace"` } func (c *ReceiverConfig) Validate() error { if c.Token != "" && c.Workspace != "" { return nil } - return fmt.Errorf("invalid slack receiver config, workspace: %s, token: ", c.Workspace) + return fmt.Errorf("invalid slack receiver config, workspace: %s, token: %s", c.Workspace, c.Token) } func (c *ReceiverConfig) AsMap() map[string]interface{} { @@ -78,11 +79,14 @@ type NotificationConfig struct { SubscriptionConfig `mapstructure:",squash"` } +// Validate validates whether notification config contains required fields or not +// channel_name is not mandatory because in NotifyToReceiver flow, channel_name +// is being passed from the request (not from the config) func (c *NotificationConfig) Validate() error { - if c.Token != "" && c.Workspace != "" && c.ChannelName != "" { + if c.Token != "" && c.Workspace != "" { return nil } - return fmt.Errorf("invalid slack notification config, workspace: %s, token: , channel_name: %s", c.Workspace, c.ChannelName) + return fmt.Errorf("invalid slack notification config, workspace: %s, token: %s, channel_name: %s", c.Workspace, c.Token, c.ChannelName) } func (c *NotificationConfig) AsMap() map[string]interface{} { diff --git a/plugins/receivers/slack/config/default_cortex_alert_template_body.goyaml b/plugins/receivers/slack/config/default_cortex_alert_template_body.goyaml index ef074efc..e196f39f 100644 --- a/plugins/receivers/slack/config/default_cortex_alert_template_body.goyaml +++ b/plugins/receivers/slack/config/default_cortex_alert_template_body.goyaml @@ -1,5 +1,5 @@ [[define "__alert_severity_prefix_emoji" -]] - [[if ne .Variables.status "firing" -]] + [[if ne .Data.status "firing" -]] :white_check_mark: [[- else if eq .Labels.severity "CRITICAL" -]] :fire: @@ -10,13 +10,13 @@ [[- end]] [[- end]] [[ define "slack.pretext" -]] - [[- template "__alert_severity_prefix_emoji" . ]] ([[ .Variables.status | toUpper ]][[ if eq .Variables.status "firing" ]]:[[ .Variables.num_alerts_firing ]][[ end ]]) - [[- if eq .Variables.status "resolved" ]] ~([[ .Labels.severity | toUpper ]])~ + [[- template "__alert_severity_prefix_emoji" . ]] ([[ .Data.status | toUpper ]][[ if eq .Data.status "firing" ]]:[[ .Data.numAlertsFiring ]][[ end ]]) + [[- if eq .Data.status "resolved" ]] ~([[ .Labels.severity | toUpper ]])~ [[- else ]] *([[ .Labels.severity | toUpper ]])* [[- end]] [[ .Labels.alertname ]] [[- end ]] [[define "slack.color" -]] -[[if eq .Variables.status "firing" -]] +[[if eq .Data.status "firing" -]] [[if eq .Labels.severity "WARNING" -]] warning [[- else if eq .Labels.severity "CRITICAL" -]] @@ -28,14 +28,14 @@ good [[- end]] [[- end]] -[[ define "slack.body"]] -[[- .Variables.summary -]] +[[define "slack.body"]] +[[- .Labels.summary -]] [[ end]] [[define "slack.dashboard"]] -[[- if .Variables.dashboard]][[.Variables.dashboard]][[else]]https://radar.odpf.io[[end]] +[[- if .Data.dashboard]][[.Data.dashboard]][[else]][[.Data.defaultDashboard]][[end]] [[- end -]] [[define "slack.runbook"]] -[[- if .Variables.playbook]][[.Variables.playbook]][[end]] +[[- if .Data.playbook]][[.Data.playbook]][[end]] [[- end -]] username: "Siren" icon_emoji: ":eagle:" @@ -43,7 +43,7 @@ link_names: false attachments: - title: "" pretext: "[[template "slack.pretext" . ]]" - text: "[[ template "slack.body" . ]]" + text: "[[template "slack.body" . ]]" color: "[[template "slack.color" . ]]" actions: - type: button diff --git a/plugins/receivers/slack/config_test.go b/plugins/receivers/slack/config_test.go index a50b60a1..e3d54295 100644 --- a/plugins/receivers/slack/config_test.go +++ b/plugins/receivers/slack/config_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/odpf/siren/pkg/secret" ) func TestSlackCredentialConfig(t *testing.T) { @@ -106,13 +107,13 @@ func TestNotificationConfig(t *testing.T) { ChannelName: "channel", }, ReceiverConfig: ReceiverConfig{ - Token: "token", + Token: secret.MaskableString("token"), Workspace: "workspace"}, } if diff := cmp.Diff(map[string]interface{}{ "channel_name": "channel", - "token": "token", + "token": secret.MaskableString("token"), "workspace": "workspace", }, nc.AsMap()); diff != "" { t.Errorf("result not match\n%v", diff) diff --git a/plugins/receivers/slack/mocks/encryptor.go b/plugins/receivers/slack/mocks/encryptor.go index 6659fae9..c893a084 100644 --- a/plugins/receivers/slack/mocks/encryptor.go +++ b/plugins/receivers/slack/mocks/encryptor.go @@ -2,7 +2,10 @@ package mocks -import mock "github.com/stretchr/testify/mock" +import ( + secret "github.com/odpf/siren/pkg/secret" + mock "github.com/stretchr/testify/mock" +) // Encryptor is an autogenerated mock type for the Encryptor type type Encryptor struct { @@ -18,18 +21,18 @@ func (_m *Encryptor) EXPECT() *Encryptor_Expecter { } // Decrypt provides a mock function with given fields: str -func (_m *Encryptor) Decrypt(str string) (string, error) { +func (_m *Encryptor) Decrypt(str secret.MaskableString) (secret.MaskableString, error) { ret := _m.Called(str) - var r0 string - if rf, ok := ret.Get(0).(func(string) string); ok { + var r0 secret.MaskableString + if rf, ok := ret.Get(0).(func(secret.MaskableString) secret.MaskableString); ok { r0 = rf(str) } else { - r0 = ret.Get(0).(string) + r0 = ret.Get(0).(secret.MaskableString) } var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { + if rf, ok := ret.Get(1).(func(secret.MaskableString) error); ok { r1 = rf(str) } else { r1 = ret.Error(1) @@ -44,36 +47,36 @@ type Encryptor_Decrypt_Call struct { } // Decrypt is a helper method to define mock.On call -// - str string +// - str secret.MaskableString func (_e *Encryptor_Expecter) Decrypt(str interface{}) *Encryptor_Decrypt_Call { return &Encryptor_Decrypt_Call{Call: _e.mock.On("Decrypt", str)} } -func (_c *Encryptor_Decrypt_Call) Run(run func(str string)) *Encryptor_Decrypt_Call { +func (_c *Encryptor_Decrypt_Call) Run(run func(str secret.MaskableString)) *Encryptor_Decrypt_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) + run(args[0].(secret.MaskableString)) }) return _c } -func (_c *Encryptor_Decrypt_Call) Return(_a0 string, _a1 error) *Encryptor_Decrypt_Call { +func (_c *Encryptor_Decrypt_Call) Return(_a0 secret.MaskableString, _a1 error) *Encryptor_Decrypt_Call { _c.Call.Return(_a0, _a1) return _c } // Encrypt provides a mock function with given fields: str -func (_m *Encryptor) Encrypt(str string) (string, error) { +func (_m *Encryptor) Encrypt(str secret.MaskableString) (secret.MaskableString, error) { ret := _m.Called(str) - var r0 string - if rf, ok := ret.Get(0).(func(string) string); ok { + var r0 secret.MaskableString + if rf, ok := ret.Get(0).(func(secret.MaskableString) secret.MaskableString); ok { r0 = rf(str) } else { - r0 = ret.Get(0).(string) + r0 = ret.Get(0).(secret.MaskableString) } var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { + if rf, ok := ret.Get(1).(func(secret.MaskableString) error); ok { r1 = rf(str) } else { r1 = ret.Error(1) @@ -88,19 +91,19 @@ type Encryptor_Encrypt_Call struct { } // Encrypt is a helper method to define mock.On call -// - str string +// - str secret.MaskableString func (_e *Encryptor_Expecter) Encrypt(str interface{}) *Encryptor_Encrypt_Call { return &Encryptor_Encrypt_Call{Call: _e.mock.On("Encrypt", str)} } -func (_c *Encryptor_Encrypt_Call) Run(run func(str string)) *Encryptor_Encrypt_Call { +func (_c *Encryptor_Encrypt_Call) Run(run func(str secret.MaskableString)) *Encryptor_Encrypt_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string)) + run(args[0].(secret.MaskableString)) }) return _c } -func (_c *Encryptor_Encrypt_Call) Return(_a0 string, _a1 error) *Encryptor_Encrypt_Call { +func (_c *Encryptor_Encrypt_Call) Return(_a0 secret.MaskableString, _a1 error) *Encryptor_Encrypt_Call { _c.Call.Return(_a0, _a1) return _c } diff --git a/plugins/receivers/slack/mocks/slack_caller.go b/plugins/receivers/slack/mocks/slack_caller.go index 4ac4b901..f8371aef 100644 --- a/plugins/receivers/slack/mocks/slack_caller.go +++ b/plugins/receivers/slack/mocks/slack_caller.go @@ -5,8 +5,10 @@ package mocks import ( context "context" - slack "github.com/odpf/siren/plugins/receivers/slack" + secret "github.com/odpf/siren/pkg/secret" mock "github.com/stretchr/testify/mock" + + slack "github.com/odpf/siren/plugins/receivers/slack" ) // SlackCaller is an autogenerated mock type for the SlackCaller type @@ -70,11 +72,11 @@ func (_c *SlackCaller_ExchangeAuth_Call) Return(_a0 slack.Credential, _a1 error) } // GetWorkspaceChannels provides a mock function with given fields: ctx, token -func (_m *SlackCaller) GetWorkspaceChannels(ctx context.Context, token string) ([]slack.Channel, error) { +func (_m *SlackCaller) GetWorkspaceChannels(ctx context.Context, token secret.MaskableString) ([]slack.Channel, error) { ret := _m.Called(ctx, token) var r0 []slack.Channel - if rf, ok := ret.Get(0).(func(context.Context, string) []slack.Channel); ok { + if rf, ok := ret.Get(0).(func(context.Context, secret.MaskableString) []slack.Channel); ok { r0 = rf(ctx, token) } else { if ret.Get(0) != nil { @@ -83,7 +85,7 @@ func (_m *SlackCaller) GetWorkspaceChannels(ctx context.Context, token string) ( } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, secret.MaskableString) error); ok { r1 = rf(ctx, token) } else { r1 = ret.Error(1) @@ -99,14 +101,14 @@ type SlackCaller_GetWorkspaceChannels_Call struct { // GetWorkspaceChannels is a helper method to define mock.On call // - ctx context.Context -// - token string +// - token secret.MaskableString func (_e *SlackCaller_Expecter) GetWorkspaceChannels(ctx interface{}, token interface{}) *SlackCaller_GetWorkspaceChannels_Call { return &SlackCaller_GetWorkspaceChannels_Call{Call: _e.mock.On("GetWorkspaceChannels", ctx, token)} } -func (_c *SlackCaller_GetWorkspaceChannels_Call) Run(run func(ctx context.Context, token string)) *SlackCaller_GetWorkspaceChannels_Call { +func (_c *SlackCaller_GetWorkspaceChannels_Call) Run(run func(ctx context.Context, token secret.MaskableString)) *SlackCaller_GetWorkspaceChannels_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string)) + run(args[0].(context.Context), args[1].(secret.MaskableString)) }) return _c } diff --git a/plugins/receivers/slack/notification_service.go b/plugins/receivers/slack/notification_service.go index fc467d2e..c0acc10d 100644 --- a/plugins/receivers/slack/notification_service.go +++ b/plugins/receivers/slack/notification_service.go @@ -2,6 +2,7 @@ package slack import ( "context" + "fmt" "github.com/mitchellh/mapstructure" "github.com/odpf/siren/core/notification" @@ -19,13 +20,15 @@ const ( // SlackNotificationService is a notification plugin service layer for slack type SlackNotificationService struct { - client SlackCaller + cryptoClient Encryptor + client SlackCaller } // NewNotificationService returns slack service struct. This service implement [receiver.Notifier] interface. -func NewNotificationService(client SlackCaller) *SlackNotificationService { +func NewNotificationService(client SlackCaller, cryptoClient Encryptor) *SlackNotificationService { return &SlackNotificationService{ - client: client, + client: client, + cryptoClient: cryptoClient, } } @@ -71,6 +74,46 @@ func (s *SlackNotificationService) Publish(ctx context.Context, notificationMess return false, nil } +func (s *SlackNotificationService) PreHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) { + notificationConfig := &NotificationConfig{} + if err := mapstructure.Decode(notificationConfigMap, notificationConfig); err != nil { + return nil, fmt.Errorf("failed to transform configurations to slack notification config: %w", err) + } + + if err := notificationConfig.Validate(); err != nil { + return nil, err + } + + cipher, err := s.cryptoClient.Encrypt(notificationConfig.Token) + if err != nil { + return nil, fmt.Errorf("slack token encryption failed: %w", err) + } + + notificationConfig.Token = cipher + + return notificationConfig.AsMap(), nil +} + +func (s *SlackNotificationService) PostHookTransformConfigs(ctx context.Context, notificationConfigMap map[string]interface{}) (map[string]interface{}, error) { + notificationConfig := &NotificationConfig{} + if err := mapstructure.Decode(notificationConfigMap, notificationConfig); err != nil { + return nil, fmt.Errorf("failed to transform configurations to notification config: %w", err) + } + + if err := notificationConfig.Validate(); err != nil { + return nil, err + } + + token, err := s.cryptoClient.Decrypt(notificationConfig.Token) + if err != nil { + return nil, fmt.Errorf("slack token decryption failed: %w", err) + } + + notificationConfig.Token = token + + return notificationConfig.AsMap(), nil +} + func (s *SlackNotificationService) DefaultTemplateOfProvider(templateName string) string { switch templateName { case template.ReservedName_DefaultCortex: diff --git a/plugins/receivers/slack/notification_service_test.go b/plugins/receivers/slack/notification_service_test.go index b78426f8..e3a19331 100644 --- a/plugins/receivers/slack/notification_service_test.go +++ b/plugins/receivers/slack/notification_service_test.go @@ -2,11 +2,13 @@ package slack_test import ( "context" - "errors" + "reflect" "testing" "github.com/odpf/siren/core/notification" + "github.com/odpf/siren/pkg/errors" "github.com/odpf/siren/pkg/retry" + "github.com/odpf/siren/pkg/secret" "github.com/odpf/siren/plugins/receivers/slack" "github.com/odpf/siren/plugins/receivers/slack/mocks" "github.com/stretchr/testify/mock" @@ -79,7 +81,7 @@ func TestSlackNotificationService_Publish(t *testing.T) { tt.setup(mockSlackClient) } - s := slack.NewNotificationService(mockSlackClient) + s := slack.NewNotificationService(mockSlackClient, nil) got, err := s.Publish(context.Background(), tt.notificationMessage) if (err != nil) != tt.wantErr { @@ -92,3 +94,145 @@ func TestSlackNotificationService_Publish(t *testing.T) { }) } } + +func TestSlackNotificationService_PreHookTransformConfigs(t *testing.T) { + tests := []struct { + name string + setup func(*mocks.Encryptor) + notificationConfigMap map[string]interface{} + want map[string]interface{} + wantErr bool + }{ + { + name: "should return error if failed to parse configmap to notification config", + notificationConfigMap: nil, + wantErr: true, + }, + { + name: "should return error if validate notification config failed", + notificationConfigMap: map[string]interface{}{ + "token": 123, + }, + wantErr: true, + }, + { + name: "should return error if slack token encryption failed", + notificationConfigMap: map[string]interface{}{ + "token": secret.MaskableString("a token"), + }, + setup: func(e *mocks.Encryptor) { + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return("", errors.New("some error")) + }, + wantErr: true, + }, + { + name: "should return encrypted slack token if succeed", + + notificationConfigMap: map[string]interface{}{ + "workspace": "a workspace", + "token": secret.MaskableString("a token"), + "channel_name": "channel", + }, + setup: func(e *mocks.Encryptor) { + e.EXPECT().Encrypt(mock.AnythingOfType("secret.MaskableString")).Return(secret.MaskableString("maskable-token"), nil) + }, + want: map[string]interface{}{ + "workspace": "a workspace", + "token": secret.MaskableString("maskable-token"), + "channel_name": "channel", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ( + mockEncryptor = new(mocks.Encryptor) + ) + + if tt.setup != nil { + tt.setup(mockEncryptor) + } + + s := slack.NewNotificationService(nil, mockEncryptor) + got, err := s.PreHookTransformConfigs(context.TODO(), tt.notificationConfigMap) + if (err != nil) != tt.wantErr { + t.Errorf("SlackNotificationService.PreHookTransformConfigs() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("SlackNotificationService.PreHookTransformConfigs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSlackNotificationService_PostHookTransformConfigs(t *testing.T) { + tests := []struct { + name string + setup func(*mocks.Encryptor) + notificationConfigMap map[string]interface{} + want map[string]interface{} + wantErr bool + }{ + { + name: "should return error if failed to parse configmap to notification config", + notificationConfigMap: nil, + wantErr: true, + }, + { + name: "should return error if validate notification config failed", + notificationConfigMap: map[string]interface{}{ + "token": 123, + }, + wantErr: true, + }, + { + name: "should return error if slack token decryption failed", + notificationConfigMap: map[string]interface{}{ + "token": secret.MaskableString("a token"), + }, + setup: func(e *mocks.Encryptor) { + e.EXPECT().Decrypt(mock.AnythingOfType("secret.MaskableString")).Return("", errors.New("some error")) + }, + wantErr: true, + }, + { + name: "should return encrypted slack token if succeed", + + notificationConfigMap: map[string]interface{}{ + "workspace": "a workspace", + "token": secret.MaskableString("a token"), + "channel_name": "channel", + }, + setup: func(e *mocks.Encryptor) { + e.EXPECT().Decrypt(mock.AnythingOfType("secret.MaskableString")).Return(secret.MaskableString("maskable-token"), nil) + }, + want: map[string]interface{}{ + "workspace": "a workspace", + "token": secret.MaskableString("maskable-token"), + "channel_name": "channel", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ( + mockEncryptor = new(mocks.Encryptor) + ) + + if tt.setup != nil { + tt.setup(mockEncryptor) + } + + s := slack.NewNotificationService(nil, mockEncryptor) + got, err := s.PostHookTransformConfigs(context.TODO(), tt.notificationConfigMap) + if (err != nil) != tt.wantErr { + t.Errorf("SlackNotificationService.PostHookTransformConfigs() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("SlackNotificationService.PostHookTransformConfigs() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugins/receivers/slack/receiver_service_test.go b/plugins/receivers/slack/receiver_service_test.go index 7c949dc3..1998d888 100644 --- a/plugins/receivers/slack/receiver_service_test.go +++ b/plugins/receivers/slack/receiver_service_test.go @@ -6,6 +6,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/odpf/siren/pkg/errors" + "github.com/odpf/siren/pkg/secret" "github.com/odpf/siren/plugins/receivers/slack" "github.com/odpf/siren/plugins/receivers/slack/mocks" "github.com/stretchr/testify/mock" @@ -25,15 +26,15 @@ func TestSlackReceiverService_BuildData(t *testing.T) { Description: "should return error if configuration is invalid", Setup: func(sc *mocks.SlackCaller, e *mocks.Encryptor) {}, Confs: make(map[string]interface{}), - Err: errors.New("invalid slack receiver config, workspace: , token: "), + Err: errors.New("invalid slack receiver config, workspace: , token: "), }, { Description: "should return error if failed to get workspace channels with slack client", Setup: func(sc *mocks.SlackCaller, e *mocks.Encryptor) { - sc.EXPECT().GetWorkspaceChannels(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("string")).Return(nil, errors.New("some error")) + sc.EXPECT().GetWorkspaceChannels(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("secret.MaskableString")).Return(nil, errors.New("some error")) }, Confs: map[string]interface{}{ - "token": "key", + "token": secret.MaskableString("key"), "workspace": "odpf", }, Err: errors.New("could not get channels: some error"), @@ -41,7 +42,7 @@ func TestSlackReceiverService_BuildData(t *testing.T) { { Description: "should return nil error if success populating receiver.Receiver", Setup: func(sc *mocks.SlackCaller, e *mocks.Encryptor) { - sc.EXPECT().GetWorkspaceChannels(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("string")).Return([]slack.Channel{ + sc.EXPECT().GetWorkspaceChannels(mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("secret.MaskableString")).Return([]slack.Channel{ { ID: "id", Name: "name", @@ -49,7 +50,7 @@ func TestSlackReceiverService_BuildData(t *testing.T) { }, nil) }, Confs: map[string]interface{}{ - "token": "key", + "token": secret.MaskableString("key"), "workspace": "odpf", }, }, @@ -108,7 +109,7 @@ func TestSlackReceiverService_BuildNotificationConfig(t *testing.T) { }, ExpectedConfigMap: map[string]interface{}{ "channel_name": "odpf_warning", - "token": "", + "token": secret.MaskableString(""), "workspace": "", }, }, @@ -122,7 +123,7 @@ func TestSlackReceiverService_BuildNotificationConfig(t *testing.T) { }, ExpectedConfigMap: map[string]interface{}{ "channel_name": "odpf_warning", - "token": "123", + "token": secret.MaskableString("123"), "workspace": "", }, }, diff --git a/plugins/receivers/slack/slack.go b/plugins/receivers/slack/slack.go index 60cea52a..f4f7abbe 100644 --- a/plugins/receivers/slack/slack.go +++ b/plugins/receivers/slack/slack.go @@ -1,16 +1,20 @@ package slack -import "context" +import ( + "context" + + "github.com/odpf/siren/pkg/secret" +) //go:generate mockery --name=Encryptor -r --case underscore --with-expecter --structname Encryptor --filename encryptor.go --output=./mocks type Encryptor interface { - Encrypt(str string) (string, error) - Decrypt(str string) (string, error) + Encrypt(str secret.MaskableString) (secret.MaskableString, error) + Decrypt(str secret.MaskableString) (secret.MaskableString, error) } //go:generate mockery --name=SlackCaller -r --case underscore --with-expecter --structname SlackCaller --filename slack_caller.go --output=./mocks type SlackCaller interface { ExchangeAuth(ctx context.Context, authCode, clientID, clientSecret string) (Credential, error) - GetWorkspaceChannels(ctx context.Context, token string) ([]Channel, error) + GetWorkspaceChannels(ctx context.Context, token secret.MaskableString) ([]Channel, error) Notify(ctx context.Context, conf NotificationConfig, message Message) error } diff --git a/test/e2e_test/cortex_rule_test.go b/test/e2e_test/cortex_rule_test.go index 638c18f8..8bcd6f09 100644 --- a/test/e2e_test/cortex_rule_test.go +++ b/test/e2e_test/cortex_rule_test.go @@ -32,9 +32,9 @@ func (s *CortexRuleTestSuite) SetupTest() { Level: "debug", }, Service: server.Config{ - Port: apiPort, + Port: apiPort, + EncryptionKey: testEncryptionKey, }, - EncryptionKey: testEncryptionKey, } s.testBench, err = InitCortexEnvironment(s.appConfig) diff --git a/test/e2e_test/cortex_subscription_test.go b/test/e2e_test/cortex_subscription_test.go index 0b0e1099..a36f516e 100644 --- a/test/e2e_test/cortex_subscription_test.go +++ b/test/e2e_test/cortex_subscription_test.go @@ -30,9 +30,9 @@ func (s *CortexSubscriptionTestSuite) SetupTest() { Level: "debug", }, Service: server.Config{ - Port: apiPort, + Port: apiPort, + EncryptionKey: testEncryptionKey, }, - EncryptionKey: testEncryptionKey, } s.testBench, err = InitCortexEnvironment(s.appConfig)