docs: improve documentation

Makefile

@@ -2,7 +2,7 @@ NAME="github.com/odpf/siren"
 LAST_COMMIT := $(shell git rev-parse --short HEAD)
 LAST_TAG := "$(shell git rev-list --tags --max-count=1)"
 APP_VERSION := "$(shell git describe --tags ${LAST_TAG})-next"
-PROTON_COMMIT := "9ece66237001172a087325bd536ad6d944c801d9"
+PROTON_COMMIT := "f3654990b4e82d50daa25a3c41b7de37060807d9"
 
 .PHONY: all build test clean dist vet proto install
 
@@ -37,6 +37,10 @@ proto: ## Generate the protobuf files
 clean: ## Clean the build artifacts
 	rm -rf siren dist/
 
+update-swagger-md:
+	@echo "> updating reference api docs"
+	@npx swagger-markdown -i proto/siren.swagger.yaml -o docs/docs/reference/api.md
+
 install: ## install required dependencies
 	@echo "> installing dependencies"
 	go mod tidy
@@ -50,5 +54,16 @@ install: ## install required dependencies
 	go get -d github.com/bufbuild/buf/cmd/buf@v1.7.0
 	go get github.com/envoyproxy/protoc-gen-validate@v0.6.7
 
+clean-doc:
+	@echo "> cleaning up auto-generated docs"
+	@rm -rf ./docs/docs/reference/cli
+	@rm -f ./docs/docs/reference/api.md
+
+# Generates the config file documentation.
+# remove ansi color & escape html
+doc: clean-doc update-swagger-md
+	@echo "> generate cli docs"
+	@go run . reference -s=false | sed '1 s,.*,# CLI,' > ./docs/docs/reference/cli.md
+
 help: ## Display this help message
 	@cat $(MAKEFILE_LIST) | grep -e "^[a-zA-Z_\-]*: *.*## *" | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

cli/alert.go

@@ -39,7 +39,7 @@ func alertsCmd(cmdxConfig *cmdx.Config) *cobra.Command {
 }
 
 func listAlertsCmd(cmdxConfig *cmdx.Config) *cobra.Command {
-	var providerName string
+	var providerType string
 	var providerId uint64
 	var resouceName string
 	var startTime uint64
@@ -71,7 +71,7 @@ func listAlertsCmd(cmdxConfig *cmdx.Config) *cobra.Command {
 			defer cancel()
 
 			res, err := client.ListAlerts(ctx, &sirenv1beta1.ListAlertsRequest{
-				ProviderName: providerName,
+				ProviderType: providerType,
 				ProviderId:   providerId,
 				ResourceName: resouceName,
 				StartTime:    startTime,
@@ -110,8 +110,8 @@ func listAlertsCmd(cmdxConfig *cmdx.Config) *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVar(&providerName, "provider-name", "", "provider name")
-	cmd.MarkFlagRequired("provider-name")
+	cmd.Flags().StringVar(&providerType, "provider-type", "", "provider type")
+	cmd.MarkFlagRequired("provider-type")
 	cmd.Flags().Uint64Var(&providerId, "provider-id", 0, "provider id")
 	cmd.MarkFlagRequired("provider-id")
 	cmd.Flags().StringVar(&resouceName, "resource-name", "", "resource name")

cli/client.go

@@ -14,7 +14,7 @@ import (
 )
 
 type ClientConfig struct {
-	Host string `yaml:"host" cmdx:"host"`
+	Host string `yaml:"host" cmdx:"host" default:"localhost:8080"`
 }
 
 func loadClientConfig(cmd *cobra.Command, cmdxConfig *cmdx.Config) (*ClientConfig, error) {
@@ -24,14 +24,25 @@ func loadClientConfig(cmd *cobra.Command, cmdxConfig *cmdx.Config) (*ClientConfi
 		&clientConfig,
 		cmdx.WithFlags(cmd.Flags()),
 	); err != nil {
-		if !errors.Is(err, new(config.ConfigFileNotFoundError)) {
+		if !errors.As(err, new(config.ConfigFileNotFoundError)) {
 			return nil, err
 		}
 	}
 
+	if err := validateClientConfig(&clientConfig); err != nil {
+		return nil, err
+	}
+
 	return &clientConfig, nil
 }
 
+func validateClientConfig(cfg *ClientConfig) error {
+	if cfg.Host == "" {
+		return errors.New("`host` is missing")
+	}
+	return nil
+}
+
 func createConnection(ctx context.Context, host string) (*grpc.ClientConn, error) {
 	opts := []grpc.DialOption{
 		grpc.WithTransportCredentials(insecure.NewCredentials()),

cli/deps.go

@@ -19,6 +19,7 @@ import (
 	"github.com/odpf/siren/pkg/retry"
 	"github.com/odpf/siren/pkg/secret"
 	"github.com/odpf/siren/plugins/providers/cortex"
+	"github.com/odpf/siren/plugins/receivers/file"
 	"github.com/odpf/siren/plugins/receivers/httpreceiver"
 	"github.com/odpf/siren/plugins/receivers/pagerduty"
 	"github.com/odpf/siren/plugins/receivers/slack"
@@ -96,6 +97,7 @@ func InitAPIDeps(
 	slackReceiverService := slack.NewReceiverService(slackClient, encryptor)
 	httpReceiverService := httpreceiver.NewReceiverService()
 	pagerDutyReceiverService := pagerduty.NewReceiverService()
+	fileReceiverService := file.NewReceiverService()
 
 	receiverRepository := postgres.NewReceiverRepository(pgClient)
 	receiverService := receiver.NewService(
@@ -104,6 +106,7 @@ func InitAPIDeps(
 			receiver.TypeSlack:     slackReceiverService,
 			receiver.TypeHTTP:      httpReceiverService,
 			receiver.TypePagerDuty: pagerDutyReceiverService,
+			receiver.TypeFile:      fileReceiverService,
 		},
 	)
 
@@ -119,11 +122,13 @@ func InitAPIDeps(
 	slackNotificationService := slack.NewNotificationService(slackClient, encryptor)
 	pagerdutyNotificationService := pagerduty.NewNotificationService(pagerdutyClient)
 	httpreceiverNotificationService := httpreceiver.NewNotificationService(httpreceiverClient)
+	fileNotificationService := file.NewNotificationService()
 
 	notifierRegistry := map[string]notification.Notifier{
 		receiver.TypeSlack:     slackNotificationService,
 		receiver.TypePagerDuty: pagerdutyNotificationService,
 		receiver.TypeHTTP:      httpreceiverNotificationService,
+		receiver.TypeFile:      fileNotificationService,
 	}
 
 	notificationService := notification.NewService(logger, queue, receiverService, subscriptionService, notifierRegistry)

cli/help.go

@@ -5,9 +5,11 @@ import "github.com/MakeNowJust/heredoc"
 var envHelp = map[string]string{
 	"short": "List of supported environment variables",
 	"long": heredoc.Doc(`
+			` + "```" + `
 			ODPF_CONFIG_DIR: the directory where siren will store configuration files. Default:
 			"$XDG_CONFIG_HOME/odpf" or "$HOME/.config/odpf".
 			NO_COLOR: set to any value to avoid printing ANSI escape sequences for color output.
 			CLICOLOR: set to "0" to disable printing ANSI colors in output.
+			` + "```" + `
 		`),
 }

cli/job.go

@@ -19,7 +19,12 @@ func jobCmd(cmdxConfig *cmdx.Config) *cobra.Command {
 		Use:     "job <command>",
 		Aliases: []string{"jobs"},
 		Short:   "Manage siren jobs",
-		Long:    "Jobs management commands.",
+		Long: heredoc.Doc(`
+			Execute a siren's job.
+			
+			A job is a task in Siren that could be executed and stopped once the task is done. 
+			The Job is usually run as a CronJob to be executed on a specified time.
+		`),
 		Example: heredoc.Doc(`
 			$ siren job run cleanup_queue
 		`),

cli/namespace.go

@@ -23,7 +23,7 @@ func namespacesCmd(cmdxConfig *cmdx.Config) *cobra.Command {
 		Long: heredoc.Doc(`
 			Work with namespaces.
 			
-			namespaces are used for multi-tenancy for a given provider.
+			Namespaces are used for multi-tenancy for a given provider.
 		`),
 		Annotations: map[string]string{
 			"group:core": "true",

cli/receiver.go

@@ -361,6 +361,11 @@ func deleteReceiverCmd(cmdxConfig *cmdx.Config) *cobra.Command {
 }
 
 func notifyReceiverCmd(cmdxConfig *cmdx.Config) *cobra.Command {
+
+	type notifyReceiverRequest struct {
+		Payload map[string]interface{} `mapstructure:"payload" yaml:"payload"`
+	}
+
 	var id uint64
 	var filePath string
 	cmd := &cobra.Command{
@@ -400,19 +405,22 @@ func notifyReceiverCmd(cmdxConfig *cmdx.Config) *cobra.Command {
 				return errors.New("no response from server")
 			}
 
-			notifyReceiverReq := &sirenv1beta1.NotifyReceiverRequest{}
-			notifyReceiverReq.Id = rcv.GetReceiver().GetId()
-			switch rcv.GetReceiver().GetType() {
-			case "slack":
-				var slackConfig *structpb.Struct
-				if err := parseFile(filePath, &slackConfig); err != nil {
-					return err
-				}
+			notifyReceiverReq := &notifyReceiverRequest{}
+			if err := parseFile(filePath, &notifyReceiverReq); err != nil {
+				return err
+			}
+
+			payloadPB, err := structpb.NewStruct(notifyReceiverReq.Payload)
+			if err != nil {
+				return err
+			}
 
-				notifyReceiverReq.Payload = slackConfig
+			notifyReceiverReqPB := &sirenv1beta1.NotifyReceiverRequest{
+				Id:      id,
+				Payload: payloadPB,
 			}
 
-			_, err = client.NotifyReceiver(ctx, notifyReceiverReq)
+			_, err = client.NotifyReceiver(ctx, notifyReceiverReqPB)
 			if err != nil {
 				return err
 			}
@@ -428,7 +436,7 @@ func notifyReceiverCmd(cmdxConfig *cmdx.Config) *cobra.Command {
 
 	cmd.Flags().Uint64Var(&id, "id", 0, "receiver id")
 	cmd.MarkFlagRequired("id")
-	cmd.Flags().StringVarP(&filePath, "file", "f", "", "Path to the receiver notification config")
+	cmd.Flags().StringVarP(&filePath, "file", "f", "", "Path to the receiver notification message")
 	cmd.MarkFlagRequired("file")
 
 	return cmd

cli/root.go

@@ -54,7 +54,6 @@ func New() *cobra.Command {
 	cmdx.SetClientHook(rootCmd, func(cmd *cobra.Command) {
 		// client config
 		cmd.PersistentFlags().StringP("host", "h", "", "Siren API service to connect to")
-		cmd.MarkPersistentFlagRequired("host")
 	})
 
 	return rootCmd

cli/server.go

@@ -115,6 +115,9 @@ func serverMigrateCommand() *cobra.Command {
 			if err := postgres.Migrate(cfg.DB); err != nil {
 				return err
 			}
+			printer.Success("Migration done")
+			printer.Space()
+			printer.SuccessIcon()
 			return nil
 		},
 	}

cli/template.go

@@ -466,7 +466,7 @@ func UploadTemplate(client sirenv1beta1.SirenServiceClient, yamlFile []byte) (ui
 	}
 
 	if data.GetRules() == nil {
-		return 0, errors.New("no response from server")
+		return template.GetId(), nil
 	}
 
 	associatedRules := data.GetRules()
@@ -494,11 +494,11 @@ func UploadTemplate(client sirenv1beta1.SirenServiceClient, yamlFile []byte) (ui
 		})
 
 		if err != nil {
-			fmt.Println("failed to update rule of ID: ", associatedRule.Id, "\tname: ", associatedRule.Name)
-			return 0, err
+			return 0, fmt.Errorf("failed to update rule of ID: %d\tname: %s", associatedRule.Id, associatedRule.Name)
 		}
 		fmt.Println("successfully updated rule of ID: ", associatedRule.Id, "\tname: ", associatedRule.Name)
 	}
+
 	return template.GetId(), nil
 }
 

cli/worker.go

@@ -21,8 +21,12 @@ func workerCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "worker <command> <worker_command>",
 		Aliases: []string{"w"},
-		Short:   "Siren's worker command",
-		Long:    "Worker management commands.",
+		Short:   "Start or manage Siren's workers",
+		Long: heredoc.Doc(`
+			A command to start or manage Siren's workers.
+
+			A worker is an instance in Siren that run detached from the server.
+		`),
 		Example: heredoc.Doc(`
 			$ siren worker start notification_handler
 			$ siren worker start notification_handler -c ./config.yaml

config/config.go

@@ -31,17 +31,17 @@ func Load(configFile string) (Config, error) {
 }
 
 type Log struct {
-	Level         string `mapstructure:"level" default:"info"`
-	GCPCompatible bool   `mapstructure:"gcp_compatible" default:"true"`
+	Level         string `mapstructure:"level" yaml:"level" default:"info"`
+	GCPCompatible bool   `mapstructure:"gcp_compatible" yaml:"gcp_compatible" default:"true"`
 }
 
 // Config contains the application configuration
 type Config struct {
 	DB           db.Config                `mapstructure:"db"`
 	Cortex       cortex.Config            `mapstructure:"cortex"`
-	NewRelic     telemetry.NewRelicConfig `mapstructure:"newrelic"`
-	Service      server.Config            `mapstructure:"service"`
-	Log          Log                      `mapstructure:"log"`
-	Receivers    receivers.Config         `mapstructure:"receivers"`
-	Notification notification.Config      `mapstructure:"notification"`
+	NewRelic     telemetry.NewRelicConfig `mapstructure:"newrelic" yaml:"newrelic"`
+	Service      server.Config            `mapstructure:"service" yaml:"service"`
+	Log          Log                      `mapstructure:"log" yaml:"log"`
+	Receivers    receivers.Config         `mapstructure:"receivers" yaml:"receivers"`
+	Notification notification.Config      `mapstructure:"notification" yaml:"notification"`
 }

config/init.go

@@ -4,6 +4,7 @@ import (
 	"os"
 
 	"github.com/mcuadros/go-defaults"
+	"github.com/odpf/siren/core/receiver"
 	"gopkg.in/yaml.v2"
 )
 
@@ -12,12 +13,23 @@ func Init(configFile string) error {
 
 	defaults.SetDefaults(cfg)
 
+	cfg.DB.Driver = "postgres"
+	cfg.DB.URL = "postgres://postgres:@localhost:5432/siren_development?sslmode=disable"
+
+	if len(cfg.Notification.MessageHandler.ReceiverTypes) == 0 {
+		cfg.Notification.MessageHandler.ReceiverTypes = receiver.SupportedTypes
+	}
+
+	if len(cfg.Notification.DLQHandler.ReceiverTypes) == 0 {
+		cfg.Notification.DLQHandler.ReceiverTypes = receiver.SupportedTypes
+	}
+
 	data, err := yaml.Marshal(cfg)
 	if err != nil {
 		return err
 	}
 
-	if err := os.WriteFile(configFile, data, 0655); err != nil {
+	if err := os.WriteFile(configFile, data, os.ModePerm); err != nil {
 		return err
 	}
 

core/namespace/service.go

@@ -59,6 +59,8 @@ func (s *Service) Create(ctx context.Context, ns *Namespace) error {
 		return err
 	}
 
+	ns.ID = encryptedNamespace.ID
+
 	return nil
 }
 
@@ -74,8 +76,8 @@ func (s *Service) Get(ctx context.Context, id uint64) (*Namespace, error) {
 	return s.decrypt(encryptedNamespace)
 }
 
-func (s *Service) Update(ctx context.Context, namespace *Namespace) error {
-	encryptedNamespace, err := s.encrypt(namespace)
+func (s *Service) Update(ctx context.Context, ns *Namespace) error {
+	encryptedNamespace, err := s.encrypt(ns)
 	if err != nil {
 		return err
 	}
@@ -94,6 +96,8 @@ func (s *Service) Update(ctx context.Context, namespace *Namespace) error {
 		return err
 	}
 
+	ns.ID = encryptedNamespace.ID
+
 	return nil
 }
 

core/notification/config.go

@@ -7,14 +7,14 @@ import (
 )
 
 type Config struct {
-	Queue          queues.Config `mapstructure:"queue"`
-	MessageHandler HandlerConfig `mapstructure:"message_handler"`
-	DLQHandler     HandlerConfig `mapstructure:"dlq_handler"`
+	Queue          queues.Config `mapstructure:"queue" yaml:"queue"`
+	MessageHandler HandlerConfig `mapstructure:"message_handler" yaml:"message_handler"`
+	DLQHandler     HandlerConfig `mapstructure:"dlq_handler" yaml:"dlq_handler"`
 }
 
 type HandlerConfig struct {
-	Enabled       bool          `mapstructure:"enabled"`
-	PollDuration  time.Duration `mapstructure:"poll_duration"`
-	ReceiverTypes []string      `mapstructure:"receiver_types"`
-	BatchSize     int           `mapstructure:"batch_size"`
+	Enabled       bool          `mapstructure:"enabled" yaml:"enabled" default:"true"`
+	PollDuration  time.Duration `mapstructure:"poll_duration" yaml:"poll_duration" default:"5s"`
+	ReceiverTypes []string      `mapstructure:"receiver_types" yaml:"receiver_types"`
+	BatchSize     int           `mapstructure:"batch_size" yaml:"batch_size" default:"1"`
 }