Grouped email notifications

changelog/unreleased/enhancement-mail-notifications-grouping.md

@@ -0,0 +1,6 @@
+Enhancement: Part IV: Grouping of mail notifications
+
+Part IV: Mail notifications can now be grouped on a daily or weekly basis
+
+https://github.com/owncloud/ocis/pull/10838
+https://github.com/owncloud/ocis/issues/10793

services/notifications/README.md

@@ -46,6 +46,35 @@ Custom email templates referenced via `NOTIFICATIONS_EMAIL_TEMPLATE_PATH` must a
 The `templates/html` subfolder contains a default HTML template provided by ocis. When using a custom HTML template, hosted images can either be linked with standard HTML code like ```<img src="https://raw.githubusercontent.com/owncloud/core/master/core/img/logo-mail.gif" alt="logo-mail"/>``` or embedded as a CID source ```<img src="cid:logo-mail.gif" alt="logo-mail"/>```. In the latter case, image files must be located in the `templates/html/img` subfolder. Supported embedded image types are png, jpeg, and gif.
 Consider that embedding images via a CID resource may not be fully supported in all email web clients.
 
+## Sending Grouped Emails
+
+The `notification` service can initiate sending emails based on events stored in the configured store that are grouped into a `daily` or `weekly` bucket. These groups contain events that get populated e.g. when the user configures `daily` or `weekly` email notifications in his personal settings in the web UI. If a user does not define any of the named groups for notification events, no event is stored.
+
+Grouped events are stored for the TTL defined in `OCIS_PERSISTENT_STORE_TTL`. This TTL can either be configured globally or individually for the notification service via the `NOTIFICATIONS_STORE_TTL` envvar.
+
+Grouped events that have passed the TTL are removed automatically without further notice or sending!
+
+To initiate sending grouped emails like via a cron job, use the `ocis notifications send-email` command. Note that the command mandatory requires at least one option which is `--daily` or `--weekly`. Note that both options can be used together.
+
+### Storing
+
+The `notifications` service persists information via the configured store in `NOTIFICATIONS_STORE`. Possible stores are:
+-   `memory`: Basic in-memory store. Will not survive a restart. This is not recommended for this service.
+-   `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
+-   `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store). This is the default value.
+-   `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
+
+Other store types may work but are not supported currently.
+
+Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
+
+Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
+
+Store specific notes:
+-   When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
+-   When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
+-   When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
+
 ## Translations
 
 The `notifications` service has embedded translations sourced via transifex to provide a basic set of translated languages. These embedded translations are available for all deployment scenarios.

services/notifications/pkg/command/root.go

@@ -15,6 +15,7 @@ func GetCommands(cfg *config.Config) cli.Commands {
 		Server(cfg),
 
 		// interaction with this service
+		SendEmail(cfg),
 
 		// infos about this service
 		Health(cfg),

services/notifications/pkg/command/send_email.go

@@ -0,0 +1,57 @@
+package command
+
+import (
+	"github.com/cs3org/reva/v2/pkg/events"
+	"github.com/cs3org/reva/v2/pkg/events/stream"
+	"github.com/owncloud/ocis/v2/services/notifications/pkg/config"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+)
+
+// SendEmail triggers the sending of grouped email notifications for daily or weekly emails.
+func SendEmail(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "send-email",
+		Usage: "Send grouped email notifications with daily or weekly interval. Specify at least one of the flags '--daily' or '--weekly'.",
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:    "daily",
+				Aliases: []string{"d"},
+				Usage:   "Sends grouped daily email notifications.",
+			},
+			&cli.BoolFlag{
+				Name:    "weekly",
+				Aliases: []string{"w"},
+				Usage:   "Sends grouped weekly email notifications.",
+			},
+		},
+		Action: func(c *cli.Context) error {
+			daily := c.Bool("daily")
+			weekly := c.Bool("weekly")
+			if !daily && !weekly {
+				return errors.New("at least one of '--daily' or '--weekly' must be set")
+			}
+			s, err := stream.NatsFromConfig(cfg.Service.Name, false, stream.NatsConfig(cfg.Notifications.Events))
+			if err != nil {
+				return err
+			}
+			if daily {
+				err = events.Publish(c.Context, s, events.SendEmailsEvent{
+					Interval: "daily",
+				})
+				if err != nil {
+					return err
+				}
+			}
+			if weekly {
+				err = events.Publish(c.Context, s, events.SendEmailsEvent{
+					Interval: "weekly",
+				})
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+	}
+}

services/notifications/pkg/command/server.go

@@ -3,6 +3,10 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/cs3org/reva/v2/pkg/store"
+	ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
+	microstore "go-micro.dev/v4/store"
+	"reflect"
 
 	"github.com/oklog/run"
 	"github.com/urfave/cli/v2"
@@ -81,7 +85,14 @@ func Server(cfg *config.Config) *cli.Command {
 				events.SpaceUnshared{},
 				events.SpaceMembershipExpired{},
 				events.ScienceMeshInviteTokenGenerated{},
+				events.SendEmailsEvent{},
 			}
+			registeredEvents := make(map[string]events.Unmarshaller)
+			for _, e := range evs {
+				typ := reflect.TypeOf(e)
+				registeredEvents[typ.String()] = e
+			}
+
 			client, err := stream.NatsFromConfig(cfg.Service.Name, false, stream.NatsConfig(cfg.Notifications.Events))
 			if err != nil {
 				return err
@@ -109,7 +120,21 @@ func Server(cfg *config.Config) *cli.Command {
 				logger.Fatal().Err(err).Str("addr", cfg.Notifications.RevaGateway).Msg("could not get reva gateway selector")
 			}
 			valueService := settingssvc.NewValueService("com.owncloud.api.settings", grpcClient)
-			svc := service.NewEventsNotifier(evts, channel, logger, gatewaySelector, valueService, cfg.ServiceAccount.ServiceAccountID, cfg.ServiceAccount.ServiceAccountSecret, cfg.Notifications.EmailTemplatePath, cfg.Notifications.DefaultLanguage, cfg.WebUIURL, cfg.Notifications.TranslationPath)
+			historyClient := ehsvc.NewEventHistoryService("com.owncloud.api.eventhistory", grpcClient)
+
+			notificationStore := store.Create(
+				store.Store(cfg.Store.Store),
+				store.TTL(cfg.Store.TTL),
+				microstore.Nodes(cfg.Store.Nodes...),
+				microstore.Database(cfg.Store.Database),
+				microstore.Table(cfg.Store.Table),
+				store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
+			)
+
+			svc := service.NewEventsNotifier(evts, channel, logger, gatewaySelector, valueService,
+				cfg.ServiceAccount.ServiceAccountID, cfg.ServiceAccount.ServiceAccountSecret,
+				cfg.Notifications.EmailTemplatePath, cfg.Notifications.DefaultLanguage, cfg.WebUIURL,
+				cfg.Notifications.TranslationPath, cfg.Notifications.SMTP.Sender, notificationStore, historyClient, registeredEvents)
 
 			gr.Add(svc.Run, func(error) {
 				cancel()

services/notifications/pkg/config/config.go

@@ -3,6 +3,7 @@ package config
 
 import (
 	"context"
+	"time"
 
 	"github.com/owncloud/ocis/v2/ocis-pkg/shared"
 )
@@ -24,6 +25,8 @@ type Config struct {
 	ServiceAccount ServiceAccount       `yaml:"service_account"`
 
 	Context context.Context `yaml:"-"`
+
+	Store Store `yaml:"store"`
 }
 
 // Notifications defines the config options for the notifications service.
@@ -65,3 +68,14 @@ type ServiceAccount struct {
 	ServiceAccountID     string `yaml:"service_account_id" env:"OCIS_SERVICE_ACCOUNT_ID;NOTIFICATIONS_SERVICE_ACCOUNT_ID" desc:"The ID of the service account the service should use. See the 'auth-service' service description for more details." introductionVersion:"5.0"`
 	ServiceAccountSecret string `yaml:"service_account_secret" env:"OCIS_SERVICE_ACCOUNT_SECRET;NOTIFICATIONS_SERVICE_ACCOUNT_SECRET" desc:"The service account secret." introductionVersion:"5.0"`
 }
+
+// Store configures the store to use
+type Store struct {
+	Store        string        `yaml:"store" env:"OCIS_PERSISTENT_STORE;NOTIFICATIONS_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"7.1"`
+	Nodes        []string      `yaml:"nodes" env:"OCIS_PERSISTENT_STORE_NODES;NOTIFICATIONS_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"7.1"`
+	Database     string        `yaml:"database" env:"NOTIFICATIONS_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"7.1"`
+	Table        string        `yaml:"table" env:"NOTIFICATIONS_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"7.1"`
+	TTL          time.Duration `yaml:"ttl" env:"OCIS_PERSISTENT_STORE_TTL;NOTIFICATIONS_STORE_TTL" desc:"Time to live for notifications in the store. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details." introductionVersion:"7.1"`
+	AuthUsername string        `yaml:"username" env:"OCIS_PERSISTENT_STORE_AUTH_USERNAME;NOTIFICATIONS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"7.1"`
+	AuthPassword string        `yaml:"password" env:"OCIS_PERSISTENT_STORE_AUTH_PASSWORD;NOTIFICATIONS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"7.1"`
+}

services/notifications/pkg/config/defaults/defaultconfig.go

@@ -4,6 +4,7 @@ import (
 	"github.com/owncloud/ocis/v2/ocis-pkg/shared"
 	"github.com/owncloud/ocis/v2/ocis-pkg/structs"
 	"github.com/owncloud/ocis/v2/services/notifications/pkg/config"
+	"time"
 )
 
 // FullDefaultConfig returns a fully initialized default configuration
@@ -40,6 +41,13 @@ func DefaultConfig() *config.Config {
 			},
 			RevaGateway: shared.DefaultRevaConfig().Address,
 		},
+		Store: config.Store{
+			Store:    "nats-js-kv",
+			Nodes:    []string{"127.0.0.1:9233"},
+			Database: "notifications",
+			Table:    "",
+			TTL:      336 * time.Hour,
+		},
 	}
 }
 

services/notifications/pkg/email/composer.go

@@ -3,6 +3,7 @@ package email
 import (
 	"bytes"
 	"embed"
+	"github.com/pkg/errors"
 	"strings"
 	"text/template"
 
@@ -61,6 +62,65 @@ func NewHTMLTemplate(mt MessageTemplate, locale, defaultLocale string, translati
 	return mt, nil
 }
 
+// NewGroupedTextTemplate replace the body message template placeholders with the translated template
+func NewGroupedTextTemplate(gmt GroupedMessageTemplate, vars map[string]string, locale, defaultLocale string, translationPath string, mts []MessageTemplate, mtsVars []map[string]string) (GroupedMessageTemplate, error) {
+	if len(mts) != len(mtsVars) {
+		return gmt, errors.New("number of templates does not match number of variables")
+	}
+
+	var err error
+	t := l10n.NewTranslatorFromCommonConfig(defaultLocale, _domain, translationPath, _translationFS, "l10n/locale").Locale(locale)
+	gmt.Subject, err = composeMessage(t.Get(gmt.Subject), vars)
+	if err != nil {
+		return gmt, err
+	}
+	gmt.Greeting, err = composeMessage(t.Get(gmt.Greeting), vars)
+	if err != nil {
+		return gmt, err
+	}
+
+	bodyParts := make([]string, 0, len(mtsVars))
+	for i, mt := range mts {
+		bodyPart, err := composeMessage(t.Get(mt.MessageBody), mtsVars[i])
+		if err != nil {
+			return gmt, err
+		}
+		bodyParts = append(bodyParts, bodyPart)
+	}
+	gmt.MessageBody = strings.Join(bodyParts, "\n\n\n")
+	return gmt, nil
+}
+
+// NewGroupedHTMLTemplate replace the body message template placeholders with the translated template
+func NewGroupedHTMLTemplate(gmt GroupedMessageTemplate, vars map[string]string, locale, defaultLocale string, translationPath string, mts []MessageTemplate, mtsVars []map[string]string) (GroupedMessageTemplate, error) {
+	if len(mts) != len(mtsVars) {
+		return gmt, errors.New("number of templates does not match number of variables")
+	}
+
+	var err error
+	t := l10n.NewTranslatorFromCommonConfig(defaultLocale, _domain, translationPath, _translationFS, "l10n/locale").Locale(locale)
+	gmt.Subject, err = composeMessage(t.Get(gmt.Subject), vars)
+	if err != nil {
+		return gmt, err
+	}
+	gmt.Greeting, err = composeMessage(newlineToBr(t.Get(gmt.Greeting)), vars)
+	if err != nil {
+		return gmt, err
+	}
+
+	bodyParts := make([]string, 0, len(mtsVars))
+	for i, mt := range mts {
+		bodyPart, err := composeMessage(t.Get(mt.MessageBody), mtsVars[i])
+		if err != nil {
+			return gmt, err
+		}
+		bodyParts = append(bodyParts, bodyPart)
+	}
+	gmt.MessageBody = strings.Join(bodyParts, "<br><br><br>")
+
+	return gmt, nil
+}
+
 // composeMessage renders the message based on template
 func composeMessage(tmpl string, vars map[string]string) (string, error) {
 	tpl, err := template.New("").Parse(replacePlaceholders(tmpl))

services/notifications/pkg/email/email.go

@@ -67,6 +67,53 @@ func RenderEmailTemplate(mt MessageTemplate, locale, defaultLocale string, email
 	}, nil
 }
 
+// RenderGroupedEmailTemplate is responsible to prepare a message which than can be used to notify the user via email.
+func RenderGroupedEmailTemplate(gmt GroupedMessageTemplate, vars map[string]string, locale, defaultLocale string, emailTemplatePath string, translationPath string, mts []MessageTemplate, mtsVars []map[string]string) (*channels.Message, error) {
+	textMt, err := NewGroupedTextTemplate(gmt, vars, locale, defaultLocale, translationPath, mts, mtsVars)
+	if err != nil {
+		return nil, err
+	}
+	tpl, err := parseTemplate(emailTemplatePath, gmt.textTemplate)
+	if err != nil {
+		return nil, err
+	}
+	textBody, err := groupedEmailTemplate(tpl, textMt)
+	if err != nil {
+		return nil, err
+	}
+
+	escapedMtsVars := make([]map[string]string, 0, len(mtsVars))
+	for _, m := range mtsVars {
+		escapedMtsVars = append(escapedMtsVars, escapeStringMap(m))
+	}
+	htmlMt, err := NewGroupedHTMLTemplate(gmt, escapeStringMap(vars), locale, defaultLocale, translationPath, mts, escapedMtsVars)
+	if err != nil {
+		return nil, err
+	}
+	htmlTpl, err := parseTemplate(emailTemplatePath, gmt.htmlTemplate)
+	if err != nil {
+		return nil, err
+	}
+	htmlBody, err := groupedEmailTemplate(htmlTpl, htmlMt)
+	if err != nil {
+		return nil, err
+	}
+	var data map[string][]byte
+	if emailTemplatePath != "" {
+		data, err = readImages(emailTemplatePath)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &channels.Message{
+		Subject:      textMt.Subject,
+		TextBody:     textBody,
+		HTMLBody:     htmlBody,
+		AttachInline: data,
+	}, nil
+}
+
 // emailTemplate builds the email template. It does not use any user provided input, so it is safe to use template.HTML.
 func emailTemplate(tpl *template.Template, mt MessageTemplate) (string, error) {
 	str, err := executeTemplate(tpl, map[string]interface{}{
@@ -80,6 +127,18 @@ func emailTemplate(tpl *template.Template, mt MessageTemplate) (string, error) {
 	return str, err
 }
 
+// groupedEmailTemplate builds the email template. It does not use any user provided input, so it is safe to use template.HTML.
+func groupedEmailTemplate(tpl *template.Template, gmt GroupedMessageTemplate) (string, error) {
+	str, err := executeTemplate(tpl, map[string]interface{}{
+		"Greeting":    template.HTML(strings.TrimSpace(gmt.Greeting)),    // #nosec G203
+		"MessageBody": template.HTML(strings.TrimSpace(gmt.MessageBody)), // #nosec G203
+	})
+	if err != nil {
+		return "", err
+	}
+	return str, err
+}
+
 func parseTemplate(emailTemplatePath string, file string) (*template.Template, error) {
 	if emailTemplatePath != "" {
 		return template.ParseFiles(filepath.Join(emailTemplatePath, file))