Add transactional (tx) messaging capability.

This commit adds a new API `POST /api/tx` that sends an ad-hoc message
to a subscriber based on a pre-defined transactional template. This is
a large commit that adds the following:

- New campaign / tx template types on the UI. tx templates have an
  additional subject field.
- New fields `type` and `subject` to the templates table.
- Refactor template CRUD operations and models.
- Refactor template func assignment in manager.
- Add pre-compiled template caching to manager runtime.
- Pre-compile all tx templates into memory on program boot to avoid
  expensive template compilation on ad-hoc tx messages.

cmd/handlers.go

@@ -138,6 +138,8 @@ func initHTTPHandlers(e *echo.Echo, app *App) {
 	g.PUT("/api/templates/:id/default", handleTemplateSetDefault)
 	g.DELETE("/api/templates/:id", handleDeleteTemplate)
 
+	g.POST("/api/tx", handleSendTxMessage)
+
 	if app.constants.BounceWebhooksEnabled {
 		// Private authenticated bounce endpoint.
 		g.POST("/webhooks/bounce", handleBounceWebhook)

cmd/init.go

@@ -426,6 +426,20 @@ func initCampaignManager(q *models.Queries, cs *constants, app *App) *manager.Ma
 	}, newManagerStore(q), campNotifCB, app.i18n, lo)
 }
 
+func initTxTemplates(m *manager.Manager, app *App) {
+	tpls, err := app.core.GetTemplates(models.TemplateTypeTx, false)
+	if err != nil {
+		lo.Fatalf("error loading transactional templates: %v", err)
+	}
+
+	for _, t := range tpls {
+		if err := t.Compile(app.manager.GenericTemplateFuncs()); err != nil {
+			lo.Fatalf("error compiling transactional template %d: %v", t.ID, err)
+		}
+		m.CacheTpl(t.ID, &t)
+	}
+}
+
 // initImporter initializes the bulk subscriber importer.
 func initImporter(q *models.Queries, db *sqlx.DB, app *App) *subimporter.Importer {
 	return subimporter.New(

cmd/install.go

@@ -109,20 +109,17 @@ func install(lastVer string, db *sqlx.DB, fs stuffbin.FileSystem, prompt, idempo
 		lo.Fatalf("error creating subscriber: %v", err)
 	}
 
-	// Default template.
-	tplBody, err := fs.Get("/static/email-templates/default.tpl")
+	// Default campaign template.
+	campTpl, err := fs.Get("/static/email-templates/default.tpl")
 	if err != nil {
 		lo.Fatalf("error reading default e-mail template: %v", err)
 	}
 
-	var tplID int
-	if err := q.CreateTemplate.Get(&tplID,
-		"Default template",
-		string(tplBody.ReadBytes()),
-	); err != nil {
-		lo.Fatalf("error creating default template: %v", err)
+	var campTplID int
+	if err := q.CreateTemplate.Get(&campTplID, "Default campaign template", models.TemplateTypeCampaign, "", campTpl.ReadBytes()); err != nil {
+		lo.Fatalf("error creating default campaign template: %v", err)
 	}
-	if _, err := q.SetDefaultTemplate.Exec(tplID); err != nil {
+	if _, err := q.SetDefaultTemplate.Exec(campTplID); err != nil {
 		lo.Fatalf("error setting default template: %v", err)
 	}
 
@@ -146,12 +143,22 @@ func install(lastVer string, db *sqlx.DB, fs stuffbin.FileSystem, prompt, idempo
 		json.RawMessage("[]"),
 		pq.StringArray{"test-campaign"},
 		emailMsgr,
-		1,
+		campTplID,
 		pq.Int64Array{1},
 	); err != nil {
 		lo.Fatalf("error creating sample campaign: %v", err)
 	}
 
+	// Sample tx template.
+	txTpl, err := fs.Get("/static/email-templates/sample-tx.tpl")
+	if err != nil {
+		lo.Fatalf("error reading default e-mail template: %v", err)
+	}
+
+	if _, err := q.CreateTemplate.Exec("Sample transactional template", models.TemplateTypeTx, "Welcome {{ .Subscriber.Name }}", txTpl.ReadBytes()); err != nil {
+		lo.Fatalf("error creating sample transactional template: %v", err)
+	}
+
 	lo.Printf("setup complete")
 	lo.Printf(`run the program and access the dashboard at %s`, ko.MustString("app.address"))
 }

cmd/main.go

@@ -189,6 +189,7 @@ func main() {
 	app.manager = initCampaignManager(app.queries, app.constants, app)
 	app.importer = initImporter(app.queries, db, app)
 	app.notifTpls = initNotifTemplates("/email-templates/*.html", fs, app.i18n, app.constants)
+	initTxTemplates(app.manager, app)
 
 	if ko.Bool("bounce.enabled") {
 		app.bounce = initBounceManager(app)

cmd/templates.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"regexp"
 	"strconv"
+	"strings"
 
 	"github.com/knadh/listmonk/models"
 	"github.com/labstack/echo/v4"
@@ -48,7 +49,7 @@ func handleGetTemplates(c echo.Context) error {
 		return c.JSON(http.StatusOK, okResp{out})
 	}
 
-	out, err := app.core.GetTemplates(noBody)
+	out, err := app.core.GetTemplates("", noBody)
 	if err != nil {
 		return err
 	}
@@ -63,10 +64,15 @@ func handlePreviewTemplate(c echo.Context) error {
 
 		id, _ = strconv.Atoi(c.Param("id"))
 		body  = c.FormValue("body")
+		typ   = c.FormValue("typ")
 	)
 
+	if typ == "" {
+		typ = models.TemplateTypeCampaign
+	}
+
 	if body != "" {
-		if !regexpTplTag.MatchString(body) {
+		if typ == models.TemplateTypeCampaign && !regexpTplTag.MatchString(body) {
 			return echo.NewHTTPError(http.StatusBadRequest,
 				app.i18n.Ts("templates.placeholderHelp", "placeholder", tplTag))
 		}
@@ -120,16 +126,33 @@ func handleCreateTemplate(c echo.Context) error {
 	}
 
 	if err := validateTemplate(o, app); err != nil {
-		return err
+		return echo.NewHTTPError(http.StatusBadRequest, err.Error())
+	}
+
+	// Subject is only relevant for fixed tx templates. For campaigns,
+	// the subject changes per campaign and is on models.Campaign.
+	if o.Type == models.TemplateTypeCampaign {
+		o.Subject = ""
 	}
 
-	out, err := app.core.CreateTemplate(o.Name, []byte(o.Body))
+	// Compile the template and validate.
+	if err := o.Compile(app.manager.GenericTemplateFuncs()); err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, err.Error())
+	}
+
+	// Create the template the in the DB.
+	out, err := app.core.CreateTemplate(o.Name, o.Type, o.Subject, []byte(o.Body))
 	if err != nil {
 		return err
 	}
 
-	return c.JSON(http.StatusOK, okResp{out})
+	// If it's a transactional template, cache it in the manager
+	// to be used for arbitrary incoming tx message pushes.
+	if o.Type == models.TemplateTypeTx {
+		app.manager.CacheTpl(out.ID, &o)
+	}
 
+	return c.JSON(http.StatusOK, okResp{out})
 }
 
 // handleUpdateTemplate handles template modification.
@@ -152,11 +175,27 @@ func handleUpdateTemplate(c echo.Context) error {
 		return echo.NewHTTPError(http.StatusBadRequest, err.Error())
 	}
 
-	out, err := app.core.UpdateTemplate(id, o.Name, []byte(o.Body))
+	// Subject is only relevant for fixed tx templates. For campaigns,
+	// the subject changes per campaign and is on models.Campaign.
+	if o.Type == models.TemplateTypeCampaign {
+		o.Subject = ""
+	}
+
+	// Compile the template and validate.
+	if err := o.Compile(app.manager.GenericTemplateFuncs()); err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, err.Error())
+	}
+
+	out, err := app.core.UpdateTemplate(id, o.Name, o.Type, o.Subject, []byte(o.Body))
 	if err != nil {
 		return err
 	}
 
+	// If it's a transactional template, cache it.
+	if o.Type == models.TemplateTypeTx {
+		app.manager.CacheTpl(out.ID, &o)
+	}
+
 	return c.JSON(http.StatusOK, okResp{out})
 
 }
@@ -194,19 +233,27 @@ func handleDeleteTemplate(c echo.Context) error {
 		return err
 	}
 
+	// Delete cached template.
+	app.manager.DeleteTpl(id)
+
 	return c.JSON(http.StatusOK, okResp{true})
 }
 
-// validateTemplate validates template fields.
+// compileTemplate validates template fields.
 func validateTemplate(o models.Template, app *App) error {
 	if !strHasLen(o.Name, 1, stdInputMaxLen) {
 		return errors.New(app.i18n.T("campaigns.fieldInvalidName"))
 	}
 
-	if !regexpTplTag.MatchString(o.Body) {
+	if o.Type == models.TemplateTypeCampaign && !regexpTplTag.MatchString(o.Body) {
 		return echo.NewHTTPError(http.StatusBadRequest,
 			app.i18n.Ts("templates.placeholderHelp", "placeholder", tplTag))
 	}
 
+	if o.Type == models.TemplateTypeTx && strings.TrimSpace(o.Subject) == "" {
+		return echo.NewHTTPError(http.StatusBadRequest,
+			app.i18n.Ts("globals.messages.missingFields", "name", "subject"))
+	}
+
 	return nil
 }

cmd/tx.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"net/http"
+	"net/textproto"
+
+	"github.com/knadh/listmonk/internal/manager"
+	"github.com/knadh/listmonk/models"
+	"github.com/labstack/echo/v4"
+)
+
+// handleSendTxMessage handles the sending of a transactional message.
+func handleSendTxMessage(c echo.Context) error {
+	var (
+		app = c.Get("app").(*App)
+		m   models.TxMessage
+	)
+
+	if err := c.Bind(&m); err != nil {
+		return err
+	}
+
+	// Validate input.
+	if r, err := validateTxMessage(m, app); err != nil {
+		return err
+	} else {
+		m = r
+	}
+
+	// Get the cached tx template.
+	tpl, err := app.manager.GetTpl(m.TemplateID)
+	if err != nil {
+		return err
+	}
+
+	// Get the subscriber.
+	sub, err := app.core.GetSubscriber(m.SubscriberID, "", m.SubscriberEmail)
+	if err != nil {
+		return err
+	}
+
+	// Render the message.
+	if err := m.Render(sub, tpl); err != nil {
+		return err
+	}
+
+	// Prepare the final message.
+	msg := manager.Message{}
+	msg.Subscriber = sub
+	msg.To = []string{sub.Email}
+	msg.From = m.FromEmail
+	msg.Subject = m.Subject
+	msg.ContentType = m.ContentType
+	msg.Messenger = m.Messenger
+	msg.Body = m.Body
+
+	// Optional headers.
+	if len(m.Headers) != 0 {
+		msg.Headers = make(textproto.MIMEHeader)
+		for _, set := range msg.Campaign.Headers {
+			for hdr, val := range set {
+				msg.Headers.Add(hdr, val)
+			}
+		}
+	}
+
+	if err := app.manager.PushMessage(msg); err != nil {
+		app.log.Printf("error sending message (%s): %v", msg.Subject, err)
+		return err
+	}
+
+	return c.JSON(http.StatusOK, okResp{true})
+}
+
+func validateTxMessage(m models.TxMessage, app *App) (models.TxMessage, error) {
+	if m.SubscriberEmail == "" && m.SubscriberID == 0 {
+		return m, echo.NewHTTPError(http.StatusBadRequest,
+			app.i18n.Ts("globals.messages.missingFields", "name", "subscriber_email or subscriber_id"))
+	}
+
+	if m.SubscriberEmail != "" {
+		em, err := app.importer.SanitizeEmail(m.SubscriberEmail)
+		if err != nil {
+			return m, echo.NewHTTPError(http.StatusBadRequest, err.Error())
+		}
+		m.SubscriberEmail = em
+	}
+
+	if m.FromEmail == "" {
+		m.FromEmail = app.constants.FromEmail
+	}
+
+	if m.Messenger == "" {
+		m.Messenger = emailMsgr
+	} else if !app.manager.HasMessenger(m.Messenger) {
+		return m, echo.NewHTTPError(http.StatusBadRequest, app.i18n.Ts("campaigns.fieldInvalidMessenger", "name", m.Messenger))
+	}
+
+	return m, nil
+}

cmd/upgrade.go

@@ -32,6 +32,7 @@ var migList = []migFunc{
 	{"v1.0.0", migrations.V1_0_0},
 	{"v2.0.0", migrations.V2_0_0},
 	{"v2.1.0", migrations.V2_1_0},
+	{"v2.2.0", migrations.V2_2_0},
 }
 
 // upgrade upgrades the database to the current version by running SQL migration files

frontend/src/assets/style.scss

@@ -521,14 +521,14 @@ body.is-noscroll {
     color: $grey;
   }
 
-  &.private, &.scheduled, &.paused {
+  &.private, &.scheduled, &.paused, &.tx {
     $color: #ed7b00;
     color: $color;
     background: #fff7e6;
     border: 1px solid lighten($color, 37%);
     box-shadow: 1px 1px 0 lighten($color, 37%);
   }
-  &.public, &.running, &.list {
+  &.public, &.running, &.list, &.campaign {
     $color: $primary;
     color: lighten($color, 20%);;
     background: #e6f7ff;
@@ -800,6 +800,11 @@ section.analytics {
 }
 
 /* Template form */
+.templates {
+  td .tag {
+    min-width: 100px;
+  }
+}
 .template-modal {
   .template-modal-content {
     height: 95vh;

frontend/src/views/Campaign.vue

@@ -84,7 +84,10 @@
                 <b-field :label="$tc('globals.terms.template')" label-position="on-border">
                   <b-select :placeholder="$tc('globals.terms.template')" v-model="form.templateId"
                     name="template" :disabled="!canEdit" required>
-                    <option v-for="t in templates" :value="t.id" :key="t.id">{{ t.name }}</option>
+                    <template v-for="t in templates">
+                      <option v-if="t.type === 'campaign'"
+                        :value="t.id" :key="t.id">{{ t.name }}</option>
+                    </template>
                   </b-select>
                 </b-field>