From 42c6d831df05b47f2b0015a6a39b63f049cae201 Mon Sep 17 00:00:00 2001 From: Hunter Kehoe Date: Sun, 10 Sep 2023 10:50:12 -0600 Subject: [PATCH] store extras in cache --- server/message_cache.go | 60 ++++++++++++++++++++++++++++++------ server/message_cache_test.go | 12 +++++--- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/server/message_cache.go b/server/message_cache.go index 8a613ff1c..fb609ecde 100644 --- a/server/message_cache.go +++ b/server/message_cache.go @@ -46,6 +46,7 @@ const ( sender TEXT NOT NULL, user TEXT NOT NULL, content_type TEXT NOT NULL, + extras TEXT NOT NULL, encoding TEXT NOT NULL, published INT NOT NULL ); @@ -64,43 +65,43 @@ const ( COMMIT; ` insertMessageQuery = ` - INSERT INTO messages (mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_deleted, sender, user, content_type, encoding, published) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO messages (mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_deleted, sender, user, content_type, extras, encoding, published) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` deleteMessageQuery = `DELETE FROM messages WHERE mid = ?` updateMessagesForTopicExpiryQuery = `UPDATE messages SET expires = ? WHERE topic = ?` selectRowIDFromMessageID = `SELECT id FROM messages WHERE mid = ?` // Do not include topic, see #336 and TestServer_PollSinceID_MultipleTopics selectMessagesByIDQuery = ` - SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding + SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding FROM messages WHERE mid = ? ` selectMessagesSinceTimeQuery = ` - SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding + SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding FROM messages WHERE topic = ? AND time >= ? AND published = 1 ORDER BY time, id ` selectMessagesSinceTimeIncludeScheduledQuery = ` - SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding + SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding FROM messages WHERE topic = ? AND time >= ? ORDER BY time, id ` selectMessagesSinceIDQuery = ` - SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding + SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding FROM messages WHERE topic = ? AND id > ? AND published = 1 ORDER BY time, id ` selectMessagesSinceIDIncludeScheduledQuery = ` - SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding + SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding FROM messages WHERE topic = ? AND (id > ? OR published = 0) ORDER BY time, id ` selectMessagesDueQuery = ` - SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding + SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding FROM messages WHERE time <= ? AND published = 0 ORDER BY time, id @@ -122,7 +123,7 @@ const ( // Schema management queries const ( - currentSchemaVersion = 12 + currentSchemaVersion = 13 createSchemaVersionTableQuery = ` CREATE TABLE IF NOT EXISTS schemaVersion ( id INT PRIMARY KEY, @@ -246,6 +247,11 @@ const ( migrate11To12AlterMessagesTableQuery = ` ALTER TABLE messages ADD COLUMN content_type TEXT NOT NULL DEFAULT(''); ` + + // 12 -> 13 + migrate12To13AlterMessagesTableQuery = ` + ALTER TABLE messages ADD COLUMN extras TEXT NOT NULL DEFAULT(''); + ` ) var ( @@ -262,6 +268,7 @@ var ( 9: migrateFrom9, 10: migrateFrom10, 11: migrateFrom11, + 12: migrateFrom12, } ) @@ -367,6 +374,14 @@ func (c *messageCache) addMessages(ms []*message) error { } actionsStr = string(actionsBytes) } + var extrasStr string + if len(m.Extras) > 0 { + extrasBytes, err := json.Marshal(m.Extras) + if err != nil { + return err + } + extrasStr = string(extrasBytes) + } var sender string if m.Sender.IsValid() { sender = m.Sender.String() @@ -392,6 +407,7 @@ func (c *messageCache) addMessages(ms []*message) error { sender, m.User, m.ContentType, + extrasStr, m.Encoding, published, ) @@ -664,7 +680,7 @@ func readMessages(rows *sql.Rows) ([]*message, error) { func readMessage(rows *sql.Rows) (*message, error) { var timestamp, expires, attachmentSize, attachmentExpires int64 var priority int - var id, topic, msg, title, tagsStr, click, icon, actionsStr, attachmentName, attachmentType, attachmentURL, sender, user, contentType, encoding string + var id, topic, msg, title, tagsStr, click, icon, actionsStr, attachmentName, attachmentType, attachmentURL, sender, user, contentType, extrasStr, encoding string err := rows.Scan( &id, ×tamp, @@ -685,6 +701,7 @@ func readMessage(rows *sql.Rows) (*message, error) { &sender, &user, &contentType, + &extrasStr, &encoding, ) if err != nil { @@ -700,6 +717,12 @@ func readMessage(rows *sql.Rows) (*message, error) { return nil, err } } + var extras map[string]string + if extrasStr != "" { + if err := json.Unmarshal([]byte(extrasStr), &extras); err != nil { + return nil, err + } + } senderIP, err := netip.ParseAddr(sender) if err != nil { senderIP = netip.Addr{} // if no IP stored in database, return invalid address @@ -731,6 +754,7 @@ func readMessage(rows *sql.Rows) (*message, error) { Sender: senderIP, // Must parse assuming database must be correct User: user, ContentType: contentType, + Extras: extras, Encoding: encoding, }, nil } @@ -970,3 +994,19 @@ func migrateFrom11(db *sql.DB, _ time.Duration) error { } return tx.Commit() } + +func migrateFrom12(db *sql.DB, _ time.Duration) error { + log.Tag(tagMessageCache).Info("Migrating cache database schema: from 12 to 13") + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(migrate12To13AlterMessagesTableQuery); err != nil { + return err + } + if _, err := tx.Exec(updateSchemaVersion, 13); err != nil { + return err + } + return tx.Commit() +} diff --git a/server/message_cache_test.go b/server/message_cache_test.go index 79b7fc54c..ff334ab00 100644 --- a/server/message_cache_test.go +++ b/server/message_cache_test.go @@ -143,25 +143,27 @@ func testCacheTopics(t *testing.T, c *messageCache) { require.Equal(t, "topic2", topics["topic2"].ID) } -func TestSqliteCache_MessagesTagsPrioAndTitle(t *testing.T) { - testCacheMessagesTagsPrioAndTitle(t, newSqliteTestCache(t)) +func TestSqliteCache_MessagesTagsPrioTitleAndExtras(t *testing.T) { + testCacheMessagesTagsPrioTitleAndExtras(t, newSqliteTestCache(t)) } -func TestMemCache_MessagesTagsPrioAndTitle(t *testing.T) { - testCacheMessagesTagsPrioAndTitle(t, newMemTestCache(t)) +func TestMemCache_MessagesTagsPrioTitleAndExtras(t *testing.T) { + testCacheMessagesTagsPrioTitleAndExtras(t, newMemTestCache(t)) } -func testCacheMessagesTagsPrioAndTitle(t *testing.T, c *messageCache) { +func testCacheMessagesTagsPrioTitleAndExtras(t *testing.T, c *messageCache) { m := newDefaultMessage("mytopic", "some message") m.Tags = []string{"tag1", "tag2"} m.Priority = 5 m.Title = "some title" + m.Extras = map[string]string{"foo": "bar"} require.Nil(t, c.AddMessage(m)) messages, _ := c.Messages("mytopic", sinceAllMessages, false) require.Equal(t, []string{"tag1", "tag2"}, messages[0].Tags) require.Equal(t, 5, messages[0].Priority) require.Equal(t, "some title", messages[0].Title) + require.Equal(t, map[string]string{"foo": "bar"}, messages[0].Extras) } func TestSqliteCache_MessagesSinceID(t *testing.T) {