Skip to content

Commit

Permalink
store extras in cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@wunter8
wunter8 committed Sep 10, 2023
1 parent ed45794 commit 42c6d83
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 15 deletions.
60 changes: 50 additions & 10 deletions server/message_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ const (
sender TEXT NOT NULL,
user TEXT NOT NULL,
content_type TEXT NOT NULL,
extras TEXT NOT NULL,
encoding TEXT NOT NULL,
published INT NOT NULL
);
Expand All @@ -64,43 +65,43 @@ const (
COMMIT;
`
insertMessageQuery = `
INSERT INTO messages (mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_deleted, sender, user, content_type, encoding, published)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO messages (mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_deleted, sender, user, content_type, extras, encoding, published)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`
deleteMessageQuery = `DELETE FROM messages WHERE mid = ?`
updateMessagesForTopicExpiryQuery = `UPDATE messages SET expires = ? WHERE topic = ?`
selectRowIDFromMessageID = `SELECT id FROM messages WHERE mid = ?` // Do not include topic, see #336 and TestServer_PollSinceID_MultipleTopics
selectMessagesByIDQuery = `
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding
FROM messages
WHERE mid = ?
`
selectMessagesSinceTimeQuery = `
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding
FROM messages
WHERE topic = ? AND time >= ? AND published = 1
ORDER BY time, id
`
selectMessagesSinceTimeIncludeScheduledQuery = `
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding
FROM messages
WHERE topic = ? AND time >= ?
ORDER BY time, id
`
selectMessagesSinceIDQuery = `
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding
FROM messages
WHERE topic = ? AND id > ? AND published = 1
ORDER BY time, id
`
selectMessagesSinceIDIncludeScheduledQuery = `
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding
FROM messages
WHERE topic = ? AND (id > ? OR published = 0)
ORDER BY time, id
`
selectMessagesDueQuery = `
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, encoding
SELECT mid, time, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, content_type, extras, encoding
FROM messages
WHERE time <= ? AND published = 0
ORDER BY time, id
Expand All @@ -122,7 +123,7 @@ const (

// Schema management queries
const (
currentSchemaVersion = 12
currentSchemaVersion = 13
createSchemaVersionTableQuery = `
CREATE TABLE IF NOT EXISTS schemaVersion (
id INT PRIMARY KEY,
Expand Down Expand Up @@ -246,6 +247,11 @@ const (
migrate11To12AlterMessagesTableQuery = `
ALTER TABLE messages ADD COLUMN content_type TEXT NOT NULL DEFAULT('');
`

// 12 -> 13
migrate12To13AlterMessagesTableQuery = `
ALTER TABLE messages ADD COLUMN extras TEXT NOT NULL DEFAULT('');
`
)

var (
Expand All @@ -262,6 +268,7 @@ var (
9: migrateFrom9,
10: migrateFrom10,
11: migrateFrom11,
12: migrateFrom12,
}
)

Expand Down Expand Up @@ -367,6 +374,14 @@ func (c *messageCache) addMessages(ms []*message) error {
}
actionsStr = string(actionsBytes)
}
var extrasStr string
if len(m.Extras) > 0 {
extrasBytes, err := json.Marshal(m.Extras)
if err != nil {
return err
}
extrasStr = string(extrasBytes)
}
var sender string
if m.Sender.IsValid() {
sender = m.Sender.String()
Expand All @@ -392,6 +407,7 @@ func (c *messageCache) addMessages(ms []*message) error {
sender,
m.User,
m.ContentType,
extrasStr,
m.Encoding,
published,
)
Expand Down Expand Up @@ -664,7 +680,7 @@ func readMessages(rows *sql.Rows) ([]*message, error) {
func readMessage(rows *sql.Rows) (*message, error) {
var timestamp, expires, attachmentSize, attachmentExpires int64
var priority int
var id, topic, msg, title, tagsStr, click, icon, actionsStr, attachmentName, attachmentType, attachmentURL, sender, user, contentType, encoding string
var id, topic, msg, title, tagsStr, click, icon, actionsStr, attachmentName, attachmentType, attachmentURL, sender, user, contentType, extrasStr, encoding string
err := rows.Scan(
&id,
&timestamp,
Expand All @@ -685,6 +701,7 @@ func readMessage(rows *sql.Rows) (*message, error) {
&sender,
&user,
&contentType,
&extrasStr,
&encoding,
)
if err != nil {
Expand All @@ -700,6 +717,12 @@ func readMessage(rows *sql.Rows) (*message, error) {
return nil, err
}
}
var extras map[string]string
if extrasStr != "" {
if err := json.Unmarshal([]byte(extrasStr), &extras); err != nil {
return nil, err
}
}
senderIP, err := netip.ParseAddr(sender)
if err != nil {
senderIP = netip.Addr{} // if no IP stored in database, return invalid address
Expand Down Expand Up @@ -731,6 +754,7 @@ func readMessage(rows *sql.Rows) (*message, error) {
Sender: senderIP, // Must parse assuming database must be correct
User: user,
ContentType: contentType,
Extras: extras,
Encoding: encoding,
}, nil
}
Expand Down Expand Up @@ -970,3 +994,19 @@ func migrateFrom11(db *sql.DB, _ time.Duration) error {
}
return tx.Commit()
}

func migrateFrom12(db *sql.DB, _ time.Duration) error {
log.Tag(tagMessageCache).Info("Migrating cache database schema: from 12 to 13")
tx, err := db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.Exec(migrate12To13AlterMessagesTableQuery); err != nil {
return err
}
if _, err := tx.Exec(updateSchemaVersion, 13); err != nil {
return err
}
return tx.Commit()
}
12 changes: 7 additions & 5 deletions server/message_cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,25 +143,27 @@ func testCacheTopics(t *testing.T, c *messageCache) {
require.Equal(t, "topic2", topics["topic2"].ID)
}

func TestSqliteCache_MessagesTagsPrioAndTitle(t *testing.T) {
testCacheMessagesTagsPrioAndTitle(t, newSqliteTestCache(t))
func TestSqliteCache_MessagesTagsPrioTitleAndExtras(t *testing.T) {
testCacheMessagesTagsPrioTitleAndExtras(t, newSqliteTestCache(t))
}

func TestMemCache_MessagesTagsPrioAndTitle(t *testing.T) {
testCacheMessagesTagsPrioAndTitle(t, newMemTestCache(t))
func TestMemCache_MessagesTagsPrioTitleAndExtras(t *testing.T) {
testCacheMessagesTagsPrioTitleAndExtras(t, newMemTestCache(t))
}

func testCacheMessagesTagsPrioAndTitle(t *testing.T, c *messageCache) {
func testCacheMessagesTagsPrioTitleAndExtras(t *testing.T, c *messageCache) {
m := newDefaultMessage("mytopic", "some message")
m.Tags = []string{"tag1", "tag2"}
m.Priority = 5
m.Title = "some title"
m.Extras = map[string]string{"foo": "bar"}
require.Nil(t, c.AddMessage(m))

messages, _ := c.Messages("mytopic", sinceAllMessages, false)
require.Equal(t, []string{"tag1", "tag2"}, messages[0].Tags)
require.Equal(t, 5, messages[0].Priority)
require.Equal(t, "some title", messages[0].Title)
require.Equal(t, map[string]string{"foo": "bar"}, messages[0].Extras)
}

func TestSqliteCache_MessagesSinceID(t *testing.T) {
Expand Down

0 comments on commit 42c6d83

Please sign in to comment.