Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Simple read-only SQLite databases embedded in Go executables #1188

Closed
wants to merge 1 commit into from
Closed

Simple read-only SQLite databases embedded in Go executables #1188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions sqlite3_opt_serialize_embed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
// +build go1.21
// +build !libsqlite3 sqlite_serialize

package sqlite3

/*
#ifndef USE_LIBSQLITE3
#include <sqlite3-binding.h>
#else
#include <sqlite3.h>
#endif
#include <stdlib.h>
#include <stdint.h>
*/
import "C"

import (
"fmt"
"runtime"
"unsafe"
)

// This is the reason for requiring go1.21
var embedPin = runtime.Pinner{}

// DeserializeEmbedded is different from Deserialize in that it does NOT copy
// memory and does NOT free the memory on close. In fact it pins the memory,
// and does not give you a way to unpin. Therefore this should only be used for
// an embedded, read-only database that stays for the lifetime of your process.
//
// Because of how initial DB must be opened this can only have schema of "main".
// Leaving the argument in case that ever changes (the simplest would be if SQLite
// added a mode of "rom" or similar. But it can't be fixed just in go).
func (c *SQLiteConn) DeserializeEmbedded(b []byte, schema string) error {
// "main" is the only value `schema` can be because it must be opened with:
// "file::memory:?mode=ro"
// "file::memory:?mode=ro&cache=shared"
// "file::memory:?mode=ro&cache=private"
// See https://github.com/mattn/go-sqlite3/issues/204 as this means you
// can embed exactly one shared DB into your binary at a time.
// Maybe there will eventually be a way to open like:
// "file:unique_name.db?mode=rom"
// Then we would use the `schema` argument.
schema = "main"
var zSchema *C.char
zSchema = C.CString(schema)
defer C.free(unsafe.Pointer(zSchema))

// Pinning will future proof for a GC that moves memory. It is also
// necessary now because nothing else prevents a reassignment of `b`
// which would let GC reclaim `b`.
rom := unsafe.Pointer(&b[0])
embedPin.Pin(rom)

rc := C.sqlite3_deserialize(c.db, zSchema, (*C.uchar)(rom),
C.sqlite3_int64(len(b)), C.sqlite3_int64(len(b)),
C.SQLITE_DESERIALIZE_READONLY)
if rc != C.SQLITE_OK {
return fmt.Errorf("deserialize failed with return %v", rc)
}
return nil
}
14 changes: 14 additions & 0 deletions sqlite3_opt_serialize_embed_omit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// +build !go1.21
// +build libsqlite3,!sqlite_serialize

package sqlite3

import (
"errors"
)

import "C"

func (c *SQLiteConn) DeserializeEmbedded(b []byte, schema string) error {
return errors.New("sqlite3: DeserializeEmbedded requires go1.21+ and the sqlite_serialize build tag when using the libsqlite3 build tag")
}
193 changes: 193 additions & 0 deletions sqlite3_opt_serialize_embed_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
// +build go1.21
// +build !libsqlite3 sqlite_serialize

package sqlite3

import (
"context"
"database/sql"
"testing"
)

func TestDeserializeEmbedded(t *testing.T) {
// Connect to the database. This is one of 3 ways we can get a read-only
// in memory database for now.
db, err := sql.Open("sqlite3", "file::memory:?mode=ro")
if err != nil {
t.Fatal("Failed to open the source database:", err)
}
defer db.Close()

err = db.Ping()
if err != nil {
t.Fatal("Failed to connect to the database:", err)
}

// Confirm that the database is initially empty.
var tableCount int
err = db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'").Scan(&tableCount)
if err != nil {
t.Fatal("Failed to check the database table count:", err)
}
if tableCount != 0 {
t.Fatalf("The database is not empty; %v table(s) found.", tableCount)
}

// Deserialize db_img to database
conn, err := db.Conn(context.Background())
if err != nil {
t.Fatal("Failed to get connection to database:", err)
}
defer conn.Close()

if err := conn.Raw(func(raw interface{}) error {
// db_img is below but normally would come from embed.
return raw.(*SQLiteConn).DeserializeEmbedded(db_img, "")
}); err != nil {
t.Fatal("Failed to deserialize database image:", err)
}
conn.Close()

// Confirm that database has been loaded correctly.
var rowCount int
err = db.QueryRow(`SELECT COUNT(*) FROM foo`).Scan(&rowCount)
if err != nil {
t.Fatal("Failed to count rows in database table", err)
}
if rowCount != 3 { // 5,2137,44
t.Fatal("Database table does not have the expected count of records")
}

// Finally confirm we can't INSERT into database
_, err = db.Exec("INSERT INTO foo (bar) VALUES(42)")
if err == nil {
// Would probably crash if we get here!
t.Fatal("Database allowed insert")
}

// DeserializeEmbedded again, should be possible to re-pin the same
// memory again without issue. In theory db_img the Slice could have
// moved, but the underlying bytes shouldn't have since they were
// pinned for the first DeserializeEmbedded.
conn, err = db.Conn(context.Background())
if err != nil {
t.Fatal("Failed to get connection to database:", err)
}
defer conn.Close()

if err := conn.Raw(func(raw interface{}) error {
return raw.(*SQLiteConn).DeserializeEmbedded(db_img, "")
}); err != nil {
t.Fatal("Failed second deserialize of database image:", err)
}
conn.Close()

// Now verify that if we change db_img it doesn't matter.
// This would be equally as bad as the GC moving the memory.
db_img = []byte{ 0x00 }
err = db.QueryRow(`SELECT COUNT(*) FROM foo`).Scan(&rowCount)
if err != nil {
t.Fatal("Failed to count rows in database table", err)
}
if rowCount != 3 { // 5,2137,44
t.Fatal("Database table does not have the expected count of records")
}
}

/*
* This avoids us checking in a database to the code but is essentially the
* same as using embed with this database:
*
* sqlite3 test.db
* sqlite> PRAGMA page_size=512;
* sqlite> CREATE TABLE foo (bar int);
* sqlite> INSERT INTO foo (bar) VALUES (5),(2137),(44);
*/
var db_img = []byte{
0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61,
0x74, 0x20, 0x33, 0x00, 0x02, 0x00, 0x01, 0x01, 0x0c, 0x40, 0x20, 0x20,
0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
0x00, 0x2e, 0x5f, 0x1d, 0x0d, 0x00, 0x00, 0x00, 0x01, 0x01, 0xc6, 0x00,
0x01, 0xc6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x01,
0x06, 0x17, 0x13, 0x13, 0x01, 0x41, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x66,
0x6f, 0x6f, 0x66, 0x6f, 0x6f, 0x02, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45,
0x20, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x20, 0x66, 0x6f, 0x6f, 0x20, 0x28,
0x62, 0x61, 0x72, 0x20, 0x69, 0x6e, 0x74, 0x29, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00,
0x03, 0x01, 0xe4, 0x00, 0x01, 0xef, 0x01, 0xe9, 0x01, 0xe4, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x03, 0x03, 0x02, 0x01, 0x2c, 0x04, 0x02, 0x02, 0x02, 0x08, 0x59, 0x03,
0x01, 0x02, 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
}
2 changes: 1 addition & 1 deletion sqlite3_opt_serialize_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,6 @@ func TestSerializeDeserialize(t *testing.T) {
t.Fatal("Failed to count rows in destination database table", err)
}
if destRowCount != 1 {
t.Fatalf("Destination table does not have the expected records")
t.Fatal("Destination table does not have the expected records")
}
}