[experiment] add alternative wasm sqlite3 implementation available vi…

…a build-tag (#2863)

This allows for building GoToSocial with [SQLite transpiled to WASM](https://github.com/ncruces/go-sqlite3) and accessed through [Wazero](https://wazero.io/).

go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/miekg/dns v1.1.59
 	github.com/minio/minio-go/v7 v7.0.70
 	github.com/mitchellh/mapstructure v1.5.0
+	github.com/ncruces/go-sqlite3 v0.16.0
 	github.com/oklog/ulid v1.3.1
 	github.com/prometheus/client_golang v1.19.1
 	github.com/spf13/cobra v1.8.0
@@ -78,7 +79,7 @@ require (
 	golang.org/x/text v0.15.0
 	gopkg.in/mcuadros/go-syslog.v2 v2.3.0
 	gopkg.in/yaml.v3 v3.0.1
-	modernc.org/sqlite v1.29.8
+	modernc.org/sqlite v0.0.0-00010101000000-000000000000
 	mvdan.cc/xurls/v2 v2.5.0
 )
 
@@ -173,6 +174,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/ncruces/julianday v1.0.0 // indirect
 	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
@@ -197,6 +199,7 @@ require (
 	github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe // indirect
 	github.com/superseriousbusiness/go-png-image-structure/v2 v2.0.1-SSB // indirect
 	github.com/tdewolff/parse/v2 v2.7.14 // indirect
+	github.com/tetratelabs/wazero v1.7.2 // indirect
 	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
 	github.com/toqueteos/webbrowser v1.2.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect

go.sum

@@ -445,8 +445,12 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
+github.com/ncruces/go-sqlite3 v0.16.0 h1:O7eULuEjvSBnS1QCN+dDL/ixLQZoUGWr466A02Gx1xc=
+github.com/ncruces/go-sqlite3 v0.16.0/go.mod h1:2TmAeD93ImsKXJRsUIKohfMvt17dZSbS6pzJ3k6YYFg=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
+github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -558,6 +562,8 @@ github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03
 github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
 github.com/technologize/otel-go-contrib v1.1.1 h1:wZH9aSPNWZWIkEh3vfaKfMb15AJ80jJ1aVj/4GZdqIw=
 github.com/technologize/otel-go-contrib v1.1.1/go.mod h1:dCN/wj2WyUO8aFZFdIN+6tfJHImjTML/8r2YVYAy3So=
+github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
+github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
 github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E=
 github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
 github.com/tidwall/buntdb v1.1.2 h1:noCrqQXL9EKMtcdwJcmuVKSEjqu1ua99RHHgbLTEHRo=

internal/api/client/media/mediacreate_test.go

@@ -38,7 +38,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
@@ -77,19 +76,22 @@ type MediaCreateTestSuite struct {
 	TEST INFRASTRUCTURE
 */
 
-func (suite *MediaCreateTestSuite) SetupSuite() {
-	suite.state.Caches.Init()
+func (suite *MediaCreateTestSuite) SetupTest() {
 	testrig.StartNoopWorkers(&suite.state)
 
 	// setup standard items
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
 
-	suite.db = testrig.NewTestDB(&suite.state)
-	suite.state.DB = suite.db
+	suite.state.Caches.Init()
+
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 
+	suite.db = testrig.NewTestDB(&suite.state)
+	testrig.StandardDBSetup(suite.db, nil)
+	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
+
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(
@@ -106,21 +108,8 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
 
 	// setup module being tested
 	suite.mediaModule = mediamodule.New(suite.processor)
-}
-
-func (suite *MediaCreateTestSuite) TearDownSuite() {
-	if err := suite.db.Close(); err != nil {
-		log.Panicf(nil, "error closing db connection: %s", err)
-	}
-	testrig.StopWorkers(&suite.state)
-}
-
-func (suite *MediaCreateTestSuite) SetupTest() {
-	suite.state.Caches.Init()
-
-	testrig.StandardDBSetup(suite.db, nil)
-	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 
+	// setup test data
 	suite.testTokens = testrig.NewTestTokens()
 	suite.testClients = testrig.NewTestClients()
 	suite.testApplications = testrig.NewTestApplications()
@@ -132,6 +121,7 @@ func (suite *MediaCreateTestSuite) SetupTest() {
 func (suite *MediaCreateTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 	testrig.StandardStorageTeardown(suite.storage)
+	testrig.StopWorkers(&suite.state)
 }
 
 /*

internal/api/client/media/mediaupdate_test.go

@@ -36,7 +36,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/processing"
@@ -75,18 +74,22 @@ type MediaUpdateTestSuite struct {
 	TEST INFRASTRUCTURE
 */
 
-func (suite *MediaUpdateTestSuite) SetupSuite() {
+func (suite *MediaUpdateTestSuite) SetupTest() {
 	testrig.StartNoopWorkers(&suite.state)
 
 	// setup standard items
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
 
-	suite.db = testrig.NewTestDB(&suite.state)
-	suite.state.DB = suite.db
+	suite.state.Caches.Init()
+
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 
+	suite.db = testrig.NewTestDB(&suite.state)
+	testrig.StandardDBSetup(suite.db, nil)
+	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
+
 	suite.tc = typeutils.NewConverter(&suite.state)
 
 	testrig.StartTimelines(
@@ -103,21 +106,8 @@ func (suite *MediaUpdateTestSuite) SetupSuite() {
 
 	// setup module being tested
 	suite.mediaModule = mediamodule.New(suite.processor)
-}
-
-func (suite *MediaUpdateTestSuite) TearDownSuite() {
-	if err := suite.db.Close(); err != nil {
-		log.Panicf(nil, "error closing db connection: %s", err)
-	}
-	testrig.StopWorkers(&suite.state)
-}
-
-func (suite *MediaUpdateTestSuite) SetupTest() {
-	suite.state.Caches.Init()
-
-	testrig.StandardDBSetup(suite.db, nil)
-	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 
+	// setup test data
 	suite.testTokens = testrig.NewTestTokens()
 	suite.testClients = testrig.NewTestClients()
 	suite.testApplications = testrig.NewTestApplications()
@@ -129,6 +119,7 @@ func (suite *MediaUpdateTestSuite) SetupTest() {
 func (suite *MediaUpdateTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 	testrig.StandardStorageTeardown(suite.storage)
+	testrig.StopWorkers(&suite.state)
 }
 
 /*

internal/api/fileserver/fileserver_test.go

@@ -70,8 +70,6 @@ func (suite *FileserverTestSuite) SetupSuite() {
 	testrig.InitTestConfig()
 	testrig.InitTestLog()
 
-	suite.db = testrig.NewTestDB(&suite.state)
-	suite.state.DB = suite.db
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 
@@ -98,8 +96,12 @@ func (suite *FileserverTestSuite) SetupTest() {
 	suite.state.Caches.Init()
 	testrig.StartNoopWorkers(&suite.state)
 
+	suite.db = testrig.NewTestDB(&suite.state)
+	suite.state.DB = suite.db
+
 	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
+
 	suite.testTokens = testrig.NewTestTokens()
 	suite.testClients = testrig.NewTestClients()
 	suite.testApplications = testrig.NewTestApplications()

internal/config/global.go

@@ -52,3 +52,9 @@ func LoadEarlyFlags(cmd *cobra.Command) error {
 func BindFlags(cmd *cobra.Command) error {
 	return global.BindFlags(cmd)
 }
+
+// Reset will totally clear global
+// ConfigState{}, loading defaults.
+func Reset() {
+	global.Reset()
+}

internal/config/state.go

@@ -37,25 +37,9 @@ type ConfigState struct { //nolint
 
 // NewState returns a new initialized ConfigState instance.
 func NewState() *ConfigState {
-	viper := viper.New()
-
-	// Flag 'some-flag-name' becomes env var 'GTS_SOME_FLAG_NAME'
-	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
-	viper.SetEnvPrefix("gts")
-
-	// Load appropriate named vals from env
-	viper.AutomaticEnv()
-
-	// Create new ConfigState with defaults
-	state := &ConfigState{
-		viper:  viper,
-		config: Defaults,
-	}
-
-	// Perform initial load into viper
-	state.reloadToViper()
-
-	return state
+	st := new(ConfigState)
+	st.Reset()
+	return st
 }
 
 // Config provides safe access to the ConfigState's contained Configuration,
@@ -116,6 +100,32 @@ func (st *ConfigState) Reload() (err error) {
 	return
 }
 
+// Reset will totally clear
+// ConfigState{}, loading defaults.
+func (st *ConfigState) Reset() {
+	// Do within lock.
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+
+	// Create new viper.
+	viper := viper.New()
+
+	// Flag 'some-flag-name' becomes env var 'GTS_SOME_FLAG_NAME'
+	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+	viper.SetEnvPrefix("gts")
+
+	// Load appropriate
+	// named vals from env.
+	viper.AutomaticEnv()
+
+	// Reset variables.
+	st.viper = viper
+	st.config = Defaults
+
+	// Load into viper.
+	st.reloadToViper()
+}
+
 // reloadToViper will reload Configuration{} values into viper.
 func (st *ConfigState) reloadToViper() {
 	raw, err := st.config.MarshalMap()

internal/db/bundb/admin_test.go

@@ -74,6 +74,7 @@ func (suite *AdminTestSuite) TestCreateInstanceAccount() {
 	// we need to take an empty db for this...
 	testrig.StandardDBTeardown(suite.db)
 	// ...with tables created but no data
+	suite.db = testrig.NewTestDB(&suite.state)
 	testrig.CreateTestTables(suite.db)
 
 	// make sure there's no instance account in the db yet

internal/db/bundb/bundb.go

@@ -48,8 +48,6 @@ import (
 	"github.com/uptrace/bun/dialect/pgdialect"
 	"github.com/uptrace/bun/dialect/sqlitedialect"
 	"github.com/uptrace/bun/migrate"
-
-	"modernc.org/sqlite"
 )
 
 // DBService satisfies the DB interface
@@ -133,12 +131,12 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
 
 	switch t {
 	case "postgres":
-		db, err = pgConn(ctx, state)
+		db, err = pgConn(ctx)
 		if err != nil {
 			return nil, err
 		}
 	case "sqlite":
-		db, err = sqliteConn(ctx, state)
+		db, err = sqliteConn(ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -295,7 +293,7 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
 	return ps, nil
 }
 
-func pgConn(ctx context.Context, state *state.State) (*bun.DB, error) {
+func pgConn(ctx context.Context) (*bun.DB, error) {
 	opts, err := deriveBunDBPGOptions() //nolint:contextcheck
 	if err != nil {
 		return nil, fmt.Errorf("could not create bundb postgres options: %w", err)
@@ -326,7 +324,7 @@ func pgConn(ctx context.Context, state *state.State) (*bun.DB, error) {
 	return db, nil
 }
 
-func sqliteConn(ctx context.Context, state *state.State) (*bun.DB, error) {
+func sqliteConn(ctx context.Context) (*bun.DB, error) {
 	// validate db address has actually been set
 	address := config.GetDbAddress()
 	if address == "" {
@@ -339,9 +337,6 @@ func sqliteConn(ctx context.Context, state *state.State) (*bun.DB, error) {
 	// Open new DB instance
 	sqldb, err := sql.Open("sqlite-gts", address)
 	if err != nil {
-		if errWithCode, ok := err.(*sqlite.Error); ok {
-			err = errors.New(sqlite.ErrorCodeString[errWithCode.Code()])
-		}
 		return nil, fmt.Errorf("could not open sqlite db with address %s: %w", address, err)
 	}
 
@@ -356,11 +351,9 @@ func sqliteConn(ctx context.Context, state *state.State) (*bun.DB, error) {
 
 	// ping to check the db is there and listening
 	if err := db.PingContext(ctx); err != nil {
-		if errWithCode, ok := err.(*sqlite.Error); ok {
-			err = errors.New(sqlite.ErrorCodeString[errWithCode.Code()])
-		}
 		return nil, fmt.Errorf("sqlite ping: %w", err)
 	}
+
 	log.Infof(ctx, "connected to SQLITE database with address %s", address)
 
 	return db, nil
@@ -528,12 +521,8 @@ func buildSQLiteAddress(addr string) string {
 
 		// Use random name for in-memory instead of ':memory:', so
 		// multiple in-mem databases can be created without conflict.
-		addr = uuid.NewString()
-
-		// in-mem-specific preferences
-		// (shared cache so that tests don't fail)
-		prefs.Add("mode", "memory")
-		prefs.Add("cache", "shared")
+		addr = "/" + uuid.NewString()
+		prefs.Add("vfs", "memdb")
 	}
 
 	if dur := config.GetDbSqliteBusyTimeout(); dur > 0 {