From a1b010850382bf0a9cdb08f1752f74951b8de339 Mon Sep 17 00:00:00 2001
From: Jason Kulatunga <jason@thesparktree.com>
Date: Tue, 2 Aug 2022 22:14:23 -0700
Subject: [PATCH] Added PRAGMA settings support when connecting to SQLITE db.

When a transaction cannot lock the database, because it is already locked by another one,
SQLite by default throws an error: database is locked. This behavior is usually not appropriate when
concurrent access is needed, typically when multiple processes write to the same database.
PRAGMA busy_timeout lets you set a timeout or a handler for these events. When setting a timeout,
SQLite will try the transaction multiple times within this timeout.
https://rsqlite.r-dbi.org/reference/sqlitesetbusyhandler

retrying for 30000 milliseconds, 30seconds - this would be unreasonable for a distributed multi-tenant application,
but should be fine for local usage.

added mechanism for global settings (PRAGMA and DB level instructions).

fixes #341
---
 .../pkg/database/scrutiny_repository.go       | 28 ++++++++++++++++++-
 .../scrutiny_repository_migrations.go         | 24 ++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/webapp/backend/pkg/database/scrutiny_repository.go b/webapp/backend/pkg/database/scrutiny_repository.go
index 81f2316e..521ba7d0 100644
--- a/webapp/backend/pkg/database/scrutiny_repository.go
+++ b/webapp/backend/pkg/database/scrutiny_repository.go
@@ -62,7 +62,20 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field
 	// Gorm/SQLite setup
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 	globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
-	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
+
+	// When a transaction cannot lock the database, because it is already locked by another one,
+	// SQLite by default throws an error: database is locked. This behavior is usually not appropriate when
+	// concurrent access is needed, typically when multiple processes write to the same database.
+	// PRAGMA busy_timeout lets you set a timeout or a handler for these events. When setting a timeout,
+	// SQLite will try the transaction multiple times within this timeout.
+	// fixes #341
+	// https://rsqlite.r-dbi.org/reference/sqlitesetbusyhandler
+	// retrying for 30000 milliseconds, 30seconds - this would be unreasonable for a distributed multi-tenant application,
+	// but should be fine for local usage.
+	pragmaStr := sqlitePragmaString(map[string]string{
+		"busy_timeout": "30000",
+	})
+	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")+pragmaStr), &gorm.Config{
 		//TODO: figure out how to log database queries again.
 		//Logger: logger
 		DisableForeignKeyConstraintWhenMigrating: true,
@@ -450,3 +463,16 @@ func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []str
 	}
 	return []string{DURATION_KEY_WEEK}
 }
+
+func sqlitePragmaString(pragmas map[string]string) string {
+	q := url.Values{}
+	for key, val := range pragmas {
+		q.Add("_pragma", key+"="+val)
+	}
+
+	queryStr := q.Encode()
+	if len(queryStr) > 0 {
+		return "?" + queryStr
+	}
+	return ""
+}
diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go
index 99eb1f3f..015428cc 100644
--- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go
+++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go
@@ -355,6 +355,30 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
 		return err
 	}
 	sr.logger.Infoln("Database migration completed successfully")
+
+	//these migrations cannot be done within a transaction, so they are done as a separate group, with `UseTransaction = false`
+	sr.logger.Infoln("SQLite global configuration migrations starting. Please wait....")
+	globalMigrateOptions := gormigrate.DefaultOptions
+	globalMigrateOptions.UseTransaction = false
+	gm := gormigrate.New(sr.gormClient, globalMigrateOptions, []*gormigrate.Migration{
+		{
+			ID: "g20220802211500",
+			Migrate: func(tx *gorm.DB) error {
+				//shrink the Database (maybe necessary after 20220503113100)
+				if err := tx.Exec("VACUUM;").Error; err != nil {
+					return err
+				}
+				return nil
+			},
+		},
+	})
+
+	if err := gm.Migrate(); err != nil {
+		sr.logger.Errorf("SQLite global configuration migrations failed with error. \n Please open a github issue at https://github.com/AnalogJ/scrutiny and attach a copy of your scrutiny.db file. \n %v", err)
+		return err
+	}
+	sr.logger.Infoln("SQLite global configuration migrations completed successfully")
+
 	return nil
 }