From 0c67cf25f78994d471bf13cacb207f5673e22d7f Mon Sep 17 00:00:00 2001 From: DingDongSoLong4 <99329275+DingDongSoLong4@users.noreply.github.com> Date: Thu, 20 Jul 2023 01:07:23 +0200 Subject: [PATCH 1/3] Add database optimise task --- graphql/documents/mutations/metadata.graphql | 4 ++ graphql/schema/schema.graphql | 3 + internal/api/resolver_mutation_metadata.go | 5 ++ internal/manager/manager_tasks.go | 8 +++ internal/manager/task_optimise.go | 56 +++++++++++++++++++ pkg/sqlite/anonymise.go | 2 +- pkg/sqlite/database.go | 27 ++++++--- .../Settings/Tasks/DataManagementTasks.tsx | 42 +++++++++++++- ui/v2.5/src/core/StashService.ts | 5 ++ ui/v2.5/src/locales/en-GB.json | 3 + 10 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 internal/manager/task_optimise.go diff --git a/graphql/documents/mutations/metadata.graphql b/graphql/documents/mutations/metadata.graphql index 0d6486bed41..351105eb179 100644 --- a/graphql/documents/mutations/metadata.graphql +++ b/graphql/documents/mutations/metadata.graphql @@ -45,3 +45,7 @@ mutation BackupDatabase($input: BackupDatabaseInput!) { mutation AnonymiseDatabase($input: AnonymiseDatabaseInput!) { anonymiseDatabase(input: $input) } + +mutation OptimiseDatabase { + optimiseDatabase +} diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 3a4f6e738f5..a755487a2cb 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -314,6 +314,9 @@ type Mutation { """Anonymise the database in a separate file. Optionally returns a link to download the database file""" anonymiseDatabase(input: AnonymiseDatabaseInput!): String + """Optimises the database. Returns the job ID""" + optimiseDatabase: ID! + """Reload scrapers""" reloadScrapers: Boolean! diff --git a/internal/api/resolver_mutation_metadata.go b/internal/api/resolver_mutation_metadata.go index 6b0eba66f98..46e28581d19 100644 --- a/internal/api/resolver_mutation_metadata.go +++ b/internal/api/resolver_mutation_metadata.go @@ -208,3 +208,8 @@ func (r *mutationResolver) AnonymiseDatabase(ctx context.Context, input Anonymis return nil, nil } + +func (r *mutationResolver) OptimiseDatabase(ctx context.Context) (string, error) { + jobID := manager.GetInstance().OptimiseDatabase(ctx) + return strconv.Itoa(jobID), nil +} diff --git a/internal/manager/manager_tasks.go b/internal/manager/manager_tasks.go index d4935bee7d0..cd6d30203f9 100644 --- a/internal/manager/manager_tasks.go +++ b/internal/manager/manager_tasks.go @@ -265,6 +265,14 @@ func (s *Manager) Clean(ctx context.Context, input CleanMetadataInput) int { return s.JobManager.Add(ctx, "Cleaning...", &j) } +func (s *Manager) OptimiseDatabase(ctx context.Context) int { + j := OptimiseDatabaseJob{ + Optimiser: s.Database, + } + + return s.JobManager.Add(ctx, "Optimising database...", &j) +} + func (s *Manager) MigrateHash(ctx context.Context) int { j := job.MakeJobExec(func(ctx context.Context, progress *job.Progress) { fileNamingAlgo := config.GetInstance().GetVideoFileNamingAlgorithm() diff --git a/internal/manager/task_optimise.go b/internal/manager/task_optimise.go new file mode 100644 index 00000000000..2fb1794fb9c --- /dev/null +++ b/internal/manager/task_optimise.go @@ -0,0 +1,56 @@ +package manager + +import ( + "context" + "time" + + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" +) + +type Optimiser interface { + Analyze(ctx context.Context) error + Vacuum(ctx context.Context) error +} + +type OptimiseDatabaseJob struct { + Optimiser Optimiser +} + +func (j *OptimiseDatabaseJob) Execute(ctx context.Context, progress *job.Progress) { + logger.Info("Optimising database") + progress.SetTotal(2) + + start := time.Now() + + var err error + + progress.ExecuteTask("Analyzing database", func() { + err = j.Optimiser.Analyze(ctx) + progress.Increment() + }) + if job.IsCancelled(ctx) { + logger.Info("Stopping due to user request") + return + } + if err != nil { + logger.Errorf("Error analyzing database: %v", err) + return + } + + progress.ExecuteTask("Vacuuming database", func() { + err = j.Optimiser.Vacuum(ctx) + progress.Increment() + }) + if job.IsCancelled(ctx) { + logger.Info("Stopping due to user request") + return + } + if err != nil { + logger.Errorf("Error vacuuming database: %v", err) + return + } + + elapsed := time.Since(start) + logger.Infof("Finished optimising database after %s", elapsed) +} diff --git a/pkg/sqlite/anonymise.go b/pkg/sqlite/anonymise.go index a62b8d3c5ec..d8e6d99d6ad 100644 --- a/pkg/sqlite/anonymise.go +++ b/pkg/sqlite/anonymise.go @@ -58,7 +58,7 @@ func (db *Anonymiser) Anonymise(ctx context.Context) error { func() error { return db.anonymiseStudios(ctx) }, func() error { return db.anonymiseTags(ctx) }, func() error { return db.anonymiseMovies(ctx) }, - func() error { db.optimise(); return nil }, + func() error { return db.Optimise(ctx) }, }) }(); err != nil { // delete the database diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index 12562b5c68e..add9305d052 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -435,21 +435,28 @@ func (db *Database) RunMigrations() error { } // optimize database after migration - db.optimise() + err = db.Optimise(ctx) + if err != nil { + logger.Warnf("error while performing post-migration optimisation: %v", err) + } return nil } -func (db *Database) optimise() { - logger.Info("Optimizing database") - _, err := db.db.Exec("ANALYZE") +func (db *Database) Optimise(ctx context.Context) error { + logger.Info("Optimising database") + + err := db.Analyze(ctx) if err != nil { - logger.Warnf("error while performing post-migration optimization: %v", err) + return err } - _, err = db.db.Exec("VACUUM") + + err = db.Vacuum(ctx) if err != nil { - logger.Warnf("error while performing post-migration vacuum: %v", err) + return err } + + return nil } // Vacuum runs a VACUUM on the database, rebuilding the database file into a minimal amount of disk space. @@ -458,6 +465,12 @@ func (db *Database) Vacuum(ctx context.Context) error { return err } +// Analyze runs an ANALYZE on the database to improve query performance. +func (db *Database) Analyze(ctx context.Context) error { + _, err := db.db.ExecContext(ctx, "ANALYZE") + return err +} + func (db *Database) runCustomMigrations(ctx context.Context, fns []customMigrationFunc) error { for _, fn := range fns { if err := db.runCustomMigration(ctx, fn); err != nil { diff --git a/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx b/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx index 813620a041c..cbcca94a161 100644 --- a/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx +++ b/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx @@ -10,6 +10,7 @@ import { mutateAnonymiseDatabase, mutateMigrateSceneScreenshots, mutateMigrateBlobs, + mutateOptimiseDatabase, } from "src/core/StashService"; import { useToast } from "src/hooks/Toast"; import downloadFile from "src/utils/download"; @@ -338,6 +339,24 @@ export const DataManagementTasks: React.FC = ({ } } + async function onOptimiseDatabase() { + try { + await mutateOptimiseDatabase(); + Toast.success({ + content: intl.formatMessage( + { id: "config.tasks.added_job_to_queue" }, + { + operation_name: intl.formatMessage({ + id: "actions.optimise_database", + }), + } + ), + }); + } catch (e) { + Toast.error(e); + } + } + async function onAnonymise(download?: boolean) { try { setIsAnonymiseRunning(true); @@ -419,6 +438,25 @@ export const DataManagementTasks: React.FC = ({ setOptions={(o) => setCleanOptions(o)} /> + + + +
+ + + } + > + + @@ -519,7 +557,7 @@ export const DataManagementTasks: React.FC = ({ )} >