Add oshash support

graphql/documents/data/config.graphql

@@ -3,6 +3,8 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   databasePath
   generatedPath
   cachePath
+  calculateMD5
+  videoFileNamingAlgorithm
   previewSegments
   previewSegmentDuration
   previewExcludeStart

graphql/documents/data/scene-slim.graphql

@@ -1,6 +1,7 @@
 fragment SlimSceneData on Scene {
   id
   checksum
+  oshash
   title
   details
   url

graphql/documents/data/scene.graphql

@@ -1,6 +1,7 @@
 fragment SceneData on Scene {
   id
   checksum
+  oshash
   title
   details
   url

graphql/documents/mutations/metadata.graphql

@@ -22,6 +22,10 @@ mutation MetadataClean {
   metadataClean
 }
 
+mutation MigrateHashNaming {
+  migrateHashNaming
+}
+
 mutation StopJob {
   stopJob
 }

graphql/schema/schema.graphql

@@ -2,6 +2,8 @@
 type Query {
   """Find a scene by ID or Checksum"""
   findScene(id: ID, checksum: String): Scene
+  findSceneByHash(input: SceneHashInput!): Scene
+
   """A function which queries Scene objects"""
   findScenes(scene_filter: SceneFilterType, scene_ids: [Int!], filter: FindFilterType): FindScenesResultType!
 
@@ -158,6 +160,8 @@ type Mutation {
   metadataAutoTag(input: AutoTagMetadataInput!): String!
   """Clean metadata. Returns the job ID"""
   metadataClean: String!
+  """Migrate generated files for the current hash naming"""
+  migrateHashNaming: String!
 
   """Reload scrapers"""
   reloadScrapers: Boolean!

graphql/schema/types/config.graphql

@@ -17,6 +17,11 @@ enum PreviewPreset {
   "X264_VERYSLOW", veryslow
 }
 
+enum HashAlgorithm {
+  MD5
+  "oshash", OSHASH
+}
+
 input ConfigGeneralInput {
   """Array of file paths to content"""
   stashes: [String!]
@@ -26,6 +31,10 @@ input ConfigGeneralInput {
   generatedPath: String
   """Path to cache"""
   cachePath: String
+  """Whether to calculate MD5 checksums for scene video files"""
+  calculateMD5: Boolean!
+  """Hash algorithm to use for generated file naming"""
+  videoFileNamingAlgorithm: HashAlgorithm!
   """Number of segments in a preview file"""
   previewSegments: Int
   """Preview segment duration, in seconds"""
@@ -71,6 +80,10 @@ type ConfigGeneralResult {
   generatedPath: String!
   """Path to cache"""
   cachePath: String!
+  """Whether to calculate MD5 checksums for scene video files"""
+  calculateMD5: Boolean!
+  """Hash algorithm to use for generated file naming"""
+  videoFileNamingAlgorithm: HashAlgorithm!
   """Number of segments in a preview file"""
   previewSegments: Int!
   """Preview segment duration, in seconds"""

graphql/schema/types/scene.graphql

@@ -25,7 +25,8 @@ type SceneMovie {
 
 type Scene {
   id: ID!
-  checksum: String!
+  checksum: String
+  oshash: String
   title: String
   details: String
   url: String
@@ -139,6 +140,11 @@ type SceneParserResultType {
   results: [SceneParserResult!]!
 }
 
+input SceneHashInput {
+  checksum: String
+  oshash: String
+}
+
 type SceneStreamEndpoint {
   url: String!
   mime_type: String

main.go

@@ -13,7 +13,12 @@ import (
 
 func main() {
 	manager.Initialize()
-	database.Initialize(config.GetDatabasePath())
+
+	// perform the post-migration for new databases
+	if database.Initialize(config.GetDatabasePath()) {
+		manager.GetInstance().PostMigrate()
+	}
+
 	api.Start()
 	blockForever()
 }

pkg/api/migrate.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/stashapp/stash/pkg/database"
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/manager"
 )
 
 type migrateData struct {
@@ -80,6 +81,9 @@ func doMigrateHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// perform post-migration operations
+	manager.GetInstance().PostMigrate()
+
 	// if no backup path was provided, then delete the created backup
 	if formBackupPath == "" {
 		err = os.Remove(backupPath)

pkg/api/resolver_model_scene.go

@@ -8,6 +8,20 @@ import (
 	"github.com/stashapp/stash/pkg/utils"
 )
 
+func (r *sceneResolver) Checksum(ctx context.Context, obj *models.Scene) (*string, error) {
+	if obj.Checksum.Valid {
+		return &obj.Checksum.String, nil
+	}
+	return nil, nil
+}
+
+func (r *sceneResolver) Oshash(ctx context.Context, obj *models.Scene) (*string, error) {
+	if obj.OSHash.Valid {
+		return &obj.OSHash.String, nil
+	}
+	return nil, nil
+}
+
 func (r *sceneResolver) Title(ctx context.Context, obj *models.Scene) (*string, error) {
 	if obj.Title.Valid {
 		return &obj.Title.String, nil

pkg/api/resolver_mutation_configure.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"path/filepath"
 
@@ -45,6 +46,21 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		config.Set(config.Cache, input.CachePath)
 	}
 
+	if !input.CalculateMd5 && input.VideoFileNamingAlgorithm == models.HashAlgorithmMd5 {
+		return makeConfigGeneralResult(), errors.New("calculateMD5 must be true if using MD5")
+	}
+
+	if input.VideoFileNamingAlgorithm != config.GetVideoFileNamingAlgorithm() {
+		// validate changing VideoFileNamingAlgorithm
+		if err := manager.ValidateVideoFileNamingAlgorithm(input.VideoFileNamingAlgorithm); err != nil {
+			return makeConfigGeneralResult(), err
+		}
+
+		config.Set(config.VideoFileNamingAlgorithm, input.VideoFileNamingAlgorithm)
+	}
+
+	config.Set(config.CalculateMD5, input.CalculateMd5)
+
 	if input.PreviewSegments != nil {
 		config.Set(config.PreviewSegments, *input.PreviewSegments)
 	}

pkg/api/resolver_mutation_metadata.go

@@ -37,6 +37,11 @@ func (r *mutationResolver) MetadataClean(ctx context.Context) (string, error) {
 	return "todo", nil
 }
 
+func (r *mutationResolver) MigrateHashNaming(ctx context.Context) (string, error) {
+	manager.GetInstance().MigrateHash()
+	return "todo", nil
+}
+
 func (r *mutationResolver) JobStatus(ctx context.Context) (*models.MetadataUpdateStatus, error) {
 	status := manager.GetInstance().Status
 	ret := models.MetadataUpdateStatus{

pkg/api/resolver_mutation_scene.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stashapp/stash/pkg/database"
 	"github.com/stashapp/stash/pkg/manager"
+	"github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/utils"
 )
@@ -197,7 +198,7 @@ func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, tx *sqlx.T
 
 	// only update the cover image if provided and everything else was successful
 	if coverImageData != nil {
-		err = manager.SetSceneScreenshot(scene.Checksum, coverImageData)
+		err = manager.SetSceneScreenshot(scene.GetHash(config.GetVideoFileNamingAlgorithm()), coverImageData)
 		if err != nil {
 			return nil, err
 		}
@@ -417,7 +418,7 @@ func (r *mutationResolver) SceneDestroy(ctx context.Context, input models.SceneD
 	// if delete generated is true, then delete the generated files
 	// for the scene
 	if input.DeleteGenerated != nil && *input.DeleteGenerated {
-		manager.DeleteGeneratedSceneFiles(scene)
+		manager.DeleteGeneratedSceneFiles(scene, config.GetVideoFileNamingAlgorithm())
 	}
 
 	// if delete file is true, then delete the file as well
@@ -453,11 +454,12 @@ func (r *mutationResolver) ScenesDestroy(ctx context.Context, input models.Scene
 		return false, err
 	}
 
+	fileNamingAlgo := config.GetVideoFileNamingAlgorithm()
 	for _, scene := range scenes {
 		// if delete generated is true, then delete the generated files
 		// for the scene
 		if input.DeleteGenerated != nil && *input.DeleteGenerated {
-			manager.DeleteGeneratedSceneFiles(scene)
+			manager.DeleteGeneratedSceneFiles(scene, fileNamingAlgo)
 		}
 
 		// if delete file is true, then delete the file as well
@@ -528,7 +530,7 @@ func (r *mutationResolver) SceneMarkerDestroy(ctx context.Context, id string) (b
 
 	if scene != nil {
 		seconds := int(marker.Seconds)
-		manager.DeleteSceneMarkerFiles(scene, seconds)
+		manager.DeleteSceneMarkerFiles(scene, seconds, config.GetVideoFileNamingAlgorithm())
 	}
 
 	return true, nil
@@ -597,7 +599,7 @@ func changeMarker(ctx context.Context, changeType int, changedMarker models.Scen
 
 		if scene != nil {
 			seconds := int(existingMarker.Seconds)
-			manager.DeleteSceneMarkerFiles(scene, seconds)
+			manager.DeleteSceneMarkerFiles(scene, seconds, config.GetVideoFileNamingAlgorithm())
 		}
 	}
 

pkg/api/resolver_query_configuration.go

@@ -47,6 +47,8 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		DatabasePath:              config.GetDatabasePath(),
 		GeneratedPath:             config.GetGeneratedPath(),
 		CachePath:                 config.GetCachePath(),
+		CalculateMd5:              config.IsCalculateMD5(),
+		VideoFileNamingAlgorithm:  config.GetVideoFileNamingAlgorithm(),
 		PreviewSegments:           config.GetPreviewSegments(),
 		PreviewSegmentDuration:    config.GetPreviewSegmentDuration(),
 		PreviewExcludeStart:       config.GetPreviewExcludeStart(),

pkg/api/resolver_query_find_scene.go

@@ -21,6 +21,28 @@ func (r *queryResolver) FindScene(ctx context.Context, id *string, checksum *str
 	return scene, err
 }
 
+func (r *queryResolver) FindSceneByHash(ctx context.Context, input models.SceneHashInput) (*models.Scene, error) {
+	qb := models.NewSceneQueryBuilder()
+	var scene *models.Scene
+	var err error
+
+	if input.Checksum != nil {
+		scene, err = qb.FindByChecksum(*input.Checksum)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if scene == nil && input.Oshash != nil {
+		scene, err = qb.FindByOSHash(*input.Oshash)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return scene, err
+}
+
 func (r *queryResolver) FindScenes(ctx context.Context, sceneFilter *models.SceneFilterType, sceneIds []int, filter *models.FindFilterType) (*models.FindScenesResultType, error) {
 	qb := models.NewSceneQueryBuilder()
 	scenes, total := qb.Query(sceneFilter, filter)

pkg/api/routes_scene.go

@@ -67,8 +67,9 @@ func getSceneFileContainer(scene *models.Scene) ffmpeg.Container {
 
 func (rs sceneRoutes) StreamDirect(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
+	fileNamingAlgo := config.GetVideoFileNamingAlgorithm()
 
-	filepath := manager.GetInstance().Paths.Scene.GetStreamPath(scene.Path, scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetStreamPath(scene.Path, scene.GetHash(fileNamingAlgo))
 	manager.RegisterStream(filepath, &w)
 	http.ServeFile(w, r, filepath)
 	manager.WaitAndDeregisterStream(filepath, &w, r)
@@ -171,7 +172,7 @@ func (rs sceneRoutes) streamTranscode(w http.ResponseWriter, r *http.Request, vi
 
 func (rs sceneRoutes) Screenshot(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
-	filepath := manager.GetInstance().Paths.Scene.GetScreenshotPath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetScreenshotPath(scene.GetHash(config.GetVideoFileNamingAlgorithm()))
 
 	// fall back to the scene image blob if the file isn't present
 	screenshotExists, _ := utils.FileExists(filepath)
@@ -186,13 +187,13 @@ func (rs sceneRoutes) Screenshot(w http.ResponseWriter, r *http.Request) {
 
 func (rs sceneRoutes) Preview(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
-	filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewPath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewPath(scene.GetHash(config.GetVideoFileNamingAlgorithm()))
 	utils.ServeFileNoCache(w, r, filepath)
 }
 
 func (rs sceneRoutes) Webp(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
-	filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewImagePath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewImagePath(scene.GetHash(config.GetVideoFileNamingAlgorithm()))
 	http.ServeFile(w, r, filepath)
 }
 
@@ -248,14 +249,14 @@ func (rs sceneRoutes) ChapterVtt(w http.ResponseWriter, r *http.Request) {
 func (rs sceneRoutes) VttThumbs(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
 	w.Header().Set("Content-Type", "text/vtt")
-	filepath := manager.GetInstance().Paths.Scene.GetSpriteVttFilePath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetSpriteVttFilePath(scene.GetHash(config.GetVideoFileNamingAlgorithm()))
 	http.ServeFile(w, r, filepath)
 }
 
 func (rs sceneRoutes) VttSprite(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
 	w.Header().Set("Content-Type", "image/jpeg")
-	filepath := manager.GetInstance().Paths.Scene.GetSpriteImageFilePath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetSpriteImageFilePath(scene.GetHash(config.GetVideoFileNamingAlgorithm()))
 	http.ServeFile(w, r, filepath)
 }
 
@@ -269,7 +270,7 @@ func (rs sceneRoutes) SceneMarkerStream(w http.ResponseWriter, r *http.Request)
 		http.Error(w, http.StatusText(404), 404)
 		return
 	}
-	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPath(scene.Checksum, int(sceneMarker.Seconds))
+	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPath(scene.GetHash(config.GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds))
 	http.ServeFile(w, r, filepath)
 }
 
@@ -283,7 +284,7 @@ func (rs sceneRoutes) SceneMarkerPreview(w http.ResponseWriter, r *http.Request)
 		http.Error(w, http.StatusText(404), 404)
 		return
 	}
-	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.Checksum, int(sceneMarker.Seconds))
+	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.GetHash(config.GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds))
 
 	// If the image doesn't exist, send the placeholder
 	exists, _ := utils.FileExists(filepath)