Add oshash support

graphql/documents/data/config.graphql

@@ -3,6 +3,8 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   databasePath
   generatedPath
   cachePath
+  calculateMD5
+  useMD5
   maxTranscodeSize
   maxStreamingTranscodeSize
   forceMkv

graphql/documents/data/scene-slim.graphql

@@ -1,6 +1,7 @@
 fragment SlimSceneData on Scene {
   id
   checksum
+  oshash
   title
   details
   url

graphql/documents/data/scene.graphql

@@ -1,6 +1,7 @@
 fragment SceneData on Scene {
   id
   checksum
+  oshash
   title
   details
   url

graphql/documents/mutations/metadata.graphql

@@ -22,6 +22,10 @@ mutation MetadataClean {
   metadataClean
 }
 
+mutation MigrateHashNaming {
+  migrateHashNaming
+}
+
 mutation StopJob {
   stopJob
 }

graphql/schema/schema.graphql

@@ -2,6 +2,8 @@
 type Query {
   """Find a scene by ID or Checksum"""
   findScene(id: ID, checksum: String): Scene
+  findSceneByHash(input: SceneHashInput!): Scene
+
   """A function which queries Scene objects"""
   findScenes(scene_filter: SceneFilterType, scene_ids: [Int!], filter: FindFilterType): FindScenesResultType!
 
@@ -155,6 +157,8 @@ type Mutation {
   metadataAutoTag(input: AutoTagMetadataInput!): String!
   """Clean metadata. Returns the job ID"""
   metadataClean: String!
+  """Migrate generated files for the current hash naming"""
+  migrateHashNaming: String!
 
   """Reload scrapers"""
   reloadScrapers: Boolean!

graphql/schema/types/config.graphql

@@ -16,6 +16,10 @@ input ConfigGeneralInput {
   generatedPath: String
   """Path to cache"""
   cachePath: String
+  """Whether to calculate MD5 checksums for scene video files"""
+  calculateMD5: Boolean!
+  """Whether to use MD5 checksums for generated file naming"""
+  useMD5: Boolean!
   """Max generated transcode size"""
   maxTranscodeSize: StreamingResolutionEnum
   """Max streaming transcode size"""
@@ -53,6 +57,10 @@ type ConfigGeneralResult {
   generatedPath: String!
   """Path to cache"""
   cachePath: String!
+  """Whether to calculate MD5 checksums for scene video files"""
+  calculateMD5: Boolean!
+  """Whether to use MD5 checksums for generated file naming"""
+  useMD5: Boolean!
   """Max generated transcode size"""
   maxTranscodeSize: StreamingResolutionEnum
   """Max streaming transcode size"""

graphql/schema/types/scene.graphql

@@ -25,7 +25,8 @@ type SceneMovie {
 
 type Scene {
   id: ID!
-  checksum: String!
+  checksum: String
+  oshash: String
   title: String
   details: String
   url: String
@@ -138,4 +139,9 @@ type SceneParserResult {
 type SceneParserResultType {
   count: Int!
   results: [SceneParserResult!]!
-}
+}
+
+input SceneHashInput {
+  checksum: String
+  oshash: String
+}

main.go

@@ -13,7 +13,12 @@ import (
 
 func main() {
 	manager.Initialize()
-	database.Initialize(config.GetDatabasePath())
+
+	// perform the post-migration for new databases
+	if database.Initialize(config.GetDatabasePath()) {
+		manager.GetInstance().PostMigrate()
+	}
+
 	api.Start()
 	blockForever()
 }

pkg/api/migrate.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/stashapp/stash/pkg/database"
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/manager"
 )
 
 type migrateData struct {
@@ -80,6 +81,9 @@ func doMigrateHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// perform post-migration operations
+	manager.GetInstance().PostMigrate()
+
 	// if no backup path was provided, then delete the created backup
 	if formBackupPath == "" {
 		err = os.Remove(backupPath)

pkg/api/resolver_model_scene.go

@@ -5,10 +5,25 @@ import (
 
 	"github.com/stashapp/stash/pkg/api/urlbuilders"
 	"github.com/stashapp/stash/pkg/manager"
+	"github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
+func (r *sceneResolver) Checksum(ctx context.Context, obj *models.Scene) (*string, error) {
+	if obj.Checksum.Valid {
+		return &obj.Checksum.String, nil
+	}
+	return nil, nil
+}
+
+func (r *sceneResolver) Oshash(ctx context.Context, obj *models.Scene) (*string, error) {
+	if obj.OSHash.Valid {
+		return &obj.OSHash.String, nil
+	}
+	return nil, nil
+}
+
 func (r *sceneResolver) Title(ctx context.Context, obj *models.Scene) (*string, error) {
 	if obj.Title.Valid {
 		return &obj.Title.String, nil
@@ -83,7 +98,7 @@ func (r *sceneResolver) Paths(ctx context.Context, obj *models.Scene) (*models.S
 
 func (r *sceneResolver) IsStreamable(ctx context.Context, obj *models.Scene) (bool, error) {
 	// ignore error
-	ret, _ := manager.IsStreamable(obj)
+	ret, _ := manager.IsStreamable(obj, config.IsUseMD5())
 	return ret, nil
 }
 

pkg/api/resolver_mutation_configure.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"path/filepath"
 
@@ -45,6 +46,21 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		config.Set(config.Cache, input.CachePath)
 	}
 
+	if !input.CalculateMd5 && input.UseMd5 {
+		return makeConfigGeneralResult(), errors.New("calculateMD5 must be true if using MD5")
+	}
+
+	if input.UseMd5 != config.IsUseMD5() {
+		// validate changing UseMD5
+		if err := manager.ValidateUseMD5(input.UseMd5); err != nil {
+			return makeConfigGeneralResult(), err
+		}
+
+		config.Set(config.UseMD5, input.UseMd5)
+	}
+
+	config.Set(config.CalculateMD5, input.CalculateMd5)
+
 	if input.MaxTranscodeSize != nil {
 		config.Set(config.MaxTranscodeSize, input.MaxTranscodeSize.String())
 	}

pkg/api/resolver_mutation_metadata.go

@@ -37,6 +37,11 @@ func (r *mutationResolver) MetadataClean(ctx context.Context) (string, error) {
 	return "todo", nil
 }
 
+func (r *mutationResolver) MigrateHashNaming(ctx context.Context) (string, error) {
+	manager.GetInstance().MigrateHash()
+	return "todo", nil
+}
+
 func (r *mutationResolver) JobStatus(ctx context.Context) (*models.MetadataUpdateStatus, error) {
 	status := manager.GetInstance().Status
 	ret := models.MetadataUpdateStatus{

pkg/api/resolver_mutation_scene.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stashapp/stash/pkg/database"
 	"github.com/stashapp/stash/pkg/manager"
+	"github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/utils"
 )
@@ -197,7 +198,7 @@ func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, tx *sqlx.T
 
 	// only update the cover image if provided and everything else was successful
 	if coverImageData != nil {
-		err = manager.SetSceneScreenshot(scene.Checksum, coverImageData)
+		err = manager.SetSceneScreenshot(scene.GetHash(config.IsUseMD5()), coverImageData)
 		if err != nil {
 			return nil, err
 		}
@@ -417,7 +418,7 @@ func (r *mutationResolver) SceneDestroy(ctx context.Context, input models.SceneD
 	// if delete generated is true, then delete the generated files
 	// for the scene
 	if input.DeleteGenerated != nil && *input.DeleteGenerated {
-		manager.DeleteGeneratedSceneFiles(scene)
+		manager.DeleteGeneratedSceneFiles(scene, config.IsUseMD5())
 	}
 
 	// if delete file is true, then delete the file as well
@@ -453,11 +454,12 @@ func (r *mutationResolver) ScenesDestroy(ctx context.Context, input models.Scene
 		return false, err
 	}
 
+	useMD5 := config.IsUseMD5()
 	for _, scene := range scenes {
 		// if delete generated is true, then delete the generated files
 		// for the scene
 		if input.DeleteGenerated != nil && *input.DeleteGenerated {
-			manager.DeleteGeneratedSceneFiles(scene)
+			manager.DeleteGeneratedSceneFiles(scene, useMD5)
 		}
 
 		// if delete file is true, then delete the file as well
@@ -528,7 +530,7 @@ func (r *mutationResolver) SceneMarkerDestroy(ctx context.Context, id string) (b
 
 	if scene != nil {
 		seconds := int(marker.Seconds)
-		manager.DeleteSceneMarkerFiles(scene, seconds)
+		manager.DeleteSceneMarkerFiles(scene, seconds, config.IsUseMD5())
 	}
 
 	return true, nil
@@ -597,7 +599,7 @@ func changeMarker(ctx context.Context, changeType int, changedMarker models.Scen
 
 		if scene != nil {
 			seconds := int(existingMarker.Seconds)
-			manager.DeleteSceneMarkerFiles(scene, seconds)
+			manager.DeleteSceneMarkerFiles(scene, seconds, config.IsUseMD5())
 		}
 	}
 

pkg/api/resolver_query_configuration.go

@@ -46,6 +46,8 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		DatabasePath:              config.GetDatabasePath(),
 		GeneratedPath:             config.GetGeneratedPath(),
 		CachePath:                 config.GetCachePath(),
+		CalculateMd5:              config.IsCalculateMD5(),
+		UseMd5:                    config.IsUseMD5(),
 		MaxTranscodeSize:          &maxTranscodeSize,
 		MaxStreamingTranscodeSize: &maxStreamingTranscodeSize,
 		ForceMkv:                  config.GetForceMKV(),

pkg/api/resolver_query_find_scene.go

@@ -21,6 +21,28 @@ func (r *queryResolver) FindScene(ctx context.Context, id *string, checksum *str
 	return scene, err
 }
 
+func (r *queryResolver) FindSceneByHash(ctx context.Context, input models.SceneHashInput) (*models.Scene, error) {
+	qb := models.NewSceneQueryBuilder()
+	var scene *models.Scene
+	var err error
+
+	if input.Checksum != nil {
+		scene, err = qb.FindByChecksum(*input.Checksum)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if scene == nil && input.Oshash != nil {
+		scene, err = qb.FindByOSHash(*input.Oshash)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return scene, err
+}
+
 func (r *queryResolver) FindScenes(ctx context.Context, sceneFilter *models.SceneFilterType, sceneIds []int, filter *models.FindFilterType) (*models.FindScenesResultType, error) {
 	qb := models.NewSceneQueryBuilder()
 	scenes, total := qb.Query(sceneFilter, filter)

pkg/api/routes_scene.go

@@ -61,13 +61,14 @@ func (rs sceneRoutes) Stream(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// detect if not a streamable file and try to transcode it instead
-	filepath := manager.GetInstance().Paths.Scene.GetStreamPath(scene.Path, scene.Checksum)
+	useMD5 := config.IsUseMD5()
+	filepath := manager.GetInstance().Paths.Scene.GetStreamPath(scene.Path, scene.GetHash(useMD5))
 	videoCodec := scene.VideoCodec.String
 	audioCodec := ffmpeg.MissingUnsupported
 	if scene.AudioCodec.Valid {
 		audioCodec = ffmpeg.AudioCodec(scene.AudioCodec.String)
 	}
-	hasTranscode, _ := manager.HasTranscode(scene)
+	hasTranscode := manager.HasTranscode(scene, useMD5)
 	if ffmpeg.IsValidCodec(videoCodec) && ffmpeg.IsValidCombo(videoCodec, ffmpeg.Container(container)) && ffmpeg.IsValidAudioForContainer(audioCodec, ffmpeg.Container(container)) || hasTranscode {
 		manager.RegisterStream(filepath, &w)
 		http.ServeFile(w, r, filepath)
@@ -150,7 +151,7 @@ func (rs sceneRoutes) Stream(w http.ResponseWriter, r *http.Request) {
 
 func (rs sceneRoutes) Screenshot(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
-	filepath := manager.GetInstance().Paths.Scene.GetScreenshotPath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetScreenshotPath(scene.GetHash(config.IsUseMD5()))
 
 	// fall back to the scene image blob if the file isn't present
 	screenshotExists, _ := utils.FileExists(filepath)
@@ -165,13 +166,13 @@ func (rs sceneRoutes) Screenshot(w http.ResponseWriter, r *http.Request) {
 
 func (rs sceneRoutes) Preview(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
-	filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewPath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewPath(scene.GetHash(config.IsUseMD5()))
 	http.ServeFile(w, r, filepath)
 }
 
 func (rs sceneRoutes) Webp(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
-	filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewImagePath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewImagePath(scene.GetHash(config.IsUseMD5()))
 	http.ServeFile(w, r, filepath)
 }
 
@@ -227,14 +228,14 @@ func (rs sceneRoutes) ChapterVtt(w http.ResponseWriter, r *http.Request) {
 func (rs sceneRoutes) VttThumbs(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
 	w.Header().Set("Content-Type", "text/vtt")
-	filepath := manager.GetInstance().Paths.Scene.GetSpriteVttFilePath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetSpriteVttFilePath(scene.GetHash(config.IsUseMD5()))
 	http.ServeFile(w, r, filepath)
 }
 
 func (rs sceneRoutes) VttSprite(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
 	w.Header().Set("Content-Type", "image/jpeg")
-	filepath := manager.GetInstance().Paths.Scene.GetSpriteImageFilePath(scene.Checksum)
+	filepath := manager.GetInstance().Paths.Scene.GetSpriteImageFilePath(scene.GetHash(config.IsUseMD5()))
 	http.ServeFile(w, r, filepath)
 }
 
@@ -248,7 +249,7 @@ func (rs sceneRoutes) SceneMarkerStream(w http.ResponseWriter, r *http.Request)
 		http.Error(w, http.StatusText(404), 404)
 		return
 	}
-	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPath(scene.Checksum, int(sceneMarker.Seconds))
+	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPath(scene.GetHash(config.IsUseMD5()), int(sceneMarker.Seconds))
 	http.ServeFile(w, r, filepath)
 }
 
@@ -262,7 +263,7 @@ func (rs sceneRoutes) SceneMarkerPreview(w http.ResponseWriter, r *http.Request)
 		http.Error(w, http.StatusText(404), 404)
 		return
 	}
-	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.Checksum, int(sceneMarker.Seconds))
+	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.GetHash(config.IsUseMD5()), int(sceneMarker.Seconds))
 
 	// If the image doesn't exist, send the placeholder
 	exists, _ := utils.FileExists(filepath)