Export performance optimization

go.mod

@@ -14,10 +14,9 @@ require (
 	github.com/gorilla/sessions v1.2.0
 	github.com/gorilla/websocket v1.4.0
 	github.com/h2non/filetype v1.0.8
-	// this is required for generate
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/json-iterator/go v1.1.9
 	github.com/mattn/go-sqlite3 v1.13.0
 	github.com/rs/cors v1.6.0
 	github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f

go.sum

@@ -343,6 +343,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
@@ -399,6 +400,8 @@ github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqx
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
@@ -463,6 +466,10 @@ github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:F
 github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU=
 github.com/monoculum/formam v0.0.0-20180901015400-4e68be1d79ba/go.mod h1:RKgILGEJq24YyJ2ban8EO0RUVSJlF1pGsEvoLEACr/Q=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

pkg/manager/jsonschema/mappings.go

@@ -1,8 +1,8 @@
 package jsonschema
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/json-iterator/go"
 	"os"
 )
 
@@ -31,6 +31,7 @@ func LoadMappingsFile(filePath string) (*Mappings, error) {
 	if err != nil {
 		return nil, err
 	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	jsonParser := json.NewDecoder(file)
 	err = jsonParser.Decode(&mappings)
 	if err != nil {

pkg/manager/jsonschema/movie.go

@@ -1,8 +1,8 @@
 package jsonschema
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/json-iterator/go"
 	"os"
 
 	"github.com/stashapp/stash/pkg/models"
@@ -30,6 +30,7 @@ func LoadMovieFile(filePath string) (*Movie, error) {
 	if err != nil {
 		return nil, err
 	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	jsonParser := json.NewDecoder(file)
 	err = jsonParser.Decode(&movie)
 	if err != nil {

pkg/manager/jsonschema/performer.go

@@ -1,8 +1,8 @@
 package jsonschema
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/json-iterator/go"
 	"os"
 
 	"github.com/stashapp/stash/pkg/models"
@@ -38,6 +38,7 @@ func LoadPerformerFile(filePath string) (*Performer, error) {
 	if err != nil {
 		return nil, err
 	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	jsonParser := json.NewDecoder(file)
 	err = jsonParser.Decode(&performer)
 	if err != nil {

pkg/manager/jsonschema/scene.go

@@ -1,8 +1,8 @@
 package jsonschema
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/json-iterator/go"
 	"os"
 
 	"github.com/stashapp/stash/pkg/models"
@@ -59,6 +59,7 @@ func LoadSceneFile(filePath string) (*Scene, error) {
 	if err != nil {
 		return nil, err
 	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	jsonParser := json.NewDecoder(file)
 	err = jsonParser.Decode(&scene)
 	if err != nil {

pkg/manager/jsonschema/scraped.go

@@ -1,8 +1,8 @@
 package jsonschema
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/json-iterator/go"
 	"github.com/stashapp/stash/pkg/models"
 	"os"
 )
@@ -31,6 +31,7 @@ func LoadScrapedFile(filePath string) ([]ScrapedItem, error) {
 	if err != nil {
 		return nil, err
 	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	jsonParser := json.NewDecoder(file)
 	err = jsonParser.Decode(&scraped)
 	if err != nil {

pkg/manager/jsonschema/studio.go

@@ -1,8 +1,8 @@
 package jsonschema
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/json-iterator/go"
 	"github.com/stashapp/stash/pkg/models"
 	"os"
 )
@@ -22,6 +22,7 @@ func LoadStudioFile(filePath string) (*Studio, error) {
 	if err != nil {
 		return nil, err
 	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	jsonParser := json.NewDecoder(file)
 	err = jsonParser.Decode(&studio)
 	if err != nil {

pkg/manager/jsonschema/utils.go

@@ -2,7 +2,8 @@ package jsonschema
 
 import (
 	"bytes"
-	"encoding/json"
+	"github.com/json-iterator/go"
+
 	"io/ioutil"
 	"time"
 )
@@ -25,6 +26,7 @@ func marshalToFile(filePath string, j interface{}) error {
 
 func encode(j interface{}) ([]byte, error) {
 	buffer := &bytes.Buffer{}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	encoder := json.NewEncoder(buffer)
 	encoder.SetEscapeHTML(false)
 	encoder.SetIndent("", "  ")

pkg/manager/manager.go

@@ -154,10 +154,6 @@ func (s *singleton) RefreshConfig() {
 		_ = utils.EnsureDir(s.Paths.Generated.Markers)
 		_ = utils.EnsureDir(s.Paths.Generated.Transcodes)
 
-		_ = utils.EnsureDir(s.Paths.JSON.Performers)
-		_ = utils.EnsureDir(s.Paths.JSON.Scenes)
-		_ = utils.EnsureDir(s.Paths.JSON.Galleries)
-		_ = utils.EnsureDir(s.Paths.JSON.Studios)
-		_ = utils.EnsureDir(s.Paths.JSON.Movies)
+		paths.EnsureJSONDirs()
 	}
 }

pkg/manager/paths/paths_json.go

@@ -2,10 +2,13 @@ package paths
 
 import (
 	"github.com/stashapp/stash/pkg/manager/config"
+	"github.com/stashapp/stash/pkg/utils"
 	"path/filepath"
 )
 
 type jsonPaths struct {
+	Metadata string
+
 	MappingsFile string
 	ScrapedFile  string
 
@@ -18,6 +21,7 @@ type jsonPaths struct {
 
 func newJSONPaths() *jsonPaths {
 	jp := jsonPaths{}
+	jp.Metadata = config.GetMetadataPath()
 	jp.MappingsFile = filepath.Join(config.GetMetadataPath(), "mappings.json")
 	jp.ScrapedFile = filepath.Join(config.GetMetadataPath(), "scraped.json")
 	jp.Performers = filepath.Join(config.GetMetadataPath(), "performers")
@@ -28,6 +32,21 @@ func newJSONPaths() *jsonPaths {
 	return &jp
 }
 
+func GetJSONPaths() *jsonPaths {
+	jp := newJSONPaths()
+	return jp
+}
+
+func EnsureJSONDirs() {
+	jsonPaths := GetJSONPaths()
+	utils.EnsureDir(jsonPaths.Metadata)
+	utils.EnsureDir(jsonPaths.Scenes)
+	utils.EnsureDir(jsonPaths.Galleries)
+	utils.EnsureDir(jsonPaths.Performers)
+	utils.EnsureDir(jsonPaths.Studios)
+	utils.EnsureDir(jsonPaths.Movies)
+}
+
 func (jp *jsonPaths) PerformerJSONPath(checksum string) string {
 	return filepath.Join(jp.Performers, checksum+".json")
 }