Add partial import functionality (#812)

graphql/documents/mutations/metadata.graphql

@@ -10,6 +10,10 @@ mutation ExportObjects($input: ExportObjectsInput!) {
   exportObjects(input: $input)
 }
 
+mutation ImportObjects($input: ImportObjectsInput!) {
+  importObjects(input: $input)
+}
+
 mutation MetadataScan($input: ScanMetadataInput!) {
   metadataScan(input: $input)
 }

graphql/schema/schema.graphql

@@ -164,9 +164,12 @@ type Mutation {
   """Returns a link to download the result"""
   exportObjects(input: ExportObjectsInput!): String
 
-  """Start an import. Returns the job ID"""
+  """Performs an incremental import. Returns the job ID"""
+  importObjects(input: ImportObjectsInput!): String!
+
+  """Start an full import. Completely wipes the database and imports from the metadata directory. Returns the job ID"""
   metadataImport: String!
-  """Start an export. Returns the job ID"""
+  """Start a full export. Outputs to the metadata directory. Returns the job ID"""
   metadataExport: String!
   """Start a scan. Returns the job ID"""
   metadataScan(input: ScanMetadataInput!): String!

graphql/schema/types/metadata.graphql

@@ -1,3 +1,5 @@
+scalar Upload
+
 input GenerateMetadataInput {
   sprites: Boolean!
   previews: Boolean!
@@ -65,3 +67,21 @@ input ExportObjectsInput {
   galleries: ExportObjectTypeInput
   includeDependencies: Boolean
 }
+
+enum ImportDuplicateEnum {
+  IGNORE
+  OVERWRITE
+  FAIL
+}
+
+enum ImportMissingRefEnum {
+  IGNORE
+  FAIL
+  CREATE
+}
+
+input ImportObjectsInput {
+  file: Upload!
+  duplicateBehaviour: ImportDuplicateEnum!
+  missingRefBehaviour: ImportMissingRefEnum!
+}

pkg/api/resolver_mutation_metadata.go

@@ -19,6 +19,16 @@ func (r *mutationResolver) MetadataImport(ctx context.Context) (string, error) {
 	return "todo", nil
 }
 
+func (r *mutationResolver) ImportObjects(ctx context.Context, input models.ImportObjectsInput) (string, error) {
+	t := manager.CreateImportTask(config.GetVideoFileNamingAlgorithm(), input)
+	_, err := manager.GetInstance().RunSingleTask(t)
+	if err != nil {
+		return "", err
+	}
+
+	return "todo", nil
+}
+
 func (r *mutationResolver) MetadataExport(ctx context.Context) (string, error) {
 	manager.GetInstance().Export()
 	return "todo", nil

pkg/gallery/import.go

@@ -0,0 +1,72 @@
+package gallery
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/stashapp/stash/pkg/manager/jsonschema"
+	"github.com/stashapp/stash/pkg/models"
+)
+
+type Importer struct {
+	ReaderWriter models.GalleryReaderWriter
+	Input        jsonschema.PathMapping
+
+	gallery   models.Gallery
+	imageData []byte
+}
+
+func (i *Importer) PreImport() error {
+	currentTime := time.Now()
+	i.gallery = models.Gallery{
+		Checksum:  i.Input.Checksum,
+		Path:      i.Input.Path,
+		CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
+		UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
+	}
+
+	return nil
+}
+
+func (i *Importer) PostImport(id int) error {
+	return nil
+}
+
+func (i *Importer) Name() string {
+	return i.Input.Path
+}
+
+func (i *Importer) FindExistingID() (*int, error) {
+	existing, err := i.ReaderWriter.FindByPath(i.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	if existing != nil {
+		id := existing.ID
+		return &id, nil
+	}
+
+	return nil, nil
+}
+
+func (i *Importer) Create() (*int, error) {
+	created, err := i.ReaderWriter.Create(i.gallery)
+	if err != nil {
+		return nil, fmt.Errorf("error creating gallery: %s", err.Error())
+	}
+
+	id := created.ID
+	return &id, nil
+}
+
+func (i *Importer) Update(id int) error {
+	gallery := i.gallery
+	gallery.ID = id
+	_, err := i.ReaderWriter.Update(gallery)
+	if err != nil {
+		return fmt.Errorf("error updating existing gallery: %s", err.Error())
+	}
+
+	return nil
+}

pkg/gallery/import_test.go

@@ -0,0 +1,147 @@
+package gallery
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stashapp/stash/pkg/manager/jsonschema"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/mocks"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	galleryPath         = "galleryPath"
+	galleryPathErr      = "galleryPathErr"
+	existingGalleryPath = "existingGalleryPath"
+
+	galleryID         = 1
+	idErr             = 2
+	existingGalleryID = 100
+)
+
+func TestImporterName(t *testing.T) {
+	i := Importer{
+		Input: jsonschema.PathMapping{
+			Path: galleryPath,
+		},
+	}
+
+	assert.Equal(t, galleryPath, i.Name())
+}
+
+func TestImporterPreImport(t *testing.T) {
+	i := Importer{
+		Input: jsonschema.PathMapping{
+			Path: galleryPath,
+		},
+	}
+
+	err := i.PreImport()
+	assert.Nil(t, err)
+}
+
+func TestImporterFindExistingID(t *testing.T) {
+	readerWriter := &mocks.GalleryReaderWriter{}
+
+	i := Importer{
+		ReaderWriter: readerWriter,
+		Input: jsonschema.PathMapping{
+			Path: galleryPath,
+		},
+	}
+
+	errFindByPath := errors.New("FindByPath error")
+	readerWriter.On("FindByPath", galleryPath).Return(nil, nil).Once()
+	readerWriter.On("FindByPath", existingGalleryPath).Return(&models.Gallery{
+		ID: existingGalleryID,
+	}, nil).Once()
+	readerWriter.On("FindByPath", galleryPathErr).Return(nil, errFindByPath).Once()
+
+	id, err := i.FindExistingID()
+	assert.Nil(t, id)
+	assert.Nil(t, err)
+
+	i.Input.Path = existingGalleryPath
+	id, err = i.FindExistingID()
+	assert.Equal(t, existingGalleryID, *id)
+	assert.Nil(t, err)
+
+	i.Input.Path = galleryPathErr
+	id, err = i.FindExistingID()
+	assert.Nil(t, id)
+	assert.NotNil(t, err)
+
+	readerWriter.AssertExpectations(t)
+}
+
+func TestCreate(t *testing.T) {
+	readerWriter := &mocks.GalleryReaderWriter{}
+
+	gallery := models.Gallery{
+		Path: galleryPath,
+	}
+
+	galleryErr := models.Gallery{
+		Path: galleryPathErr,
+	}
+
+	i := Importer{
+		ReaderWriter: readerWriter,
+		gallery:      gallery,
+	}
+
+	errCreate := errors.New("Create error")
+	readerWriter.On("Create", gallery).Return(&models.Gallery{
+		ID: galleryID,
+	}, nil).Once()
+	readerWriter.On("Create", galleryErr).Return(nil, errCreate).Once()
+
+	id, err := i.Create()
+	assert.Equal(t, galleryID, *id)
+	assert.Nil(t, err)
+
+	i.gallery = galleryErr
+	id, err = i.Create()
+	assert.Nil(t, id)
+	assert.NotNil(t, err)
+
+	readerWriter.AssertExpectations(t)
+}
+
+func TestUpdate(t *testing.T) {
+	readerWriter := &mocks.GalleryReaderWriter{}
+
+	gallery := models.Gallery{
+		Path: galleryPath,
+	}
+
+	galleryErr := models.Gallery{
+		Path: galleryPathErr,
+	}
+
+	i := Importer{
+		ReaderWriter: readerWriter,
+		gallery:      gallery,
+	}
+
+	errUpdate := errors.New("Update error")
+
+	// id needs to be set for the mock input
+	gallery.ID = galleryID
+	readerWriter.On("Update", gallery).Return(nil, nil).Once()
+
+	err := i.Update(galleryID)
+	assert.Nil(t, err)
+
+	i.gallery = galleryErr
+
+	// need to set id separately
+	galleryErr.ID = idErr
+	readerWriter.On("Update", galleryErr).Return(nil, errUpdate).Once()
+
+	err = i.Update(idErr)
+	assert.NotNil(t, err)
+
+	readerWriter.AssertExpectations(t)
+}

pkg/manager/import.go

@@ -0,0 +1,61 @@
+package manager
+
+import (
+	"fmt"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+)
+
+type importer interface {
+	PreImport() error
+	PostImport(id int) error
+	Name() string
+	FindExistingID() (*int, error)
+	Create() (*int, error)
+	Update(id int) error
+}
+
+func performImport(i importer, duplicateBehaviour models.ImportDuplicateEnum) error {
+	if err := i.PreImport(); err != nil {
+		return err
+	}
+
+	// try to find an existing object with the same name
+	name := i.Name()
+	existing, err := i.FindExistingID()
+	if err != nil {
+		return fmt.Errorf("error finding existing objects: %s", err.Error())
+	}
+
+	var id int
+
+	if existing != nil {
+		if duplicateBehaviour == models.ImportDuplicateEnumFail {
+			return fmt.Errorf("existing object with name '%s'", name)
+		} else if duplicateBehaviour == models.ImportDuplicateEnumIgnore {
+			logger.Info("Skipping existing object")
+			return nil
+		}
+
+		// must be overwriting
+		id = *existing
+		if err := i.Update(id); err != nil {
+			return fmt.Errorf("error updating existing object: %s", err.Error())
+		}
+	} else {
+		// creating
+		createdID, err := i.Create()
+		if err != nil {
+			return fmt.Errorf("error creating object: %s", err.Error())
+		}
+
+		id = *createdID
+	}
+
+	if err := i.PostImport(id); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/manager/jsonschema/utils.go

@@ -2,10 +2,11 @@ package jsonschema
 
 import (
 	"bytes"
-	"github.com/json-iterator/go"
 
 	"io/ioutil"
 	"time"
+
+	jsoniter "github.com/json-iterator/go"
 )
 
 var nilTime = (time.Time{}).UnixNano()

pkg/manager/manager_tasks.go

@@ -153,7 +153,13 @@ func (s *singleton) Import() {
 
 		var wg sync.WaitGroup
 		wg.Add(1)
-		task := ImportTask{fileNamingAlgorithm: config.GetVideoFileNamingAlgorithm()}
+		task := ImportTask{
+			BaseDir:             config.GetMetadataPath(),
+			Reset:               true,
+			DuplicateBehaviour:  models.ImportDuplicateEnumFail,
+			MissingRefBehaviour: models.ImportMissingRefEnumFail,
+			fileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(),
+		}
 		go task.Start(&wg)
 		wg.Wait()
 	}()