Skip to content

Commit

Permalink
refactor: use context in storage driver interface
Browse files Browse the repository at this point in the history
  • Loading branch information
@theSuess
theSuess committed Jun 28, 2022
1 parent 8de844a commit 1f9b270
Show file tree
Hide file tree
Showing 17 changed files with 104 additions and 85 deletions.
4 changes: 2 additions & 2 deletions internal/api/client/admin/emojicreate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() {
suite.Empty(dbEmoji.CategoryID)

// emoji should be in storage
emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath)
emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath)
suite.NoError(err)
suite.Len(emojiBytes, dbEmoji.ImageFileSize)
emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath)
emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath)
suite.NoError(err)
suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize)
}
Expand Down
4 changes: 2 additions & 2 deletions internal/api/client/fileserver/servefile_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
suite.NoError(err)
suite.NotNil(b)

fileInStorage, err := suite.storage.Get(targetAttachment.File.Path)
fileInStorage, err := suite.storage.Get(ctx, targetAttachment.File.Path)
suite.NoError(err)
suite.NotNil(fileInStorage)
suite.Equal(b, fileInStorage)
Expand Down Expand Up @@ -206,7 +206,7 @@ func (suite *ServeFileTestSuite) TestServeSmallFileSuccessful() {
suite.NoError(err)
suite.NotNil(b)

fileInStorage, err := suite.storage.Get(targetAttachment.Thumbnail.Path)
fileInStorage, err := suite.storage.Get(ctx, targetAttachment.Thumbnail.Path)
suite.NoError(err)
suite.NotNil(fileInStorage)
suite.Equal(b, fileInStorage)
Expand Down
4 changes: 2 additions & 2 deletions internal/api/client/media/mediacreate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {

// see what's in storage *before* the request
storageKeysBeforeRequest := []string{}
iter, err := suite.storage.Iterator(nil)
iter, err := suite.storage.KVStore.Iterator(nil)
if err != nil {
panic(err)
}
Expand All @@ -164,7 +164,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {

// check what's in storage *after* the request
storageKeysAfterRequest := []string{}
iter, err = suite.storage.Iterator(nil)
iter, err = suite.storage.KVStore.Iterator(nil)
if err != nil {
panic(err)
}
Expand Down
24 changes: 12 additions & 12 deletions internal/media/manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.NotNil(dbAttachment)

// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)

Expand All @@ -101,7 +101,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)

// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)

Expand Down Expand Up @@ -160,7 +160,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
suite.NotNil(dbAttachment)

// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)

Expand All @@ -173,7 +173,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)

// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)

Expand Down Expand Up @@ -232,7 +232,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
suite.NotNil(dbAttachment)

// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)

Expand All @@ -245,7 +245,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)

// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)

Expand Down Expand Up @@ -315,7 +315,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
suite.NotNil(dbAttachment)

// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)

Expand All @@ -328,7 +328,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
suite.Equal(processedFullBytesExpected, processedFullBytes)

// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)

Expand Down Expand Up @@ -394,7 +394,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
suite.NotNil(dbAttachment)

// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)

Expand All @@ -407,7 +407,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
suite.Equal(processedFullBytesExpected, processedFullBytes)

// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)

Expand Down Expand Up @@ -475,7 +475,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
suite.NotNil(dbAttachment)

// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)

Expand All @@ -488,7 +488,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
suite.Equal(processedFullBytesExpected, processedFullBytes)

// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)

Expand Down
6 changes: 3 additions & 3 deletions internal/media/processingemoji.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
switch processState(staticState) {
case received:
// stream the original file out of storage...
stored, err := p.storage.GetStream(p.emoji.ImagePath)
stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath)
if err != nil {
p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
Expand All @@ -135,7 +135,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
}

// put the static in storage
if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil {
if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil {
p.err = fmt.Errorf("loadStatic: error storing static: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
Expand Down Expand Up @@ -211,7 +211,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)

// store this for now -- other processes can pull it out of storage as they please
if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil {
if err := p.storage.PutStream(ctx, p.emoji.ImagePath, multiReader); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}

Expand Down
8 changes: 4 additions & 4 deletions internal/media/processingmedia.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {

// stream the original file out of storage
logrus.Tracef("loadThumb: fetching attachment from storage %s", p.attachment.URL)
stored, err := p.storage.GetStream(p.attachment.File.Path)
stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
Expand All @@ -164,7 +164,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {

// put the thumbnail in storage
logrus.Tracef("loadThumb: storing new thumbnail %s", p.attachment.URL)
if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil {
if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil {
p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
Expand Down Expand Up @@ -203,7 +203,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
var decoded *imageMeta

// stream the original file out of storage...
stored, err := p.storage.GetStream(p.attachment.File.Path)
stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.fullSizeState, int32(errored))
Expand Down Expand Up @@ -343,7 +343,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
p.attachment.File.FileSize = fileSize

// store this for now -- other processes can pull it out of storage as they please
if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
if err := p.storage.PutStream(ctx, p.attachment.File.Path, clean); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
p.attachment.Cached = true
Expand Down
4 changes: 2 additions & 2 deletions internal/media/prunemeta.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,15 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path)
if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
}

if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path)
if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
}
Expand Down
16 changes: 8 additions & 8 deletions internal/media/prunemeta_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,13 @@ func (suite *PruneMetaTestSuite) TestPruneMeta() {
suite.Equal(2, totalPruned)

// media should no longer be stored
_, err = suite.storage.Get(zorkOldAvatar.File.Path)
_, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
_, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldHeader.File.Path)
_, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
_, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)

// attachments should no longer be in the db
Expand Down Expand Up @@ -110,13 +110,13 @@ func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() {
suite.Equal(2, totalPruned)

// media should no longer be stored
_, err = suite.storage.Get(zorkOldAvatar.File.Path)
_, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
_, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldHeader.File.Path)
_, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
_, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)

// attachments should no longer be in the db
Expand Down
4 changes: 2 additions & 2 deletions internal/media/pruneremote.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path)
if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
Expand All @@ -82,7 +82,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path)
if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
Expand Down
10 changes: 5 additions & 5 deletions internal/media/pruneremote_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.Equal(2, totalPruned)

// media should no longer be stored
_, err = suite.storage.Get(testAttachment.File.Path)
_, err = suite.storage.Get(ctx, testAttachment.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(testAttachment.Thumbnail.Path)
_, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)

// now recache the image....
Expand Down Expand Up @@ -98,9 +98,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same

// recached files should be back in storage
_, err = suite.storage.Get(recachedAttachment.File.Path)
_, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
suite.NoError(err)
_, err = suite.storage.Get(recachedAttachment.Thumbnail.Path)
_, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
suite.NoError(err)
}

Expand All @@ -112,7 +112,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
suite.NoError(err)
suite.True(media.Cached)
err = suite.storage.Delete(media.File.Path)
err = suite.storage.Delete(ctx, media.File.Path)
suite.NoError(err)

// Now attempt to prune remote for item with db entry no file
Expand Down
4 changes: 2 additions & 2 deletions internal/processing/media/delete.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ func (p *processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr

// delete the thumbnail from storage
if attachment.Thumbnail.Path != "" {
if err := p.storage.Delete(attachment.Thumbnail.Path); err != nil {
if err := p.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil {
errs = append(errs, fmt.Sprintf("remove thumbnail at path %s: %s", attachment.Thumbnail.Path, err))
}
}

// delete the file from storage
if attachment.File.Path != "" {
if err := p.storage.Delete(attachment.File.Path); err != nil {
if err := p.storage.Delete(ctx, attachment.File.Path); err != nil {
errs = append(errs, fmt.Sprintf("remove file at path %s: %s", attachment.File.Path, err))
}
}
Expand Down
12 changes: 6 additions & 6 deletions internal/processing/media/getfile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount

// if we have the media cached on our server already, we can now simply return it from storage
if a.Cached {
return p.retrieveFromStorage(storagePath, attachmentContent)
return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}

// if we don't have it cached, then we can assume two things:
Expand Down Expand Up @@ -221,7 +221,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
}
// ... so now we can safely return it
return p.retrieveFromStorage(storagePath, attachmentContent)
return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}

return attachmentContent, nil
Expand Down Expand Up @@ -253,15 +253,15 @@ func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, e
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize))
}

return p.retrieveFromStorage(storagePath, emojiContent)
return p.retrieveFromStorage(ctx, storagePath, emojiContent)
}

func (p *processor) retrieveFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
if url := p.storage.URL(storagePath); url != nil {
func (p *processor) retrieveFromStorage(ctx context.Context, storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
if url := p.storage.URL(ctx, storagePath); url != nil {
content.URL = url
return content, nil
}
reader, err := p.storage.GetStream(storagePath)
reader, err := p.storage.GetStream(ctx, storagePath)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err))
}
Expand Down
Loading

0 comments on commit 1f9b270

Please sign in to comment.