diff --git a/internal/api/client/admin/emojicreate_test.go b/internal/api/client/admin/emojicreate_test.go index 872b18a752..fa93ce564c 100644 --- a/internal/api/client/admin/emojicreate_test.go +++ b/internal/api/client/admin/emojicreate_test.go @@ -103,10 +103,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() { suite.Empty(dbEmoji.CategoryID) // emoji should be in storage - emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath) + emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath) suite.NoError(err) suite.Len(emojiBytes, dbEmoji.ImageFileSize) - emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath) + emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath) suite.NoError(err) suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize) } diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index c9edaca53b..a36a79a585 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -160,7 +160,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() { suite.NoError(err) suite.NotNil(b) - fileInStorage, err := suite.storage.Get(targetAttachment.File.Path) + fileInStorage, err := suite.storage.Get(ctx, targetAttachment.File.Path) suite.NoError(err) suite.NotNil(fileInStorage) suite.Equal(b, fileInStorage) @@ -206,7 +206,7 @@ func (suite *ServeFileTestSuite) TestServeSmallFileSuccessful() { suite.NoError(err) suite.NotNil(b) - fileInStorage, err := suite.storage.Get(targetAttachment.Thumbnail.Path) + fileInStorage, err := suite.storage.Get(ctx, targetAttachment.Thumbnail.Path) suite.NoError(err) suite.NotNil(fileInStorage) suite.Equal(b, fileInStorage) diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 34b48a6ad3..153cde90f3 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -138,7 +138,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() { // see what's in storage *before* the request storageKeysBeforeRequest := []string{} - iter, err := suite.storage.Iterator(nil) + iter, err := suite.storage.KVStore.Iterator(nil) if err != nil { panic(err) } @@ -164,7 +164,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() { // check what's in storage *after* the request storageKeysAfterRequest := []string{} - iter, err = suite.storage.Iterator(nil) + iter, err = suite.storage.KVStore.Iterator(nil) if err != nil { panic(err) } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index f9d9a1f37c..f557634395 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -88,7 +88,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -101,7 +101,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -160,7 +160,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -173,7 +173,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -232,7 +232,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -245,7 +245,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -315,7 +315,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -328,7 +328,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -394,7 +394,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -407,7 +407,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) @@ -475,7 +475,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { suite.NotNil(dbAttachment) // make sure the processed file is in storage - processedFullBytes, err := suite.storage.Get(attachment.File.Path) + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) suite.NoError(err) suite.NotEmpty(processedFullBytes) @@ -488,7 +488,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { suite.Equal(processedFullBytesExpected, processedFullBytes) // now do the same for the thumbnail and make sure it's what we expected - processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path) + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) suite.NoError(err) suite.NotEmpty(processedThumbnailBytes) diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 97b1db136a..ffac560526 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -113,7 +113,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { switch processState(staticState) { case received: // stream the original file out of storage... - stored, err := p.storage.GetStream(p.emoji.ImagePath) + stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath) if err != nil { p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err) atomic.StoreInt32(&p.staticState, int32(errored)) @@ -135,7 +135,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { } // put the static in storage - if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil { + if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil { p.err = fmt.Errorf("loadStatic: error storing static: %s", err) atomic.StoreInt32(&p.staticState, int32(errored)) return p.err @@ -211,7 +211,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader) // store this for now -- other processes can pull it out of storage as they please - if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil { + if err := p.storage.PutStream(ctx, p.emoji.ImagePath, multiReader); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 1ce39377ca..17fddddb7e 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -138,7 +138,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { // stream the original file out of storage logrus.Tracef("loadThumb: fetching attachment from storage %s", p.attachment.URL) - stored, err := p.storage.GetStream(p.attachment.File.Path) + stored, err := p.storage.GetStream(ctx, p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err) atomic.StoreInt32(&p.thumbState, int32(errored)) @@ -164,7 +164,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { // put the thumbnail in storage logrus.Tracef("loadThumb: storing new thumbnail %s", p.attachment.URL) - if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil { + if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil { p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err) atomic.StoreInt32(&p.thumbState, int32(errored)) return p.err @@ -203,7 +203,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { var decoded *imageMeta // stream the original file out of storage... - stored, err := p.storage.GetStream(p.attachment.File.Path) + stored, err := p.storage.GetStream(ctx, p.attachment.File.Path) if err != nil { p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err) atomic.StoreInt32(&p.fullSizeState, int32(errored)) @@ -343,7 +343,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { p.attachment.File.FileSize = fileSize // store this for now -- other processes can pull it out of storage as they please - if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil { + if err := p.storage.PutStream(ctx, p.attachment.File.Path, clean); err != nil { return fmt.Errorf("store: error storing stream: %s", err) } p.attachment.Cached = true diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go index aa838d2a45..33391beafe 100644 --- a/internal/media/prunemeta.go +++ b/internal/media/prunemeta.go @@ -69,7 +69,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod if attachment.File.Path != "" { // delete the full size attachment from storage logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path) - if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { return err } } @@ -77,7 +77,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod if attachment.Thumbnail.Path != "" { // delete the thumbnail from storage logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { return err } } diff --git a/internal/media/prunemeta_test.go b/internal/media/prunemeta_test.go index 1358208a8b..8b250e7a55 100644 --- a/internal/media/prunemeta_test.go +++ b/internal/media/prunemeta_test.go @@ -49,13 +49,13 @@ func (suite *PruneMetaTestSuite) TestPruneMeta() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(zorkOldAvatar.File.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.File.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // attachments should no longer be in the db @@ -110,13 +110,13 @@ func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(zorkOldAvatar.File.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.File.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path) + _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // attachments should no longer be in the db diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go index f7b77d32ea..03fbc419ba 100644 --- a/internal/media/pruneremote.go +++ b/internal/media/pruneremote.go @@ -73,7 +73,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media if attachment.File.Path != "" { // delete the full size attachment from storage logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path) - if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { return err } attachment.Cached = false @@ -82,7 +82,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media if attachment.Thumbnail.Path != "" { // delete the thumbnail from storage logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { return err } attachment.Cached = false diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go index 31c5128ff0..f5ed8a618a 100644 --- a/internal/media/pruneremote_test.go +++ b/internal/media/pruneremote_test.go @@ -68,9 +68,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { suite.Equal(2, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(testAttachment.File.Path) + _, err = suite.storage.Get(ctx, testAttachment.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(testAttachment.Thumbnail.Path) + _, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // now recache the image.... @@ -98,9 +98,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same // recached files should be back in storage - _, err = suite.storage.Get(recachedAttachment.File.Path) + _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) suite.NoError(err) - _, err = suite.storage.Get(recachedAttachment.Thumbnail.Path) + _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) suite.NoError(err) } @@ -112,7 +112,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() { media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) suite.NoError(err) suite.True(media.Cached) - err = suite.storage.Delete(media.File.Path) + err = suite.storage.Delete(ctx, media.File.Path) suite.NoError(err) // Now attempt to prune remote for item with db entry no file diff --git a/internal/processing/media/delete.go b/internal/processing/media/delete.go index e99b4e950f..45c2183979 100644 --- a/internal/processing/media/delete.go +++ b/internal/processing/media/delete.go @@ -24,14 +24,14 @@ func (p *processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr // delete the thumbnail from storage if attachment.Thumbnail.Path != "" { - if err := p.storage.Delete(attachment.Thumbnail.Path); err != nil { + if err := p.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil { errs = append(errs, fmt.Sprintf("remove thumbnail at path %s: %s", attachment.Thumbnail.Path, err)) } } // delete the file from storage if attachment.File.Path != "" { - if err := p.storage.Delete(attachment.File.Path); err != nil { + if err := p.storage.Delete(ctx, attachment.File.Path); err != nil { errs = append(errs, fmt.Sprintf("remove file at path %s: %s", attachment.File.Path, err)) } } diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index bfed716937..3227cb8c89 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -113,7 +113,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount // if we have the media cached on our server already, we can now simply return it from storage if a.Cached { - return p.retrieveFromStorage(storagePath, attachmentContent) + return p.retrieveFromStorage(ctx, storagePath, attachmentContent) } // if we don't have it cached, then we can assume two things: @@ -221,7 +221,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err)) } // ... so now we can safely return it - return p.retrieveFromStorage(storagePath, attachmentContent) + return p.retrieveFromStorage(ctx, storagePath, attachmentContent) } return attachmentContent, nil @@ -253,15 +253,15 @@ func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, e return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize)) } - return p.retrieveFromStorage(storagePath, emojiContent) + return p.retrieveFromStorage(ctx, storagePath, emojiContent) } -func (p *processor) retrieveFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) { - if url := p.storage.URL(storagePath); url != nil { +func (p *processor) retrieveFromStorage(ctx context.Context, storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) { + if url := p.storage.URL(ctx, storagePath); url != nil { content.URL = url return content, nil } - reader, err := p.storage.GetStream(storagePath) + reader, err := p.storage.GetStream(ctx, storagePath) if err != nil { return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err)) } diff --git a/internal/processing/media/getfile_test.go b/internal/processing/media/getfile_test.go index 4d7bc46219..7c6525abe6 100644 --- a/internal/processing/media/getfile_test.go +++ b/internal/processing/media/getfile_test.go @@ -70,9 +70,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() { testAttachment.Cached = false err := suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch it @@ -106,7 +106,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() { suite.True(dbAttachment.Cached) // the file should be back in storage at the same path as before - refreshedBytes, err := suite.storage.Get(testAttachment.File.Path) + refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path) suite.NoError(err) suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes) } @@ -119,9 +119,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() { testAttachment.Cached = false err := suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch it @@ -156,7 +156,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() { suite.True(dbAttachment.Cached) // the file should be back in storage at the same path as before - refreshedBytes, err := suite.storage.Get(testAttachment.File.Path) + refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path) suite.NoError(err) suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes) } @@ -166,16 +166,16 @@ func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() { testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] // fetch the existing thumbnail bytes from storage first - thumbnailBytes, err := suite.storage.Get(testAttachment.Thumbnail.Path) + thumbnailBytes, err := suite.storage.Get(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // uncache the file from local testAttachment.Cached = false err = suite.db.UpdateByPrimaryKey(ctx, testAttachment) suite.NoError(err) - err = suite.storage.Delete(testAttachment.File.Path) + err = suite.storage.Delete(ctx, testAttachment.File.Path) suite.NoError(err) - err = suite.storage.Delete(testAttachment.Thumbnail.Path) + err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path) suite.NoError(err) // now fetch the thumbnail diff --git a/internal/storage/local.go b/internal/storage/local.go index 0c7d28628f..da57631f9f 100644 --- a/internal/storage/local.go +++ b/internal/storage/local.go @@ -19,15 +19,32 @@ package storage import ( + "context" + "io" "net/url" "codeberg.org/gruf/go-store/kv" ) type Local struct { - *kv.KVStore + KVStore *kv.KVStore } -func (l *Local) URL(key string) *url.URL { +func (l *Local) Get(ctx context.Context, key string) ([]byte, error) { + return l.KVStore.Get(key) +} +func (l *Local) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + return l.KVStore.GetStream(key) +} +func (l *Local) PutStream(ctx context.Context, key string, r io.Reader) error { + return l.KVStore.PutStream(key, r) +} +func (l *Local) Put(ctx context.Context, key string, value []byte) error { + return l.KVStore.Put(key, value) +} +func (l *Local) Delete(ctx context.Context, key string) error { + return l.KVStore.Delete(key) +} +func (l *Local) URL(ctx context.Context, key string) *url.URL { return nil } diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 4a64378616..a151144885 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -43,8 +43,8 @@ func NewS3(mc *minio.Client, bucket string) *S3 { } } -func (s *S3) Get(key string) ([]byte, error) { - r, err := s.GetStream(key) +func (s *S3) Get(ctx context.Context, key string) ([]byte, error) { + r, err := s.GetStream(ctx, key) if err != nil { return nil, err } @@ -55,32 +55,32 @@ func (s *S3) Get(key string) ([]byte, error) { } return b, nil } -func (s *S3) GetStream(key string) (io.ReadCloser, error) { - o, err := s.mc.GetObject(context.TODO(), s.bucket, key, minio.GetObjectOptions{}) +func (s *S3) GetStream(ctx context.Context, key string) (io.ReadCloser, error) { + o, err := s.mc.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) if err != nil { err = fmt.Errorf("retrieving object from s3: %w", err) } return o, err } -func (s *S3) PutStream(key string, r io.Reader) error { - if _, err := s.mc.PutObject(context.TODO(), s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil { +func (s *S3) PutStream(ctx context.Context, key string, r io.Reader) error { + if _, err := s.mc.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil { return fmt.Errorf("uploading data stream: %w", err) } return nil } -func (s *S3) Put(key string, value []byte) error { - if _, err := s.mc.PutObject(context.TODO(), s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil { +func (s *S3) Put(ctx context.Context, key string, value []byte) error { + if _, err := s.mc.PutObject(ctx, s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil { return fmt.Errorf("uploading data slice: %w", err) } return nil } -func (s *S3) Delete(key string) error { - return s.mc.RemoveObject(context.TODO(), s.bucket, key, minio.RemoveObjectOptions{}) +func (s *S3) Delete(ctx context.Context, key string) error { + return s.mc.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) } -func (s *S3) URL(key string) *url.URL { +func (s *S3) URL(ctx context.Context, key string) *url.URL { // it's safe to ignore the error here, as we just fall back to fetching the // file if the url request fails - url, _ := s.mc.PresignedGetObject(context.TODO(), s.bucket, key, time.Hour, url.Values{ + url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{ "response-content-type": []string{mime.TypeByExtension(path.Ext(key))}, }) return url diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 198e883ce2..88577442fa 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -19,6 +19,7 @@ package storage import ( + "context" "errors" "fmt" "io" @@ -39,12 +40,12 @@ var ( // Driver implements the functionality to store and retrieve blobs // (images,video,audio) type Driver interface { - Get(key string) ([]byte, error) - GetStream(key string) (io.ReadCloser, error) - PutStream(key string, r io.Reader) error - Put(key string, value []byte) error - Delete(key string) error - URL(key string) *url.URL + Get(ctx context.Context, key string) ([]byte, error) + GetStream(ctx context.Context, key string) (io.ReadCloser, error) + PutStream(ctx context.Context, key string, r io.Reader) error + Put(ctx context.Context, key string, value []byte) error + Delete(ctx context.Context, key string) error + URL(ctx context.Context, key string) *url.URL } func AutoConfig() (Driver, error) { diff --git a/testrig/storage.go b/testrig/storage.go index b00d8878b7..b55e1ed289 100644 --- a/testrig/storage.go +++ b/testrig/storage.go @@ -19,6 +19,7 @@ package testrig import ( + "context" "fmt" "os" "path" @@ -67,14 +68,14 @@ func StandardStorageSetup(s gtsstorage.Driver, relativePath string) { if err != nil { panic(err) } - if err := s.Put(pathOriginal, bOriginal); err != nil { + if err := s.Put(context.TODO(), pathOriginal, bOriginal); err != nil { panic(err) } bSmall, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameSmall)) if err != nil { panic(err) } - if err := s.Put(pathSmall, bSmall); err != nil { + if err := s.Put(context.TODO(), pathSmall, bSmall); err != nil { panic(err) } } @@ -94,14 +95,14 @@ func StandardStorageSetup(s gtsstorage.Driver, relativePath string) { if err != nil { panic(err) } - if err := s.Put(pathOriginal, bOriginal); err != nil { + if err := s.Put(context.TODO(), pathOriginal, bOriginal); err != nil { panic(err) } bStatic, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameStatic)) if err != nil { panic(err) } - if err := s.Put(pathStatic, bStatic); err != nil { + if err := s.Put(context.TODO(), pathStatic, bStatic); err != nil { panic(err) } } @@ -115,7 +116,7 @@ func StandardStorageTeardown(s gtsstorage.Driver) { switch st := s.(type) { case *gtsstorage.Local: - iter, err := st.Iterator(nil) + iter, err := st.KVStore.Iterator(nil) if err != nil { panic(err) } @@ -125,7 +126,7 @@ func StandardStorageTeardown(s gtsstorage.Driver) { } iter.Release() for _, k := range keys { - if err := s.Delete(k); err != nil { + if err := s.Delete(context.TODO(), k); err != nil { panic(err) } }