[performance] media processing improvements

go.mod

@@ -10,9 +10,9 @@ require (
 	codeberg.org/gruf/go-errors/v2 v2.0.2
 	codeberg.org/gruf/go-kv v1.5.2
 	codeberg.org/gruf/go-logger/v2 v2.2.1
-	codeberg.org/gruf/go-mutexes v1.1.4
+	codeberg.org/gruf/go-mutexes v1.1.5
 	codeberg.org/gruf/go-runners v1.4.0
-	codeberg.org/gruf/go-store/v2 v2.0.10
+	codeberg.org/gruf/go-store/v2 v2.2.1
 	github.com/abema/go-mp4 v0.9.0
 	github.com/buckket/go-blurhash v1.1.0
 	github.com/coreos/go-oidc/v3 v3.5.0
@@ -65,10 +65,11 @@ require (
 	codeberg.org/gruf/go-atomics v1.1.0 // indirect
 	codeberg.org/gruf/go-bitutil v1.0.1 // indirect
 	codeberg.org/gruf/go-bytes v1.0.2 // indirect
-	codeberg.org/gruf/go-fastcopy v1.1.1 // indirect
+	codeberg.org/gruf/go-fastcopy v1.1.2 // indirect
 	codeberg.org/gruf/go-fastpath v1.0.3 // indirect
 	codeberg.org/gruf/go-fastpath/v2 v2.0.0 // indirect
 	codeberg.org/gruf/go-hashenc v1.0.2 // indirect
+	codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225 // indirect
 	codeberg.org/gruf/go-mangler v1.2.2 // indirect
 	codeberg.org/gruf/go-maps v1.0.3 // indirect
 	codeberg.org/gruf/go-pools v1.1.0 // indirect

go.sum

@@ -56,15 +56,17 @@ codeberg.org/gruf/go-debug v1.2.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIj
 codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4=
 codeberg.org/gruf/go-errors/v2 v2.0.2 h1:T9CqfC+ntSIQL5mdQxwHlUMod1htpgNe3P1tugxKlT4=
 codeberg.org/gruf/go-errors/v2 v2.0.2/go.mod h1:6sI75OmvXE2AtRm4WUyGMEyqEOKTsfe+CA+aBXwbtJY=
-codeberg.org/gruf/go-fastcopy v1.1.1 h1:HhPCeFdVR5pwiSVDnQEGJ+J2ny9b5QgfiESc0zrWQAY=
-codeberg.org/gruf/go-fastcopy v1.1.1/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s=
+codeberg.org/gruf/go-fastcopy v1.1.2 h1:YwmYXPsyOcRBxKEE2+w1bGAZfclHVaPijFsOVOcnNcw=
+codeberg.org/gruf/go-fastcopy v1.1.2/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s=
 codeberg.org/gruf/go-fastpath v1.0.1/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI=
 codeberg.org/gruf/go-fastpath v1.0.3 h1:3Iftz9Z2suCEgTLkQMucew+2+4Oe46JPbAM2JEhnjTU=
 codeberg.org/gruf/go-fastpath v1.0.3/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI=
 codeberg.org/gruf/go-fastpath/v2 v2.0.0 h1:iAS9GZahFhyWEH0KLhFEJR+txx1ZhMXxYzu2q5Qo9c0=
 codeberg.org/gruf/go-fastpath/v2 v2.0.0/go.mod h1:3pPqu5nZjpbRrOqvLyAK7puS1OfEtQvjd6342Cwz56Q=
 codeberg.org/gruf/go-hashenc v1.0.2 h1:U3jH6zMXZiL96czD/qaJd8OR2h7LlBzGv/2WxnMHI/g=
 codeberg.org/gruf/go-hashenc v1.0.2/go.mod h1:eK+A8clLcEN/m1nftNsRId0kfYDQnETnuIfBGZ8Gvsg=
+codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225 h1:tP9YvEBfADGG3mXkfrALLadlcbrZsFsWKZvFtUZtrt8=
+codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225/go.mod h1:B8uq4yHtIcKXhBZT9C/SYisz25lldLHMVpwZPz4ADLQ=
 codeberg.org/gruf/go-kv v1.5.2 h1:B0RkAXLUXYn3Za1NzTXOcUvAc+JUC2ZadTMkCUDa0mc=
 codeberg.org/gruf/go-kv v1.5.2/go.mod h1:al6ASW/2CbGqz2YcM8B00tvWnVi1bU1CH3HYs5tZxo4=
 codeberg.org/gruf/go-logger/v2 v2.2.1 h1:RP2u059EQKTBFV3cN8X6xDxNk2RkzqdgXGKflKqB7Oc=
@@ -73,16 +75,16 @@ codeberg.org/gruf/go-mangler v1.2.2 h1:fisdWXa6dW4p1uYdbz5Of3R4lDDFPuRqKavGI9O03
 codeberg.org/gruf/go-mangler v1.2.2/go.mod h1:X/7URkFhLBAVKkTxmqF11Oxw3A6pSSxgPeHssQaiq28=
 codeberg.org/gruf/go-maps v1.0.3 h1:VDwhnnaVNUIy5O93CvkcE2IZXnMB1+IJjzfop9V12es=
 codeberg.org/gruf/go-maps v1.0.3/go.mod h1:D5LNDxlC9rsDuVQVM6JObaVGAdHB6g2dTdOdkh1aXWA=
-codeberg.org/gruf/go-mutexes v1.1.4 h1:HWaIZavPL92SBJxNOlIXAmAT5CB2hAs72/lBN31jnzM=
-codeberg.org/gruf/go-mutexes v1.1.4/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8=
+codeberg.org/gruf/go-mutexes v1.1.5 h1:8Y8DwCGf24MyzOSaPvLrtk/B4ecVx4z+fppL6dY+PG8=
+codeberg.org/gruf/go-mutexes v1.1.5/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8=
 codeberg.org/gruf/go-pools v1.1.0 h1:LbYP24eQLl/YI1fSU2pafiwhGol1Z1zPjRrMsXpF88s=
 codeberg.org/gruf/go-pools v1.1.0/go.mod h1:ZMYpt/DjQWYC3zFD3T97QWSFKs62zAUGJ/tzvgB9D68=
 codeberg.org/gruf/go-runners v1.4.0 h1:977nVjigAdH95+VAB/a6tyBJOKk99e60h+mfHzBs/n8=
 codeberg.org/gruf/go-runners v1.4.0/go.mod h1:kUM6GYL7dC+f9Sc/XuwdvB/mB4FuI4fJFb150ADMsmw=
 codeberg.org/gruf/go-sched v1.2.0 h1:utZl/7srVcbh30rFw42LC2/cMtak4UZRxtIOt/5riNA=
 codeberg.org/gruf/go-sched v1.2.0/go.mod h1:v4ueWq+fAtAw9JYt4aFXvadI1YoOqofgHQgszRYuslA=
-codeberg.org/gruf/go-store/v2 v2.0.10 h1:/2iZ4j29A//EhM3XziJP6SxtdIcaAyPmJEv31+6XD8g=
-codeberg.org/gruf/go-store/v2 v2.0.10/go.mod h1:KMRE173S6W2sGhuIa4jY/OPIO65F9++7rmWTfZ4xTeY=
+codeberg.org/gruf/go-store/v2 v2.2.1 h1:lbvMjhMLebefiaPNLtWvPySKSYM5xN1aztSxxz+vCzU=
+codeberg.org/gruf/go-store/v2 v2.2.1/go.mod h1:pxdyfSzau8fFs1TfZlyRzhDYvZWLaj1sXpcjXpzBB6k=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -818,6 +820,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

internal/api/client/accounts/accountupdate_test.go

@@ -300,8 +300,8 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerWit
 	suite.NotEmpty(apimodelAccount.HeaderStatic)
 
 	// should be different from the values set before
-	suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.Header)
-	suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.HeaderStatic)
+	suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.Header)
+	suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.HeaderStatic)
 }
 
 func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerEmptyForm() {

internal/api/client/accounts/accountverify_test.go

@@ -74,10 +74,10 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() {
 	suite.Equal(*testAccount.Bot, apimodelAccount.Bot)
 	suite.WithinDuration(testAccount.CreatedAt, createdAt, 30*time.Second) // we lose a bit of accuracy serializing so fuzz this a bit
 	suite.Equal(testAccount.URL, apimodelAccount.URL)
-	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg", apimodelAccount.Avatar)
-	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg", apimodelAccount.AvatarStatic)
-	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.Header)
-	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.HeaderStatic)
+	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg", apimodelAccount.Avatar)
+	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg", apimodelAccount.AvatarStatic)
+	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.Header)
+	suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.HeaderStatic)
 	suite.Equal(2, apimodelAccount.FollowersCount)
 	suite.Equal(2, apimodelAccount.FollowingCount)
 	suite.Equal(5, apimodelAccount.StatusesCount)

internal/api/fileserver/servefile.go

@@ -117,14 +117,19 @@ func (m *Module) ServeFile(c *gin.Context) {
 		return
 	}
 
-	// try to slurp the first few bytes to make sure we have something
-	b := bytes.NewBuffer(make([]byte, 0, 64))
-	if _, err := io.CopyN(b, content.Content, 64); err != nil {
+	// create a "slurp" buffer ;)
+	b := make([]byte, 64)
+
+	// Try read the first 64 bytes into memory, to try return a more useful "not found" error.
+	if _, err := io.ReadFull(content.Content, b); err != nil &&
+		(err != io.ErrUnexpectedEOF && err != io.EOF) {
 		err = fmt.Errorf("ServeFile: error reading from content: %w", err)
 		apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGet)
 		return
 	}
 
 	// we're good, return the slurped bytes + the rest of the content
-	c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(b, content.Content), nil)
+	c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(
+		bytes.NewReader(b), content.Content,
+	), nil)
 }

internal/api/fileserver/servefile_test.go

@@ -99,7 +99,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalLocalFileOK() {
 		targetAttachment.AccountID,
 		media.TypeAttachment,
 		media.SizeOriginal,
-		targetAttachment.ID+".jpeg",
+		targetAttachment.ID+".jpg",
 	)
 
 	suite.Equal(http.StatusOK, code)
@@ -119,7 +119,7 @@ func (suite *ServeFileTestSuite) TestServeSmallLocalFileOK() {
 		targetAttachment.AccountID,
 		media.TypeAttachment,
 		media.SizeSmall,
-		targetAttachment.ID+".jpeg",
+		targetAttachment.ID+".jpg",
 	)
 
 	suite.Equal(http.StatusOK, code)
@@ -139,7 +139,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileOK() {
 		targetAttachment.AccountID,
 		media.TypeAttachment,
 		media.SizeOriginal,
-		targetAttachment.ID+".jpeg",
+		targetAttachment.ID+".jpg",
 	)
 
 	suite.Equal(http.StatusOK, code)
@@ -159,7 +159,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileOK() {
 		targetAttachment.AccountID,
 		media.TypeAttachment,
 		media.SizeSmall,
-		targetAttachment.ID+".jpeg",
+		targetAttachment.ID+".jpg",
 	)
 
 	suite.Equal(http.StatusOK, code)
@@ -182,7 +182,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecache() {
 		targetAttachment.AccountID,
 		media.TypeAttachment,
 		media.SizeOriginal,
-		targetAttachment.ID+".jpeg",
+		targetAttachment.ID+".jpg",
 	)
 
 	suite.Equal(http.StatusOK, code)
@@ -205,7 +205,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecache() {
 		targetAttachment.AccountID,
 		media.TypeAttachment,
 		media.SizeSmall,
-		targetAttachment.ID+".jpeg",
+		targetAttachment.ID+".jpg",
 	)
 
 	suite.Equal(http.StatusOK, code)
@@ -228,7 +228,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecacheNotFound() {
 		targetAttachment.AccountID,
 		media.TypeAttachment,
 		media.SizeOriginal,
-		targetAttachment.ID+".jpeg",
+		targetAttachment.ID+".jpg",
 	)
 
 	suite.Equal(http.StatusNotFound, code)
@@ -249,7 +249,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecacheNotFound() {
 		targetAttachment.AccountID,
 		media.TypeAttachment,
 		media.SizeSmall,
-		targetAttachment.ID+".jpeg",
+		targetAttachment.ID+".jpg",
 	)
 
 	suite.Equal(http.StatusNotFound, code)
@@ -261,7 +261,7 @@ func (suite *ServeFileTestSuite) TestServeFileNotFound() {
 		"01GMMY4G9B0QEG0PQK5Q5JGJWZ",
 		media.TypeAttachment,
 		media.SizeOriginal,
-		"01GMMY68Y7E5DJ3CA3Y9SS8524.jpeg",
+		"01GMMY68Y7E5DJ3CA3Y9SS8524.jpg",
 	)
 
 	suite.Equal(http.StatusNotFound, code)

internal/federation/dereferencing/media_test.go

@@ -21,11 +21,11 @@ package dereferencing_test
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
-	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
 type AttachmentTestSuite struct {
@@ -42,7 +42,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
 	attachmentContentType := "image/jpeg"
 	attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
 	attachmentDescription := "It's a cute plushie."
-	attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}"
+	attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az"
 
 	media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
 		StatusID:    &attachmentStatus,
@@ -116,7 +116,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
 	attachmentContentType := "image/jpeg"
 	attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
 	attachmentDescription := "It's a cute plushie."
-	attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}"
+	attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az"
 
 	processingMedia, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
 		StatusID:    &attachmentStatus,
@@ -127,11 +127,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
 	suite.NoError(err)
 	attachmentID := processingMedia.AttachmentID()
 
-	if !testrig.WaitFor(func() bool {
-		return processingMedia.Finished()
-	}) {
-		suite.FailNow("timed out waiting for media to be processed")
-	}
+	time.Sleep(time.Second * 3)
 
 	// now get the attachment from the database
 	attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)

internal/iotools/io.go

@@ -119,3 +119,41 @@ func (w *SilentWriter) Write(b []byte) (int, error) {
 func (w *SilentWriter) Error() error {
 	return w.err
 }
+
+func StreamReadFunc(read func(io.Reader) error) io.Writer {
+	// In-memory stream.
+	pr, pw := io.Pipe()
+
+	go func() {
+		var err error
+
+		defer func() {
+			// Always pass along error.
+			pr.CloseWithError(err)
+		}()
+
+		// Start reading.
+		err = read(pr)
+	}()
+
+	return pw
+}
+
+func StreamWriteFunc(write func(io.Writer) error) io.Reader {
+	// In-memory stream.
+	pr, pw := io.Pipe()
+
+	go func() {
+		var err error
+
+		defer func() {
+			// Always pass along error.
+			pw.CloseWithError(err)
+		}()
+
+		// Start writing.
+		err = write(pw)
+	}()
+
+	return pr
+}