Preserve file size when creating attachments (#23406) (#23426)

Backport #23406 by @baez90

When creating attachments (issue, release, repo) the file size (being
part of the multipart file header) is passed through the chain of
creating an attachment to ensure the MinIO client can stream the file
directly instead of having to read it to memory completely at first.

Fixes #23393

Co-authored-by: Peter <peter.kurfer@googlemail.com>
Co-authored-by: KN4CK3R <admin@oldschoolhack.me>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

Author: 4 people

Diff for: routers/api/v1/repo/issue_attachment.go

@@ -176,7 +176,7 @@ func CreateIssueAttachment(ctx *context.APIContext) {
 		filename = query
 	}
 
-	attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, &repo_model.Attachment{
+	attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{
 		Name:       filename,
 		UploaderID: ctx.Doer.ID,
 		RepoID:     ctx.Repo.Repository.ID,

Diff for: routers/api/v1/repo/issue_comment_attachment.go

@@ -180,7 +180,7 @@ func CreateIssueCommentAttachment(ctx *context.APIContext) {
 		filename = query
 	}
 
-	attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, &repo_model.Attachment{
+	attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{
 		Name:       filename,
 		UploaderID: ctx.Doer.ID,
 		RepoID:     ctx.Repo.Repository.ID,

Diff for: routers/api/v1/repo/release_attachment.go

@@ -194,7 +194,7 @@ func CreateReleaseAttachment(ctx *context.APIContext) {
 	}
 
 	// Create a new attachment and save the file
-	attach, err := attachment.UploadAttachment(file, setting.Repository.Release.AllowedTypes, &repo_model.Attachment{
+	attach, err := attachment.UploadAttachment(file, setting.Repository.Release.AllowedTypes, header.Size, &repo_model.Attachment{
 		Name:       filename,
 		UploaderID: ctx.Doer.ID,
 		RepoID:     release.RepoID,

Diff for: routers/web/repo/attachment.go

@@ -44,7 +44,7 @@ func uploadAttachment(ctx *context.Context, repoID int64, allowedTypes string) {
 	}
 	defer file.Close()
 
-	attach, err := attachment.UploadAttachment(file, allowedTypes, &repo_model.Attachment{
+	attach, err := attachment.UploadAttachment(file, allowedTypes, header.Size, &repo_model.Attachment{
 		Name:       header.Filename,
 		UploaderID: ctx.Doer.ID,
 		RepoID:     repoID,

Diff for: services/attachment/attachment.go

@@ -19,14 +19,14 @@ import (
 )
 
 // NewAttachment creates a new attachment object, but do not verify.
-func NewAttachment(attach *repo_model.Attachment, file io.Reader) (*repo_model.Attachment, error) {
+func NewAttachment(attach *repo_model.Attachment, file io.Reader, size int64) (*repo_model.Attachment, error) {
 	if attach.RepoID == 0 {
 		return nil, fmt.Errorf("attachment %s should belong to a repository", attach.Name)
 	}
 
 	err := db.WithTx(db.DefaultContext, func(ctx context.Context) error {
 		attach.UUID = uuid.New().String()
-		size, err := storage.Attachments.Save(attach.RelativePath(), file, -1)
+		size, err := storage.Attachments.Save(attach.RelativePath(), file, size)
 		if err != nil {
 			return fmt.Errorf("Create: %w", err)
 		}
@@ -39,7 +39,7 @@ func NewAttachment(attach *repo_model.Attachment, file io.Reader) (*repo_model.A
 }
 
 // UploadAttachment upload new attachment into storage and update database
-func UploadAttachment(file io.Reader, allowedTypes string, opts *repo_model.Attachment) (*repo_model.Attachment, error) {
+func UploadAttachment(file io.Reader, allowedTypes string, fileSize int64, opts *repo_model.Attachment) (*repo_model.Attachment, error) {
 	buf := make([]byte, 1024)
 	n, _ := util.ReadAtMost(file, buf)
 	buf = buf[:n]
@@ -48,5 +48,5 @@ func UploadAttachment(file io.Reader, allowedTypes string, opts *repo_model.Atta
 		return nil, err
 	}
 
-	return NewAttachment(opts, io.MultiReader(bytes.NewReader(buf), file))
+	return NewAttachment(opts, io.MultiReader(bytes.NewReader(buf), file), fileSize)
 }

Diff for: services/attachment/attachment_test.go

@@ -36,7 +36,7 @@ func TestUploadAttachment(t *testing.T) {
 		RepoID:     1,
 		UploaderID: user.ID,
 		Name:       filepath.Base(fPath),
-	}, f)
+	}, f, -1)
 	assert.NoError(t, err)
 
 	attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attach.UUID)

Diff for: services/mailer/incoming/incoming_handler.go

@@ -87,7 +87,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, doer *u
 		attachmentIDs := make([]string, 0, len(content.Attachments))
 		if setting.Attachment.Enabled {
 			for _, attachment := range content.Attachments {
-				a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, &repo_model.Attachment{
+				a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, int64(len(attachment.Content)), &repo_model.Attachment{
 					Name:       attachment.Name,
 					UploaderID: doer.ID,
 					RepoID:     issue.Repo.ID,

Diff for: services/release/release_test.go

@@ -106,11 +106,13 @@ func TestRelease_Create(t *testing.T) {
 		IsTag:        false,
 	}, nil, ""))
 
+	testPlayload := "testtest"
+
 	attach, err := attachment.NewAttachment(&repo_model.Attachment{
 		RepoID:     repo.ID,
 		UploaderID: user.ID,
 		Name:       "test.txt",
-	}, strings.NewReader("testtest"))
+	}, strings.NewReader(testPlayload), int64(len([]byte(testPlayload))))
 	assert.NoError(t, err)
 
 	release := repo_model.Release{
@@ -239,11 +241,12 @@ func TestRelease_Update(t *testing.T) {
 	assert.Equal(t, tagName, release.TagName)
 
 	// Add new attachments
+	samplePayload := "testtest"
 	attach, err := attachment.NewAttachment(&repo_model.Attachment{
 		RepoID:     repo.ID,
 		UploaderID: user.ID,
 		Name:       "test.txt",
-	}, strings.NewReader("testtest"))
+	}, strings.NewReader(samplePayload), int64(len([]byte(samplePayload))))
 	assert.NoError(t, err)
 
 	assert.NoError(t, UpdateRelease(user, gitRepo, release, []string{attach.UUID}, nil, nil))