Skip to content

Commit

Permalink
[FIX] Azure upload logic
Browse files Browse the repository at this point in the history
Signed-off-by: rita.canavarro <rita.canavarro@farfetch.com>
  • Loading branch information
rita.canavarro committed Jul 28, 2023
1 parent 7525c01 commit 0d765de
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 6 deletions.
12 changes: 9 additions & 3 deletions providers/azure/azure.go
Original file line number Diff line number Diff line change
Expand Up @@ -295,17 +295,23 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) {
}

// Upload the contents of the reader as an object into the bucket.
func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error {
func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) {
level.Debug(b.logger).Log("msg", "uploading blob", "blob", name)
size, err := objstore.TryToGetSize(r)
if err != nil {
return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name)
}

blobClient := b.containerClient.NewBlockBlobClient(name)
opts := &blockblob.UploadStreamOptions{
BlockSize: 3 * 1024 * 1024,
Concurrency: 4,
}

if _, err := blobClient.UploadStream(ctx, r, opts); err != nil {
return errors.Wrapf(err, "cannot upload Azure blob, address: %s", name)
return 0, errors.Wrapf(err, "cannot upload Azure blob, address: %s", name)
}
return nil
return size, nil
}

// Delete removes the object with the given name.
Expand Down
7 changes: 4 additions & 3 deletions providers/s3/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e
if size < int64(partSize) {
partSize = 0
}
if _, err := b.client.PutObject(
resp, err := b.client.PutObject(
ctx,
b.name,
name,
Expand All @@ -511,11 +511,12 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e
// TODO(bwplotka): Consider adjusting this number to GOMAXPROCS or to expose this in config if it becomes bottleneck.
NumThreads: 4,
},
); err != nil {
)
if err != nil {
return 0, errors.Wrap(err, "upload s3 object")
}

return size, nil
return resp.Size, nil
}

// Attributes returns information about the specified object.
Expand Down

0 comments on commit 0d765de

Please sign in to comment.