Skip to content

Commit

Permalink
Fix various naming issues (Azure#19067)
Browse files Browse the repository at this point in the history
* Fix various naming issues

Use BlockSize and Concurrency consistenty across operations.
Removed XMS prefix from ContentCRC64.
Renamed list blobs API and its dependent types to include the word Flat.

* renamed test vars
  • Loading branch information
jhendrixMSFT authored Sep 12, 2022
1 parent bb83d11 commit 00ba803
Show file tree
Hide file tree
Showing 12 changed files with 90 additions and 85 deletions.
2 changes: 1 addition & 1 deletion sdk/storage/azblob/blob/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ func (b *Client) download(ctx context.Context, writer io.WriterAt, o downloadOpt
OperationName: "downloadBlobToWriterAt",
TransferSize: count,
ChunkSize: o.BlockSize,
Parallelism: o.Parallelism,
Concurrency: o.Concurrency,
Operation: func(chunkStart int64, count int64, ctx context.Context) error {

downloadBlobOptions := o.getDownloadBlobOptions(HTTPRange{
Expand Down
12 changes: 6 additions & 6 deletions sdk/storage/azblob/blob/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ type downloadOptions struct {
CpkInfo *CpkInfo
CpkScopeInfo *CpkScopeInfo

// Parallelism indicates the maximum number of blocks to download in parallel (0=default)
Parallelism uint16
// Concurrency indicates the maximum number of blocks to download in parallel (0=default)
Concurrency uint16

// RetryReaderOptionsPerBlock is used when downloading each block.
RetryReaderOptionsPerBlock RetryReaderOptions
Expand Down Expand Up @@ -154,8 +154,8 @@ type DownloadBufferOptions struct {
// CpkScopeInfo contains a group of parameters for client provided encryption scope.
CpkScopeInfo *CpkScopeInfo

// Parallelism indicates the maximum number of blocks to download in parallel (0=default)
Parallelism uint16
// Concurrency indicates the maximum number of blocks to download in parallel (0=default)
Concurrency uint16

// RetryReaderOptionsPerBlock is used when downloading each block.
RetryReaderOptionsPerBlock RetryReaderOptions
Expand All @@ -179,8 +179,8 @@ type DownloadFileOptions struct {
CpkInfo *CpkInfo
CpkScopeInfo *CpkScopeInfo

// Parallelism indicates the maximum number of blocks to download in parallel. The default value is 5.
Parallelism uint16
// Concurrency indicates the maximum number of blocks to download in parallel. The default value is 5.
Concurrency uint16

// RetryReaderOptionsPerBlock is used when downloading each block.
RetryReaderOptionsPerBlock RetryReaderOptions
Expand Down
4 changes: 2 additions & 2 deletions sdk/storage/azblob/blockblob/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ func (bb *Client) uploadFromReader(ctx context.Context, reader io.ReaderAt, read
if o.BlockSize < blob.DefaultDownloadBlockSize { // If the block size is smaller than 4MB, round up to 4MB
o.BlockSize = blob.DefaultDownloadBlockSize
}
// StageBlock will be called with blockSize blocks and a Parallelism of (BufferSize / BlockSize).
// StageBlock will be called with blockSize blocks and a Concurrency of (BufferSize / BlockSize).
}
}

Expand All @@ -352,7 +352,7 @@ func (bb *Client) uploadFromReader(ctx context.Context, reader io.ReaderAt, read
OperationName: "uploadFromReader",
TransferSize: readerSize,
ChunkSize: o.BlockSize,
Parallelism: o.Parallelism,
Concurrency: o.Concurrency,
Operation: func(offset int64, count int64, ctx context.Context) error {
// This function is called once per block.
// It is passed this block's offset within the buffer and its count of bytes
Expand Down
4 changes: 2 additions & 2 deletions sdk/storage/azblob/blockblob/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3188,8 +3188,8 @@ func (s *BlockBlobUnrecordedTestsSuite) TestUploadStreamToBlobProperties() {
// Perform UploadStream
_, err = bbClient.UploadStream(context.Background(), blobContentReader,
&blockblob.UploadStreamOptions{
BufferSize: bufferSize,
MaxBuffers: maxBuffers,
BlockSize: bufferSize,
Concurrency: maxBuffers,
Metadata: testcommon.BasicMetadata,
Tags: testcommon.BasicBlobTagsMap,
HTTPHeaders: &testcommon.BasicHeaders,
Expand Down
29 changes: 17 additions & 12 deletions sdk/storage/azblob/blockblob/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,9 @@ type uploadFromReaderOptions struct {
CpkInfo *blob.CpkInfo
CpkScopeInfo *blob.CpkScopeInfo

// Parallelism indicates the maximum number of blocks to upload in parallel (0=default)
Parallelism uint16
// Concurrency indicates the maximum number of blocks to upload in parallel (0=default)
Concurrency uint16

// Optional header, Specifies the transactional crc64 for the body, to be validated by the service.
TransactionalContentCRC64 *[]byte
// Specify the transactional md5 for the body, to be validated by the service.
Expand Down Expand Up @@ -244,13 +245,17 @@ func (o *uploadFromReaderOptions) getCommitBlockListOptions() *CommitBlockListOp
// UploadStreamOptions provides set of configurations for UploadStream operation
type UploadStreamOptions struct {
// transferManager provides a transferManager that controls buffer allocation/reuse and
// concurrency. This overrides BufferSize and MaxBuffers if set.
// concurrency. This overrides BlockSize and MaxConcurrency if set.
transferManager shared.TransferManager
transferMangerNotSet bool
// BufferSize sizes the buffer used to read data from source. If < 1 MiB, format to 1 MiB.
BufferSize int
// MaxBuffers defines the number of simultaneous uploads will be performed to upload the file.
MaxBuffers int

// BlockSize defines the size of the buffer used during upload. The default and mimimum value is 1 MiB.
BlockSize int

// Concurrency defines the number of concurrent uploads to be performed to upload the file.
// Each concurrent upload will create a buffer of size BlockSize. The default value is one.
Concurrency int

HTTPHeaders *blob.HTTPHeaders
Metadata map[string]string
AccessConditions *blob.AccessConditions
Expand All @@ -265,16 +270,16 @@ func (u *UploadStreamOptions) format() error {
return nil
}

if u.MaxBuffers == 0 {
u.MaxBuffers = 1
if u.Concurrency == 0 {
u.Concurrency = 1
}

if u.BufferSize < _1MiB {
u.BufferSize = _1MiB
if u.BlockSize < _1MiB {
u.BlockSize = _1MiB
}

var err error
u.transferManager, err = shared.NewStaticBuffer(u.BufferSize, u.MaxBuffers)
u.transferManager, err = shared.NewStaticBuffer(u.BlockSize, u.Concurrency)
if err != nil {
return fmt.Errorf("bug: default transfer manager could not be created: %s", err)
}
Expand Down
6 changes: 3 additions & 3 deletions sdk/storage/azblob/blockblob/responses.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,9 @@ type uploadFromReaderResponse struct {
// VersionID contains the information returned from the x-ms-version-id header response.
VersionID *string

// XMSContentCRC64 contains the information returned from the x-ms-content-crc64 header response.
// ContentCRC64 contains the information returned from the x-ms-content-crc64 header response.
// Will be a part of response only if uploading data >= internal.MaxUploadBlobBytes (= 256 * 1024 * 1024 // 256MB)
XMSContentCRC64 []byte
ContentCRC64 []byte
}

func toUploadReaderAtResponseFromUploadResponse(resp UploadResponse) uploadFromReaderResponse {
Expand Down Expand Up @@ -97,7 +97,7 @@ func toUploadReaderAtResponseFromCommitBlockListResponse(resp CommitBlockListRes
RequestID: resp.RequestID,
Version: resp.Version,
VersionID: resp.VersionID,
XMSContentCRC64: resp.XMSContentCRC64,
ContentCRC64: resp.XMSContentCRC64,
}
}

Expand Down
4 changes: 2 additions & 2 deletions sdk/storage/azblob/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,10 @@ func (c *Client) DeleteBlob(ctx context.Context, containerName string, blobName
return c.svc.NewContainerClient(containerName).NewBlobClient(blobName).Delete(ctx, o)
}

// NewListBlobsPager returns a pager for blobs starting from the specified Marker. Use an empty
// NewListBlobsFlatPager returns a pager for blobs starting from the specified Marker. Use an empty
// Marker to start enumeration from the beginning. Blob names are returned in lexicographic order.
// For more information, see https://docs.microsoft.com/rest/api/storageservices/list-blobs.
func (c *Client) NewListBlobsPager(containerName string, o *ListBlobsOptions) *runtime.Pager[ListBlobsResponse] {
func (c *Client) NewListBlobsFlatPager(containerName string, o *ListBlobsFlatOptions) *runtime.Pager[ListBlobsFlatResponse] {
return c.svc.NewContainerClient(containerName).NewListBlobsFlatPager(o)
}

Expand Down
Loading

0 comments on commit 00ba803

Please sign in to comment.