Skip to content

Commit

Permalink
stageblock | appendblock | putpages StructuredMessage (Azure#42699)
Browse files Browse the repository at this point in the history 
* regenerate and stage block uses structured message

* page and append

* testproxy

* cleanup

* fix datalake/share tests

* testproxy

* testproxy

* re-add null-safe access
  • Loading branch information
@jaschrep-msft
jaschrep-msft committed Aug 12, 2024
1 parent 26d8a12 commit 305ee6f
Show file tree
Hide file tree
Showing 29 changed files with 330 additions and 97 deletions.
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.Blobs/assets.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.Blobs",
"Tag": "net/storage/Azure.Storage.Blobs_14eb1d6279"
"Tag": "net/storage/Azure.Storage.Blobs_5d0f81abfc"
}
42 changes: 34 additions & 8 deletions sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1242,14 +1242,36 @@ internal async Task<Response<BlobAppendInfo>> AppendBlockInternal(
BlobErrors.VerifyHttpsCustomerProvidedKey(Uri, ClientConfiguration.CustomerProvidedKey);
Errors.VerifyStreamPosition(content, nameof(content));

// compute hash BEFORE attaching progress handler
ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);

content = content.WithNoDispose().WithProgress(progressHandler);
ContentHasher.GetHashResult hashResult = null;
long contentLength = (content?.Length - content?.Position) ?? 0;
long? structuredContentLength = default;
string structuredBodyType = null;
if (validationOptions != null &&
validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 &&
validationOptions.PrecalculatedChecksum.IsEmpty &&
ClientSideEncryption == null) // don't allow feature combination
{
// report progress in terms of caller bytes, not encoded bytes
structuredContentLength = contentLength;
contentLength = (content?.Length - content?.Position) ?? 0;
structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
content = content.WithNoDispose().WithProgress(progressHandler);
content = new StructuredMessageEncodingStream(
content,
Constants.StructuredMessage.DefaultSegmentContentLength,
StructuredMessage.Flags.StorageCrc64);
contentLength = (content?.Length - content?.Position) ?? 0;
}
else
{
// compute hash BEFORE attaching progress handler
hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);
content = content.WithNoDispose().WithProgress(progressHandler);
}

ResponseWithHeaders<AppendBlobAppendBlockHeaders> response;

Expand All @@ -1267,6 +1289,8 @@ internal async Task<Response<BlobAppendInfo>> AppendBlockInternal(
encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash,
encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256,
encryptionScope: ClientConfiguration.EncryptionScope,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
ifModifiedSince: conditions?.IfModifiedSince,
ifUnmodifiedSince: conditions?.IfUnmodifiedSince,
ifMatch: conditions?.IfMatch?.ToString(),
Expand All @@ -1289,6 +1313,8 @@ internal async Task<Response<BlobAppendInfo>> AppendBlockInternal(
encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash,
encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256,
encryptionScope: ClientConfiguration.EncryptionScope,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
ifModifiedSince: conditions?.IfModifiedSince,
ifUnmodifiedSince: conditions?.IfUnmodifiedSince,
ifMatch: conditions?.IfMatch?.ToString(),
Expand Down
46 changes: 36 additions & 10 deletions sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1305,22 +1305,44 @@ internal virtual async Task<Response<BlockInfo>> StageBlockInternal(

Errors.VerifyStreamPosition(content, nameof(content));

// compute hash BEFORE attaching progress handler
ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);

content = content.WithNoDispose().WithProgress(progressHandler);
ContentHasher.GetHashResult hashResult = null;
long contentLength = (content?.Length - content?.Position) ?? 0;
long? structuredContentLength = default;
string structuredBodyType = null;
if (validationOptions != null &&
validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 &&
validationOptions.PrecalculatedChecksum.IsEmpty &&
ClientSideEncryption == null) // don't allow feature combination
{
// report progress in terms of caller bytes, not encoded bytes
structuredContentLength = contentLength;
contentLength = (content?.Length - content?.Position) ?? 0;
structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
content = content.WithNoDispose().WithProgress(progressHandler);
content = new StructuredMessageEncodingStream(
content,
Constants.StructuredMessage.DefaultSegmentContentLength,
StructuredMessage.Flags.StorageCrc64);
contentLength = (content?.Length - content?.Position) ?? 0;
}
else
{
// compute hash BEFORE attaching progress handler
hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);
content = content.WithNoDispose().WithProgress(progressHandler);
}

ResponseWithHeaders<BlockBlobStageBlockHeaders> response;

if (async)
{
response = await BlockBlobRestClient.StageBlockAsync(
blockId: base64BlockId,
contentLength: (content?.Length - content?.Position) ?? 0,
contentLength: contentLength,
body: content,
transactionalContentCrc64: hashResult?.StorageCrc64AsArray,
transactionalContentMD5: hashResult?.MD5AsArray,
Expand All @@ -1329,14 +1351,16 @@ internal virtual async Task<Response<BlockInfo>> StageBlockInternal(
encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash,
encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256,
encryptionScope: ClientConfiguration.EncryptionScope,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
else
{
response = BlockBlobRestClient.StageBlock(
blockId: base64BlockId,
contentLength: (content?.Length - content?.Position) ?? 0,
contentLength: contentLength,
body: content,
transactionalContentCrc64: hashResult?.StorageCrc64AsArray,
transactionalContentMD5: hashResult?.MD5AsArray,
Expand All @@ -1345,6 +1369,8 @@ internal virtual async Task<Response<BlockInfo>> StageBlockInternal(
encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash,
encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256,
encryptionScope: ClientConfiguration.EncryptionScope,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
cancellationToken: cancellationToken);
}

Expand Down
2 changes: 2 additions & 0 deletions sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobAppendBlockHeaders.cs
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 305ee6f

Please sign in to comment.