Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Storage] OpenWrite - support for metadata, tags and headers #15023

Merged
merged 30 commits into from
Sep 9, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
78abf66
Prepare Storage for release
kasobol-msft May 6, 2020
e7907a4
pr feedback.
kasobol-msft May 6, 2020
58814fc
PR feedback.
kasobol-msft May 6, 2020
1286434
pr feedback.
kasobol-msft May 6, 2020
324afd0
merge upstream/master
kasobol-msft May 15, 2020
b216c89
Merge remote-tracking branch 'upstream/master'
kasobol-msft May 19, 2020
1c9e87d
Merge remote-tracking branch 'upstream/master'
kasobol-msft Jun 2, 2020
712ecda
Merge remote-tracking branch 'upstream/master'
kasobol-msft Jun 4, 2020
ad612b5
merge upstream/master
kasobol-msft Jun 10, 2020
5849bef
master merge
kasobol-msft Jul 7, 2020
1bcba8b
Merge remote-tracking branch 'upstream/master'
kasobol-msft Jul 29, 2020
8a2e048
Merge remote-tracking branch 'upstream/master'
kasobol-msft Aug 3, 2020
1bf033e
Merge remote-tracking branch 'upstream/master'
kasobol-msft Aug 8, 2020
b35274e
Merge remote-tracking branch 'upstream/master'
kasobol-msft Aug 10, 2020
feb0649
Merge remote-tracking branch 'upstream/master'
kasobol-msft Aug 11, 2020
5974722
Merge remote-tracking branch 'upstream/master'
kasobol-msft Aug 11, 2020
9252944
Merge remote-tracking branch 'upstream/master'
kasobol-msft Aug 14, 2020
f90886e
Merge remote-tracking branch 'upstream/master'
kasobol-msft Aug 20, 2020
3b9b259
Merge remote-tracking branch 'upstream/master'
kasobol-msft Aug 21, 2020
f97e9bd
Merge remote-tracking branch 'upstream/master'
kasobol-msft Sep 1, 2020
ae9d658
make open write work with using.
kasobol-msft Sep 1, 2020
4a24b96
move recordings to right place
kasobol-msft Sep 1, 2020
e970ed7
pr feedback.
kasobol-msft Sep 1, 2020
ed559ba
Merge remote-tracking branch 'upstream/master'
kasobol-msft Sep 1, 2020
f477f9a
Merge remote-tracking branch 'upstream/master'
kasobol-msft Sep 3, 2020
70bac18
Merge remote-tracking branch 'upstream/master'
kasobol-msft Sep 3, 2020
342d763
Merge remote-tracking branch 'upstream/master'
kasobol-msft Sep 8, 2020
4c8ee0b
Merge remote-tracking branch 'upstream/master'
kasobol-msft Sep 9, 2020
96b04f0
Ability to specify metadata tags and headers when opening write strea…
kasobol-msft Sep 9, 2020
884d270
include metadata in initial request.
kasobol-msft Sep 9, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -910,8 +910,11 @@ public partial class BlockBlobOpenWriteOptions
{
public BlockBlobOpenWriteOptions() { }
public long? BufferSize { get { throw null; } set { } }
public Azure.Storage.Blobs.Models.BlobHttpHeaders HttpHeaders { get { throw null; } set { } }
public System.Collections.Generic.IDictionary<string, string> Metadata { get { throw null; } set { } }
public Azure.Storage.Blobs.Models.BlobRequestConditions OpenConditions { get { throw null; } set { } }
public System.IProgress<long> ProgressHandler { get { throw null; } set { } }
public System.Collections.Generic.IDictionary<string, string> Tags { get { throw null; } set { } }
}
public partial class BlockInfo
{
Expand Down
11 changes: 7 additions & 4 deletions sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2121,9 +2121,9 @@ private async Task<Stream> OpenWriteInternal(
// Create Block Blob
Response<BlobContentInfo> response = await UploadInternal(
content: new MemoryStream(Array.Empty<byte>()),
blobHttpHeaders: default,
metadata: default,
tags: default,
blobHttpHeaders: options?.HttpHeaders,
metadata: options?.Metadata,
tags: options?.Tags,
conditions: options?.OpenConditions,
accessTier: default,
progressHandler: default,
Expand All @@ -2143,7 +2143,10 @@ private async Task<Stream> OpenWriteInternal(
bufferSize: options?.BufferSize ?? Constants.DefaultBufferSize,
position: position,
conditions: conditions,
progressHandler: options?.ProgressHandler);
progressHandler: options?.ProgressHandler,
blobHttpHeaders: options?.HttpHeaders,
metadata: options?.Metadata,
tags: options?.Tags);
Comment on lines +2146 to +2149
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also do this when creating the empty blob above?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

}
catch (Exception ex)
{
Expand Down
17 changes: 13 additions & 4 deletions sdk/storage/Azure.Storage.Blobs/src/BlockBlobWriteStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,19 @@ internal class BlockBlobWriteStream : StorageWriteStream
private readonly BlockBlobClient _blockBlobClient;
private readonly BlobRequestConditions _conditions;
private readonly List<string> _blockIds;
private readonly BlobHttpHeaders _blobHttpHeaders;
private readonly IDictionary<string, string> _metadata;
private readonly IDictionary<string, string> _tags;

public BlockBlobWriteStream(
BlockBlobClient blockBlobClient,
long bufferSize,
long position,
BlobRequestConditions conditions,
IProgress<long> progressHandler) : base(
IProgress<long> progressHandler,
BlobHttpHeaders blobHttpHeaders,
IDictionary<string, string> metadata,
IDictionary<string, string> tags) : base(
position,
bufferSize,
progressHandler)
Expand All @@ -32,6 +38,9 @@ public BlockBlobWriteStream(
_blockBlobClient = blockBlobClient;
_conditions = conditions ?? new BlobRequestConditions();
_blockIds = new List<string>();
_blobHttpHeaders = blobHttpHeaders;
_metadata = metadata;
_tags = tags;
}

protected override async Task AppendInternal(bool async, CancellationToken cancellationToken)
Expand Down Expand Up @@ -63,9 +72,9 @@ protected override async Task FlushInternal(bool async, CancellationToken cancel

Response<BlobContentInfo> response = await _blockBlobClient.CommitBlockListInternal(
base64BlockIds: _blockIds,
blobHttpHeaders: default,
metadata: default,
tags: default,
blobHttpHeaders: _blobHttpHeaders,
metadata: _metadata,
tags: _tags,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to set these on every Flush call? Or just the first? (And if it's only the very first, could we do it on the initial blob creation?) I don't think it's a huge deal either way, but it might lead to slightly smaller payloads if folks are passing this stream to a library that's constantly flushing.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a good idea. I overlooked empty blob creation part.

Copy link
Contributor Author

@kasobol-msft kasobol-msft Sep 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did an experiment. Last flush wins unfortunatelly (even with "default" values)

conditions: _conditions,
accessTier: default,
async: async,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

using System;
using Azure.Storage.Blobs.Specialized;
using Metadata = System.Collections.Generic.IDictionary<string, string>;
using Tags = System.Collections.Generic.IDictionary<string, string>;

namespace Azure.Storage.Blobs.Models
{
Expand All @@ -27,5 +29,20 @@ public class BlockBlobOpenWriteOptions
/// progress updates about data transfers.
/// </summary>
public IProgress<long> ProgressHandler { get; set; }

/// <summary>
/// Optional standard HTTP header properties that can be set for this block blob.
/// </summary>
public BlobHttpHeaders HttpHeaders { get; set; }

/// <summary>
/// Optional custom metadata to set for this block blob.
/// </summary>
public Metadata Metadata { get; set; }

/// <summary>
/// Options tags to set for this block blob.
/// </summary>
public Tags Tags { get; set; }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ public class CommitBlockListOptions
{
/// <summary>
/// Optional standard HTTP header properties that can be set for the
/// new append blob.
/// new block blob.
/// </summary>
public BlobHttpHeaders HttpHeaders { get; set; }

/// <summary>
/// Optional custom metadata to set for this append blob.
/// Optional custom metadata to set for this block blob.
/// </summary>
public Metadata Metadata { get; set; }

Expand Down
176 changes: 175 additions & 1 deletion sdk/storage/Azure.Storage.Blobs/tests/BlockBlobClientTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class BlockBlobClientTests : BlobTestBase
ex => ex.Status == 500 && ex.ErrorCode == BlobErrorCode.CannotVerifyCopySource.ToString();

public BlockBlobClientTests(bool async, BlobClientOptions.ServiceVersion serviceVersion)
: base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */)
: base(async, serviceVersion, RecordedTestMode.Record /* RecordedTestMode.Record /* to re-record */)
{
}

Expand Down Expand Up @@ -2299,6 +2299,180 @@ public async Task OpenWriteAsync_NewBlob()
TestHelper.AssertSequenceEqual(data, dataResult.ToArray());
}

[Test]
[ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)]
public async Task OpenWriteAsync_NewBlob_WithTags()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
await blob.UploadAsync(new MemoryStream(Array.Empty<byte>()));

Dictionary<string, string> tags = new Dictionary<string, string>() { { "testkey", "testvalue" } };

BlockBlobOpenWriteOptions options = new BlockBlobOpenWriteOptions
{
BufferSize = Constants.KB,
Tags = tags,
};

Stream stream = await blob.OpenWriteAsync(
overwrite: true,
options);

// Act
await stream.FlushAsync();

// Assert
GetBlobTagResult tagsResult = await blob.GetTagsAsync();
CollectionAssert.AreEqual(tags, tagsResult.Tags);
}

[Test]
[ServiceVersion(Min = BlobClientOptions.ServiceVersion.V2019_12_12)]
public async Task OpenWriteAsync_CreateEmptyBlob_WithTags()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
await blob.UploadAsync(new MemoryStream(Array.Empty<byte>()));

Dictionary<string, string> tags = new Dictionary<string, string>() { { "testkey", "testvalue" } };

BlockBlobOpenWriteOptions options = new BlockBlobOpenWriteOptions
{
BufferSize = Constants.KB,
Tags = tags,
};

// Act
Stream stream = await blob.OpenWriteAsync(
overwrite: true,
options);

// Assert
GetBlobTagResult tagsResult = await blob.GetTagsAsync();
CollectionAssert.AreEqual(tags, tagsResult.Tags);
}

[Test]
public async Task OpenWriteAsync_NewBlob_WithMetadata()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
await blob.UploadAsync(new MemoryStream(Array.Empty<byte>()));

Dictionary<string, string> metadata = new Dictionary<string, string>() { { "testkey", "testvalue" } };

BlockBlobOpenWriteOptions options = new BlockBlobOpenWriteOptions
{
BufferSize = Constants.KB,
Metadata = metadata,
};

Stream stream = await blob.OpenWriteAsync(
overwrite: true,
options);

// Act
await stream.FlushAsync();

// Assert
BlobProperties properties = await blob.GetPropertiesAsync();
CollectionAssert.AreEqual(metadata, properties.Metadata);
}

[Test]
public async Task OpenWriteAsync_CreateEmptyBlob_WithMetadata()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
await blob.UploadAsync(new MemoryStream(Array.Empty<byte>()));

Dictionary<string, string> metadata = new Dictionary<string, string>() { { "testkey", "testvalue" } };

BlockBlobOpenWriteOptions options = new BlockBlobOpenWriteOptions
{
BufferSize = Constants.KB,
Metadata = metadata,
};

// Act
Stream stream = await blob.OpenWriteAsync(
overwrite: true,
options);

// Assert
BlobProperties properties = await blob.GetPropertiesAsync();
CollectionAssert.AreEqual(metadata, properties.Metadata);
}

[Test]
public async Task OpenWriteAsync_NewBlob_WithHeaders()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
await blob.UploadAsync(new MemoryStream(Array.Empty<byte>()));

BlobHttpHeaders headers = new BlobHttpHeaders()
{
ContentType = "application/json",
ContentLanguage = "en",
};

BlockBlobOpenWriteOptions options = new BlockBlobOpenWriteOptions
{
BufferSize = Constants.KB,
HttpHeaders = headers,
};

Stream stream = await blob.OpenWriteAsync(
overwrite: true,
options);

// Act
await stream.FlushAsync();

// Assert
BlobProperties properties = await blob.GetPropertiesAsync();
Assert.AreEqual(headers.ContentType, properties.ContentType);
Assert.AreEqual(headers.ContentLanguage, properties.ContentLanguage);
}

[Test]
public async Task OpenWriteAsync_CreateEmptyBlob_WithHeaders()
{
// Arrange
await using DisposingContainer test = await GetTestContainerAsync();
BlockBlobClient blob = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
await blob.UploadAsync(new MemoryStream(Array.Empty<byte>()));

BlobHttpHeaders headers = new BlobHttpHeaders()
{
ContentType = "application/json",
ContentLanguage = "en",
};

BlockBlobOpenWriteOptions options = new BlockBlobOpenWriteOptions
{
BufferSize = Constants.KB,
HttpHeaders = headers,
};

// Act
Stream stream = await blob.OpenWriteAsync(
overwrite: true,
options);

// Assert
BlobProperties properties = await blob.GetPropertiesAsync();
Assert.AreEqual(headers.ContentType, properties.ContentType);
Assert.AreEqual(headers.ContentLanguage, properties.ContentLanguage);
}

[Test]
public async Task OpenWriteAsync_NewBlob_WithUsing()
{
Expand Down
Loading