From 0feaa8510e829aa9b469837d87a766d0987fd63f Mon Sep 17 00:00:00 2001 From: Sean McCullough Date: Wed, 9 Sep 2020 19:44:23 -0500 Subject: [PATCH 1/2] Added Close and RetainUncommitedData to DataLakeFileUploadOptions --- ...e.Storage.Files.DataLake.netstandard2.0.cs | 2 + .../src/DataLakeFileClient.cs | 12 +- .../src/Models/DataLakeFileUploadOptions.cs | 20 ++ .../tests/FileClientTests.cs | 25 +++ .../UploadAsync_CloseAndRetainData.json | 210 ++++++++++++++++++ .../UploadAsync_CloseAndRetainDataAsync.json | 210 ++++++++++++++++++ 6 files changed, 473 insertions(+), 6 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/tests/SessionRecords/FileClientTests/UploadAsync_CloseAndRetainData.json create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/tests/SessionRecords/FileClientTests/UploadAsync_CloseAndRetainDataAsync.json diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index 532b4c6343621..3b30a1877b6c4 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -318,11 +318,13 @@ public DataLakeFileOpenWriteOptions() { } public partial class DataLakeFileUploadOptions { public DataLakeFileUploadOptions() { } + public bool? Close { get { throw null; } set { } } public Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions Conditions { get { throw null; } set { } } public Azure.Storage.Files.DataLake.Models.PathHttpHeaders HttpHeaders { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } public string Permissions { get { throw null; } set { } } public System.IProgress ProgressHandler { get { throw null; } set { } } + public bool? RetainUncommittedData { get { throw null; } set { } } public Azure.Storage.StorageTransferOptions TransferOptions { get { throw null; } set { } } public string Umask { get { throw null; } set { } } } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index f8500730b5941..192016047540e 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -2042,7 +2042,7 @@ public virtual async Task> FlushAsync( /// flush operation completes successfully, the service raises a file change notification with a property indicating that /// this is the final update (the file stream has been closed). If "false" a change notification is raised indicating the /// file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS driver to indicate that - /// the file stream has been closed." + /// the file stream has been closed. /// /// /// Optional standard HTTP header properties that can be set for the file. @@ -4150,7 +4150,7 @@ private async Task OpenWriteInternal( } #endregion OpenWrite - #region PartitionedUplaoder + #region PartitionedUploader internal PartitionedUploader GetPartitionedUploader( StorageTransferOptions transferOptions, ArrayPool arrayPool = null, @@ -4196,8 +4196,8 @@ await client.AppendInternal( // Flush data return await client.FlushInternal( position: newPosition, - retainUncommittedData: default, - close: default, + retainUncommittedData: args.RetainUncommittedData, + close: args.Close, args.HttpHeaders, args.Conditions, async, @@ -4223,8 +4223,8 @@ await client.AppendInternal( return await client.FlushInternal( offset + size, - retainUncommittedData: default, - close: default, + retainUncommittedData: args.RetainUncommittedData, + close: args.Close, httpHeaders: args.HttpHeaders, conditions: args.Conditions, async, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileUploadOptions.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileUploadOptions.cs index 7d2307c320a3b..8cab05ec5677b 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileUploadOptions.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileUploadOptions.cs @@ -50,6 +50,26 @@ public class DataLakeFileUploadOptions /// public IProgress ProgressHandler { get; set; } + /// + /// If "true", uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted + /// after the flush operation. The default is false. Data at offsets less than the specified position are written to the + /// file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a future + /// flush operation. + /// + public bool? RetainUncommittedData { get; set; } + + /// + /// Azure Storage Events allow applications to receive notifications when files change. When Azure Storage Events are enabled, + /// a file changed event is raised. This event has a property indicating whether this is the final change to distinguish the + /// difference between an intermediate flush to a file stream and the final close of a file stream. The close query parameter + /// is valid only when the action is "flush" and change notifications are enabled. If the value of close is "true" and the + /// flush operation completes successfully, the service raises a file change notification with a property indicating that + /// this is the final update (the file stream has been closed). If "false" a change notification is raised indicating the + /// file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS driver to indicate that + /// the file stream has been closed. + /// + public bool? Close { get; set; } + /// /// Optional to configure /// parallel transfer behavior. diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs index a351c507fe7bc..72c34b6da35d8 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs @@ -2824,6 +2824,31 @@ await file.UploadAsync( TestHelper.AssertSequenceEqual(expectedData, actual.ToArray()); } + [Test] + public async Task UploadAsync_CloseAndRetainData() + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeFileClient file = test.FileSystem.GetFileClient(GetNewFileName()); + + byte[] data = GetRandomBuffer(Constants.KB); + + DataLakeFileUploadOptions options = new DataLakeFileUploadOptions + { + Close = true, + RetainUncommittedData = true + }; + + // Act + using Stream stream = new MemoryStream(data); + await file.UploadAsync(stream, options: options); + + // Assert + using var actual = new MemoryStream(); + await file.ReadToAsync(actual); + TestHelper.AssertSequenceEqual(data, actual.ToArray()); + } + [Test] [Ignore("Live tests will run out of memory")] public async Task UploadAsync_FileLarge() diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/SessionRecords/FileClientTests/UploadAsync_CloseAndRetainData.json b/sdk/storage/Azure.Storage.Files.DataLake/tests/SessionRecords/FileClientTests/UploadAsync_CloseAndRetainData.json new file mode 100644 index 0000000000000..f4b1fe38acfdc --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/SessionRecords/FileClientTests/UploadAsync_CloseAndRetainData.json @@ -0,0 +1,210 @@ +{ + "Entries": [ + { + "RequestUri": "https://seannsecanary.blob.core.windows.net/test-filesystem-5f201bfb-6364-0ec2-5428-c60b1a409893?restype=container", + "RequestMethod": "PUT", + "RequestHeaders": { + "Authorization": "Sanitized", + "traceparent": "00-373b0b5b7090114bab665f27f84c0671-ba5da75fd3da8344-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-blob-public-access": "container", + "x-ms-client-request-id": "f9633d39-0f65-bd69-c5a3-7bde6e980f24", + "x-ms-date": "Thu, 10 Sep 2020 00:43:18 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 201, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:20 GMT", + "ETag": "\u00220x8D8552283917719\u0022", + "Last-Modified": "Thu, 10 Sep 2020 00:43:19 GMT", + "Server": [ + "Windows-Azure-Blob/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "f9633d39-0f65-bd69-c5a3-7bde6e980f24", + "x-ms-request-id": "273d8469-a01e-0052-2d0b-873148000000", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + }, + { + "RequestUri": "https://seannsecanary.dfs.core.windows.net/test-filesystem-5f201bfb-6364-0ec2-5428-c60b1a409893/test-file-3262640f-b458-48b9-51b8-361557bdde2d?resource=file", + "RequestMethod": "PUT", + "RequestHeaders": { + "Authorization": "Sanitized", + "traceparent": "00-ac6c6a98f932f74488978ac7e2ffa588-a8cde5c73dac094c-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "ba29be66-0f23-2261-e53b-52d41ee064f2", + "x-ms-date": "Thu, 10 Sep 2020 00:43:18 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 201, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:19 GMT", + "ETag": "\u00220x8D855228405DBFD\u0022", + "Last-Modified": "Thu, 10 Sep 2020 00:43:20 GMT", + "Server": [ + "Windows-Azure-HDFS/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "ba29be66-0f23-2261-e53b-52d41ee064f2", + "x-ms-request-id": "a691112f-a01f-0030-0c0b-87f36f000000", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + }, + { + "RequestUri": "https://seannsecanary.dfs.core.windows.net/test-filesystem-5f201bfb-6364-0ec2-5428-c60b1a409893/test-file-3262640f-b458-48b9-51b8-361557bdde2d?action=append\u0026position=0", + "RequestMethod": "PATCH", + "RequestHeaders": { + "Authorization": "Sanitized", + "Content-Length": "1024", + "traceparent": "00-1b67afa1f37e3e4084b8e6aa91e51789-350cab00836a8a40-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "3078a384-512e-0243-3a6b-bc26f75af09b", + "x-ms-date": "Thu, 10 Sep 2020 00:43:19 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": "0EnR3NyeLTxHVYNvNuv5jz/o9svEJSSFhBqorNlUZgjETg4kxzNHcDcl6gMwSRJmv6bVb8cTaKolTFfB9oBj8\u002B97qujc1MgLoIWTnzshVDoIQu0yfvZAseZj/3vbhXq3wvcTtZ8pieveys5Cf8lvA/ccubSTMXT4OvkBedRNLxoK3N1DDp\u002Bb3rgBU7Qo/jEd8/yDv8zGI5CCGnqlKBAWkJq0N6PsLPVYgdr8kd3A0DvI5IXYynBNRLhMwfMk9ban6sMadMeOMccHKj3yyZKywUo/3UOKPsYpZvLqn2oBNIsjVKYaplx0ObO\u002B0Z8yW3ommv\u002Bo5BhzHKz88A3UpJFdBfayz3Xrlxvuy6OQzGwn9gt4wwVqWU9zsDf5HrVOn5UhYMuogIsEOwLkuhl04k8/1fkCXOCM8qymPx8\u002B8XlmoCsOb2uhfnZ/c4gChp42/s3nukm7S0O1lG4fAkBsxkmJfRruZnLjnleDguhXimsNcEBRVjcwqlToWNv9gXw1E6w/eAiER5hbAxvxOMhaXQkCEZEAGnARWdPFBg2X5h3bTW48GBFiiSin9lIcPnKLjfvR\u002Bm/qNfk8LR81v0GLxB3\u002BFupaHkT4n4hoWCQd9RthOXgSxeyt8Bh1AOvyLcrmHDI1HSeHMW622VrmgdzVHjcD04XJef4wUCbS5B5ZBXK\u002BgTzy6Nf02ggKW\u002BuRx247oOGz7llEHaMzBM0BtUxQXNToJ/\u002Bg4zZgD/l8vx0Sn4m3lkQ8phYYeWSZvuMJ8iW7uf4XqZ9um72rUt9fyF6FIGVESR4IE7a9z62bByRSVhzUwAUe\u002B/XpSPqGkVM3uLEFE519xagGsqZIWdnx05SYWPzCiwdZZyVvWgEjgYJihhf8lfSG1ZBzzg5e1aEc93Vl7jhXdS2qrKt4DWqn4jAk5tPC9T2gEoNrTrz5\u002BpKDU2KMvpNJChHpUuhIDsclzOzdOu/OWQI3xdy06yZwsLIZ9boIGySc2tvB2KNUWAO8RYHQHL\u002BjtPpgVPi3lB/qeKE2LxgNHbDHQF8TDIBe423Y\u002BKWkoJTydkC/p8cr5EW812ACHijrryjQCdxgj6h5AXGL35x5RdgNQ7nYTXXpm1sZVzUgXMRE7El905e4kuaXRqZWoAWpQpGH9NtvEmGy522ARjhzWKSsR/JT\u002B3FBVbDSQAbPk/tWcBSjmWfp1Nrny\u002BwqN0yuJO0BqVgSVkcfRp8rndFb4lRxKtYjz789w3IYay\u002BFFx1kJBwYzemiQxnNg1LHvTcIjOiYuAx1ilQuxObm\u002BeoXmi\u002BCx3i1QhRqzua8VktsdQU6EN9H7F4R7pnH6gkDvEoutekjRfa/UkrULuoluDMJwU/grOkL0K3RcNje7lcLZg==", + "StatusCode": 202, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:19 GMT", + "Server": [ + "Windows-Azure-HDFS/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "3078a384-512e-0243-3a6b-bc26f75af09b", + "x-ms-request-id": "a6911130-a01f-0030-0d0b-87f36f000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + }, + { + "RequestUri": "https://seannsecanary.dfs.core.windows.net/test-filesystem-5f201bfb-6364-0ec2-5428-c60b1a409893/test-file-3262640f-b458-48b9-51b8-361557bdde2d?action=flush\u0026position=1024\u0026retainUncommittedData=true\u0026close=true", + "RequestMethod": "PATCH", + "RequestHeaders": { + "Authorization": "Sanitized", + "Content-Length": "0", + "traceparent": "00-70c3b5ee11922143b24b597a010f61d0-5287c4753746214d-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "7a50f54d-3467-77c4-84d5-4ebfd68080c5", + "x-ms-date": "Thu, 10 Sep 2020 00:43:19 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:19 GMT", + "ETag": "\u00220x8D85522841E8BF6\u0022", + "Last-Modified": "Thu, 10 Sep 2020 00:43:20 GMT", + "Server": [ + "Windows-Azure-HDFS/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "7a50f54d-3467-77c4-84d5-4ebfd68080c5", + "x-ms-request-id": "a6911131-a01f-0030-0e0b-87f36f000000", + "x-ms-request-server-encrypted": "false", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + }, + { + "RequestUri": "https://seannsecanary.blob.core.windows.net/test-filesystem-5f201bfb-6364-0ec2-5428-c60b1a409893/test-file-3262640f-b458-48b9-51b8-361557bdde2d", + "RequestMethod": "GET", + "RequestHeaders": { + "Authorization": "Sanitized", + "traceparent": "00-6b91fe17957b6b42bc80beb1081af195-b6d7cbed5e85974e-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "02d934bf-760a-2533-de32-72c7134afd97", + "x-ms-date": "Thu, 10 Sep 2020 00:43:19 GMT", + "x-ms-range": "bytes=0-268435455", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 206, + "ResponseHeaders": { + "Accept-Ranges": "bytes", + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "Content-Type": "application/octet-stream", + "Date": "Thu, 10 Sep 2020 00:43:20 GMT", + "ETag": "\u00220x8D85522841E8BF6\u0022", + "Last-Modified": "Thu, 10 Sep 2020 00:43:20 GMT", + "Server": [ + "Windows-Azure-Blob/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "02d934bf-760a-2533-de32-72c7134afd97", + "x-ms-creation-time": "Thu, 10 Sep 2020 00:43:20 GMT", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-request-id": "273d8513-a01e-0052-2c0b-873148000000", + "x-ms-server-encrypted": "true", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": "0EnR3NyeLTxHVYNvNuv5jz/o9svEJSSFhBqorNlUZgjETg4kxzNHcDcl6gMwSRJmv6bVb8cTaKolTFfB9oBj8\u002B97qujc1MgLoIWTnzshVDoIQu0yfvZAseZj/3vbhXq3wvcTtZ8pieveys5Cf8lvA/ccubSTMXT4OvkBedRNLxoK3N1DDp\u002Bb3rgBU7Qo/jEd8/yDv8zGI5CCGnqlKBAWkJq0N6PsLPVYgdr8kd3A0DvI5IXYynBNRLhMwfMk9ban6sMadMeOMccHKj3yyZKywUo/3UOKPsYpZvLqn2oBNIsjVKYaplx0ObO\u002B0Z8yW3ommv\u002Bo5BhzHKz88A3UpJFdBfayz3Xrlxvuy6OQzGwn9gt4wwVqWU9zsDf5HrVOn5UhYMuogIsEOwLkuhl04k8/1fkCXOCM8qymPx8\u002B8XlmoCsOb2uhfnZ/c4gChp42/s3nukm7S0O1lG4fAkBsxkmJfRruZnLjnleDguhXimsNcEBRVjcwqlToWNv9gXw1E6w/eAiER5hbAxvxOMhaXQkCEZEAGnARWdPFBg2X5h3bTW48GBFiiSin9lIcPnKLjfvR\u002Bm/qNfk8LR81v0GLxB3\u002BFupaHkT4n4hoWCQd9RthOXgSxeyt8Bh1AOvyLcrmHDI1HSeHMW622VrmgdzVHjcD04XJef4wUCbS5B5ZBXK\u002BgTzy6Nf02ggKW\u002BuRx247oOGz7llEHaMzBM0BtUxQXNToJ/\u002Bg4zZgD/l8vx0Sn4m3lkQ8phYYeWSZvuMJ8iW7uf4XqZ9um72rUt9fyF6FIGVESR4IE7a9z62bByRSVhzUwAUe\u002B/XpSPqGkVM3uLEFE519xagGsqZIWdnx05SYWPzCiwdZZyVvWgEjgYJihhf8lfSG1ZBzzg5e1aEc93Vl7jhXdS2qrKt4DWqn4jAk5tPC9T2gEoNrTrz5\u002BpKDU2KMvpNJChHpUuhIDsclzOzdOu/OWQI3xdy06yZwsLIZ9boIGySc2tvB2KNUWAO8RYHQHL\u002BjtPpgVPi3lB/qeKE2LxgNHbDHQF8TDIBe423Y\u002BKWkoJTydkC/p8cr5EW812ACHijrryjQCdxgj6h5AXGL35x5RdgNQ7nYTXXpm1sZVzUgXMRE7El905e4kuaXRqZWoAWpQpGH9NtvEmGy522ARjhzWKSsR/JT\u002B3FBVbDSQAbPk/tWcBSjmWfp1Nrny\u002BwqN0yuJO0BqVgSVkcfRp8rndFb4lRxKtYjz789w3IYay\u002BFFx1kJBwYzemiQxnNg1LHvTcIjOiYuAx1ilQuxObm\u002BeoXmi\u002BCx3i1QhRqzua8VktsdQU6EN9H7F4R7pnH6gkDvEoutekjRfa/UkrULuoluDMJwU/grOkL0K3RcNje7lcLZg==" + }, + { + "RequestUri": "https://seannsecanary.blob.core.windows.net/test-filesystem-5f201bfb-6364-0ec2-5428-c60b1a409893?restype=container", + "RequestMethod": "DELETE", + "RequestHeaders": { + "Authorization": "Sanitized", + "traceparent": "00-062d8411f046ec4c9559bf3a5fc2ef51-db3081cbc7100f46-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "a10049b1-7da6-6aba-62ac-f68084f90e4e", + "x-ms-date": "Thu, 10 Sep 2020 00:43:19 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 202, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:20 GMT", + "Server": [ + "Windows-Azure-Blob/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "a10049b1-7da6-6aba-62ac-f68084f90e4e", + "x-ms-request-id": "273d8525-a01e-0052-3a0b-873148000000", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + } + ], + "Variables": { + "RandomSeed": "244692557", + "Storage_TestConfigHierarchicalNamespace": "NamespaceTenant\nseannsecanary\nU2FuaXRpemVk\nhttps://seannsecanary.blob.core.windows.net\nhttps://seannsecanary.file.core.windows.net\nhttps://seannsecanary.queue.core.windows.net\nhttps://seannsecanary.table.core.windows.net\n\n\n\n\nhttps://seannsecanary-secondary.blob.core.windows.net\nhttps://seannsecanary-secondary.file.core.windows.net\nhttps://seannsecanary-secondary.queue.core.windows.net\nhttps://seannsecanary-secondary.table.core.windows.net\n68390a19-a643-458b-b726-408abf67b4fc\nSanitized\n72f988bf-86f1-41af-91ab-2d7cd011db47\nhttps://login.microsoftonline.com/\nCloud\nBlobEndpoint=https://seannsecanary.blob.core.windows.net/;QueueEndpoint=https://seannsecanary.queue.core.windows.net/;FileEndpoint=https://seannsecanary.file.core.windows.net/;BlobSecondaryEndpoint=https://seannsecanary-secondary.blob.core.windows.net/;QueueSecondaryEndpoint=https://seannsecanary-secondary.queue.core.windows.net/;FileSecondaryEndpoint=https://seannsecanary-secondary.file.core.windows.net/;AccountName=seannsecanary;AccountKey=Sanitized\n" + } +} \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/SessionRecords/FileClientTests/UploadAsync_CloseAndRetainDataAsync.json b/sdk/storage/Azure.Storage.Files.DataLake/tests/SessionRecords/FileClientTests/UploadAsync_CloseAndRetainDataAsync.json new file mode 100644 index 0000000000000..419d1811cf8e8 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/SessionRecords/FileClientTests/UploadAsync_CloseAndRetainDataAsync.json @@ -0,0 +1,210 @@ +{ + "Entries": [ + { + "RequestUri": "https://seannsecanary.blob.core.windows.net/test-filesystem-90cb53a9-2493-ef7a-1818-ac8cd5d65a7f?restype=container", + "RequestMethod": "PUT", + "RequestHeaders": { + "Authorization": "Sanitized", + "traceparent": "00-2b05dcb1629ea14bb60946c4283dfa00-421da9e7002fa44b-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-blob-public-access": "container", + "x-ms-client-request-id": "0a0473e5-f93e-51ff-0a78-51e6d19d40c0", + "x-ms-date": "Thu, 10 Sep 2020 00:43:19 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 201, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:21 GMT", + "ETag": "\u00220x8D85522848A91BC\u0022", + "Last-Modified": "Thu, 10 Sep 2020 00:43:21 GMT", + "Server": [ + "Windows-Azure-Blob/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "0a0473e5-f93e-51ff-0a78-51e6d19d40c0", + "x-ms-request-id": "2d0a73b3-701e-0051-1a0b-87d02c000000", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + }, + { + "RequestUri": "https://seannsecanary.dfs.core.windows.net/test-filesystem-90cb53a9-2493-ef7a-1818-ac8cd5d65a7f/test-file-06eaa4a9-b9fb-7ab0-17c2-84a99a77555a?resource=file", + "RequestMethod": "PUT", + "RequestHeaders": { + "Authorization": "Sanitized", + "traceparent": "00-6a0a5ebc6fd07740aaeedbf68376b6e0-517b83dcc7411d42-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "e83e3cb4-4d86-3bd5-5adc-513e2e48e68b", + "x-ms-date": "Thu, 10 Sep 2020 00:43:20 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 201, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:21 GMT", + "ETag": "\u00220x8D85522851CB390\u0022", + "Last-Modified": "Thu, 10 Sep 2020 00:43:22 GMT", + "Server": [ + "Windows-Azure-HDFS/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "e83e3cb4-4d86-3bd5-5adc-513e2e48e68b", + "x-ms-request-id": "8d65ceb6-101f-001a-3e0b-872c7f000000", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + }, + { + "RequestUri": "https://seannsecanary.dfs.core.windows.net/test-filesystem-90cb53a9-2493-ef7a-1818-ac8cd5d65a7f/test-file-06eaa4a9-b9fb-7ab0-17c2-84a99a77555a?action=append\u0026position=0", + "RequestMethod": "PATCH", + "RequestHeaders": { + "Authorization": "Sanitized", + "Content-Length": "1024", + "traceparent": "00-3f4b96b4afeb4447afc7d1190f197f90-c710ba955c5caf4e-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "3bdfadf8-f875-eaec-efec-06664756a5a2", + "x-ms-date": "Thu, 10 Sep 2020 00:43:21 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": "x6OmYtJPQBK7zEYs3VU4bqrJWPTUSg0rwonzfHVWtpIi9h8t9t7tWWPXSi5zhJzBiRb\u002B0B5ghnyVep9IXMSdZRa0Cuc2C3fM2/\u002BR/ADfuD7tMfJfV\u002BVzwXCil3s\u002BpfyeOuQXvnm1i32eMvUfz4p8tcFhWivRPg4xQEh2lHP7jTYC8sEbyPrEN6Z0wCXwVPHmJuG63ErRzfzW5n4xPvcKoqS\u002BHN3IYO3HnSOEXH3pQexWkG2RQbHrMCTEReF7kvmvah6AMTcTHcb08j4nTXB/7aSS266JeWDz0\u002By4BBoLSwsp1PJfaBC0QnKnnU9yw\u002BO0RVg96bdKJvG9jF1LWhmr6KYz/Xh9IISU3Zg74tsRgA8IBL7CzmJT3q4BoduyaedZ/LWoP8VrNiXTqQsfD0QVqwxOQlZa6BlE3x/JfH6o4XSw4uYmaEFJnc2Z7yVWNM9q7PbPZCaR/hObZGCmobqnK8eXbi1e7kPKBKSJwfnz7apPXo\u002BH2H1NH\u002BPSiwThovsS8cRevsaiDP6yBF\u002BCCDwap3a1V0MqH1JJSk9Ji2M/6WGpds81lS7krVGR1NQdyxemuOONK0u46NJxFXR3WIGacxr\u002B2JFt5BJH0anxmEp\u002BK343otJ/sb4V8I79w4C6bFTQ9mG5H/KFOdPRRVmhx0\u002BH8\u002Bn2P7WuGsJc6OgDbyGRjWRUIKLpxQuMsmj/rmtkdCk\u002Bdc\u002BWwy2eItueXDcKgcqyP7j1TKetni7daY1NrmxW93S/xPlS\u002Bsk3tv3GSmjUgjCeTW\u002BzbyeBkdeguVmfUXNvR2XDSrzQX0TuvZap51PGCeddrmH9o4hMPTMhsUFwmGSKVgMDETdzze0qxLKQ2KqE2q9xQqMRcqTcdg2Jrq4DfiUyH5BPw/iw6eB00t7WvGWIndaR/lXBeOReJpoq6R4\u002BJ0aNXrkaNBe3wVjCnZ/FBrmoARHYcYDPcaas8Zxdhv/CNauYSF6rbmi/2Tz9/9pEmR/hgugMZrXGvu0SbUN6WjtYAGfzLb1b/JOpGJdD0XHLZ1d8fAQXtc/20YpxmAAkRH8BpP5zUWthhYVUY00dgux7Rgbr/d42QEqY/Vsz6mOThNQYQi1yWBbsG/Z\u002Bw1GB2E\u002Be9VL5OEagymO02lMPLpshwd55BUC\u002BCO3w/\u002BLouH34FiUMlPQRmUyNeHfKvn7C3exbXAFzCNMP4O1kCTlkuBLlu8QuApqmECiS7I3I9ElvLSL2jFt8hQ1S/bQGrI8SKb9l5AO5lNpStEzZ4BxFRuyk9NwXbSu2aTW23AXmoGSYf8PNYw9FtplHpNgPoqTxDLfIYrey\u002B7uTrnvNArjUaxPnQGFZeOPcXj5Dz8lSp8aRxM9OHj6s5YQIObSUeLPgcw==", + "StatusCode": 202, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:21 GMT", + "Server": [ + "Windows-Azure-HDFS/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "3bdfadf8-f875-eaec-efec-06664756a5a2", + "x-ms-request-id": "8d65ceb7-101f-001a-3f0b-872c7f000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + }, + { + "RequestUri": "https://seannsecanary.dfs.core.windows.net/test-filesystem-90cb53a9-2493-ef7a-1818-ac8cd5d65a7f/test-file-06eaa4a9-b9fb-7ab0-17c2-84a99a77555a?action=flush\u0026position=1024\u0026retainUncommittedData=true\u0026close=true", + "RequestMethod": "PATCH", + "RequestHeaders": { + "Authorization": "Sanitized", + "Content-Length": "0", + "traceparent": "00-2833c42d5c655f428602714a4745f6a6-10f98f746ac6044d-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "81453a83-aa68-6a16-6fa6-9f935ee6e4e5", + "x-ms-date": "Thu, 10 Sep 2020 00:43:21 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:21 GMT", + "ETag": "\u00220x8D855228535065A\u0022", + "Last-Modified": "Thu, 10 Sep 2020 00:43:22 GMT", + "Server": [ + "Windows-Azure-HDFS/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "81453a83-aa68-6a16-6fa6-9f935ee6e4e5", + "x-ms-request-id": "8d65ceb8-101f-001a-400b-872c7f000000", + "x-ms-request-server-encrypted": "false", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + }, + { + "RequestUri": "https://seannsecanary.blob.core.windows.net/test-filesystem-90cb53a9-2493-ef7a-1818-ac8cd5d65a7f/test-file-06eaa4a9-b9fb-7ab0-17c2-84a99a77555a", + "RequestMethod": "GET", + "RequestHeaders": { + "Authorization": "Sanitized", + "traceparent": "00-15698b713c76d341b92c1f625e7e344b-bc5c71e7c23bfe47-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "0de322e4-2410-7339-d7a2-5fb813f3f107", + "x-ms-date": "Thu, 10 Sep 2020 00:43:21 GMT", + "x-ms-range": "bytes=0-268435455", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 206, + "ResponseHeaders": { + "Accept-Ranges": "bytes", + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + "Content-Type": "application/octet-stream", + "Date": "Thu, 10 Sep 2020 00:43:22 GMT", + "ETag": "\u00220x8D855228535065A\u0022", + "Last-Modified": "Thu, 10 Sep 2020 00:43:22 GMT", + "Server": [ + "Windows-Azure-Blob/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "0de322e4-2410-7339-d7a2-5fb813f3f107", + "x-ms-creation-time": "Thu, 10 Sep 2020 00:43:22 GMT", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-request-id": "2d0a7418-701e-0051-690b-87d02c000000", + "x-ms-server-encrypted": "true", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": "x6OmYtJPQBK7zEYs3VU4bqrJWPTUSg0rwonzfHVWtpIi9h8t9t7tWWPXSi5zhJzBiRb\u002B0B5ghnyVep9IXMSdZRa0Cuc2C3fM2/\u002BR/ADfuD7tMfJfV\u002BVzwXCil3s\u002BpfyeOuQXvnm1i32eMvUfz4p8tcFhWivRPg4xQEh2lHP7jTYC8sEbyPrEN6Z0wCXwVPHmJuG63ErRzfzW5n4xPvcKoqS\u002BHN3IYO3HnSOEXH3pQexWkG2RQbHrMCTEReF7kvmvah6AMTcTHcb08j4nTXB/7aSS266JeWDz0\u002By4BBoLSwsp1PJfaBC0QnKnnU9yw\u002BO0RVg96bdKJvG9jF1LWhmr6KYz/Xh9IISU3Zg74tsRgA8IBL7CzmJT3q4BoduyaedZ/LWoP8VrNiXTqQsfD0QVqwxOQlZa6BlE3x/JfH6o4XSw4uYmaEFJnc2Z7yVWNM9q7PbPZCaR/hObZGCmobqnK8eXbi1e7kPKBKSJwfnz7apPXo\u002BH2H1NH\u002BPSiwThovsS8cRevsaiDP6yBF\u002BCCDwap3a1V0MqH1JJSk9Ji2M/6WGpds81lS7krVGR1NQdyxemuOONK0u46NJxFXR3WIGacxr\u002B2JFt5BJH0anxmEp\u002BK343otJ/sb4V8I79w4C6bFTQ9mG5H/KFOdPRRVmhx0\u002BH8\u002Bn2P7WuGsJc6OgDbyGRjWRUIKLpxQuMsmj/rmtkdCk\u002Bdc\u002BWwy2eItueXDcKgcqyP7j1TKetni7daY1NrmxW93S/xPlS\u002Bsk3tv3GSmjUgjCeTW\u002BzbyeBkdeguVmfUXNvR2XDSrzQX0TuvZap51PGCeddrmH9o4hMPTMhsUFwmGSKVgMDETdzze0qxLKQ2KqE2q9xQqMRcqTcdg2Jrq4DfiUyH5BPw/iw6eB00t7WvGWIndaR/lXBeOReJpoq6R4\u002BJ0aNXrkaNBe3wVjCnZ/FBrmoARHYcYDPcaas8Zxdhv/CNauYSF6rbmi/2Tz9/9pEmR/hgugMZrXGvu0SbUN6WjtYAGfzLb1b/JOpGJdD0XHLZ1d8fAQXtc/20YpxmAAkRH8BpP5zUWthhYVUY00dgux7Rgbr/d42QEqY/Vsz6mOThNQYQi1yWBbsG/Z\u002Bw1GB2E\u002Be9VL5OEagymO02lMPLpshwd55BUC\u002BCO3w/\u002BLouH34FiUMlPQRmUyNeHfKvn7C3exbXAFzCNMP4O1kCTlkuBLlu8QuApqmECiS7I3I9ElvLSL2jFt8hQ1S/bQGrI8SKb9l5AO5lNpStEzZ4BxFRuyk9NwXbSu2aTW23AXmoGSYf8PNYw9FtplHpNgPoqTxDLfIYrey\u002B7uTrnvNArjUaxPnQGFZeOPcXj5Dz8lSp8aRxM9OHj6s5YQIObSUeLPgcw==" + }, + { + "RequestUri": "https://seannsecanary.blob.core.windows.net/test-filesystem-90cb53a9-2493-ef7a-1818-ac8cd5d65a7f?restype=container", + "RequestMethod": "DELETE", + "RequestHeaders": { + "Authorization": "Sanitized", + "traceparent": "00-988b7a34fc02ae499a631c832adcad82-97347ec7ae977d4a-00", + "User-Agent": [ + "azsdk-net-Storage.Files.DataLake/12.5.0-alpha.20200909.1", + "(.NET Core 4.6.29017.01; Microsoft Windows 10.0.18362 )" + ], + "x-ms-client-request-id": "3baf4bd7-d703-b1d4-56a9-6ec629fd3233", + "x-ms-date": "Thu, 10 Sep 2020 00:43:21 GMT", + "x-ms-return-client-request-id": "true", + "x-ms-version": "2019-12-12" + }, + "RequestBody": null, + "StatusCode": 202, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Thu, 10 Sep 2020 00:43:22 GMT", + "Server": [ + "Windows-Azure-Blob/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "x-ms-client-request-id": "3baf4bd7-d703-b1d4-56a9-6ec629fd3233", + "x-ms-request-id": "2d0a7421-701e-0051-710b-87d02c000000", + "x-ms-version": "2019-12-12" + }, + "ResponseBody": [] + } + ], + "Variables": { + "RandomSeed": "724741532", + "Storage_TestConfigHierarchicalNamespace": "NamespaceTenant\nseannsecanary\nU2FuaXRpemVk\nhttps://seannsecanary.blob.core.windows.net\nhttps://seannsecanary.file.core.windows.net\nhttps://seannsecanary.queue.core.windows.net\nhttps://seannsecanary.table.core.windows.net\n\n\n\n\nhttps://seannsecanary-secondary.blob.core.windows.net\nhttps://seannsecanary-secondary.file.core.windows.net\nhttps://seannsecanary-secondary.queue.core.windows.net\nhttps://seannsecanary-secondary.table.core.windows.net\n68390a19-a643-458b-b726-408abf67b4fc\nSanitized\n72f988bf-86f1-41af-91ab-2d7cd011db47\nhttps://login.microsoftonline.com/\nCloud\nBlobEndpoint=https://seannsecanary.blob.core.windows.net/;QueueEndpoint=https://seannsecanary.queue.core.windows.net/;FileEndpoint=https://seannsecanary.file.core.windows.net/;BlobSecondaryEndpoint=https://seannsecanary-secondary.blob.core.windows.net/;QueueSecondaryEndpoint=https://seannsecanary-secondary.queue.core.windows.net/;FileSecondaryEndpoint=https://seannsecanary-secondary.file.core.windows.net/;AccountName=seannsecanary;AccountKey=Sanitized\n" + } +} \ No newline at end of file From 3c7bda343f11a42245de1c97863af134b65b2445 Mon Sep 17 00:00:00 2001 From: Sean McCullough Date: Fri, 11 Sep 2020 11:27:19 -0500 Subject: [PATCH 2/2] Updated Changelog --- sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md b/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md index 7a31ba44b4358..929b610fc42cb 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md @@ -4,6 +4,7 @@ - Fixed bug where Stream returned from DataLakeFileClient.OpenWrite() did not flush while disposing preventing compatibility with using keyword. - Fixed bug where DataLakeFileClient.Upload() could not upload read-only files. - Fixed bug where DataLakeBlobAccessPolicy.StartsOn and .ExpiresOn would cause the process to crash. +- Added Close and RetainUncommitedData to DataLakeFileUploadOptions. ## 12.4.0 (2020-08-31) - Fixed bug where DataLakeFileClient.Upload() would deadlock if the content stream's position was not 0.