From 791a5f76d117e6e7f9edae6ae29e55a9dfdfcbfa Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:01:47 -0500 Subject: [PATCH 1/4] Concurrent Append --- .../Azure.Storage.Files.DataLake.net6.0.cs | 28 + .../Azure.Storage.Files.DataLake.net8.0.cs | 28 + ...e.Storage.Files.DataLake.netstandard2.0.cs | 28 + .../Azure.Storage.Files.DataLake/assets.json | 2 +- .../src/DataLakeAppendFileClient.cs | 858 ++++++++++++++++++ .../src/DataLakeDirectoryClient.cs | 14 + .../src/DataLakeExtensions.cs | 13 + .../src/DataLakeFileClient.cs | 26 +- .../src/DataLakeFileSystemClient.cs | 14 + .../src/DataLakePathClient.cs | 213 ++--- .../src/Generated/FileSystemRestClient.cs | 2 +- .../Models/AppendMode.Serialization.cs | 28 + .../Models/BlobType.Serialization.cs | 28 + .../Generated/PathConcurrentAppendHeaders.cs | 29 + .../src/Generated/PathRestClient.cs | 116 ++- .../src/Generated/ServiceRestClient.cs | 2 +- .../src/Models/ConcurrentAppendResult.cs | 16 + .../src/Models/Internal/AppendMode.cs | 14 + .../src/Models/Internal/BlobType.cs | 18 + .../src/autorest.md | 2 +- .../tests/AppendFileClientTests.cs | 671 ++++++++++++++ .../tests/DataLakePartitionedUploaderTests.cs | 20 +- 22 files changed, 2018 insertions(+), 152 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/Generated/Models/AppendMode.Serialization.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/Generated/Models/BlobType.Serialization.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathConcurrentAppendHeaders.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/Models/ConcurrentAppendResult.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/Models/Internal/AppendMode.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/Models/Internal/BlobType.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs index 7a1c3bd97cad..cb515350117a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs @@ -1,5 +1,26 @@ namespace Azure.Storage.Files.DataLake { + public partial class DataLakeAppendFileClient : Azure.Storage.Files.DataLake.DataLakeFileClient + { + protected DataLakeAppendFileClient() { } + public DataLakeAppendFileClient(string connectionString, string fileSystemName, string filePath) { } + public DataLakeAppendFileClient(string connectionString, string fileSystemName, string filePath, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.AzureSasCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.AzureSasCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new System.Threading.Tasks.Task> CreateIfNotExistsAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public new Azure.Storage.Files.DataLake.DataLakeAppendFileClient WithCustomerProvidedKey(Azure.Storage.Files.DataLake.Models.DataLakeCustomerProvidedKey? customerProvidedKey) { throw null; } + } public partial class DataLakeClientOptions : Azure.Core.ClientOptions { public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_05_05) { } @@ -98,6 +119,7 @@ public DataLakeDirectoryClient(System.Uri directoryUri, Azure.Storage.StorageSha public override System.Uri GenerateUserDelegationSasUri(Azure.Storage.Sas.DataLakeSasPermissions permissions, System.DateTimeOffset expiresOn, Azure.Storage.Files.DataLake.Models.UserDelegationKey userDelegationKey, out string stringToSign) { throw null; } public override Azure.Response GetAccessControl(bool? userPrincipalName = default(bool?), Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public override System.Threading.Tasks.Task> GetAccessControlAsync(bool? userPrincipalName = default(bool?), Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Storage.Files.DataLake.DataLakeAppendFileClient GetAppendFileClient(string fileName) { throw null; } public virtual Azure.Storage.Files.DataLake.DataLakeFileClient GetFileClient(string fileName) { throw null; } public virtual Azure.Pageable GetPaths(bool recursive = false, bool userPrincipalName = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.AsyncPageable GetPathsAsync(bool recursive = false, bool userPrincipalName = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } @@ -317,6 +339,7 @@ public DataLakeFileSystemClient(System.Uri fileSystemUri, Azure.Storage.StorageS public virtual System.Uri GenerateUserDelegationSasUri(Azure.Storage.Sas.DataLakeSasBuilder builder, Azure.Storage.Files.DataLake.Models.UserDelegationKey userDelegationKey, out string stringToSign) { throw null; } public virtual Azure.Response GetAccessPolicy(Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual System.Threading.Tasks.Task> GetAccessPolicyAsync(Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Storage.Files.DataLake.DataLakeAppendFileClient GetAppendFileClient(string fileName) { throw null; } public virtual Azure.Pageable GetDeletedPaths(string pathPrefix = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.AsyncPageable GetDeletedPathsAsync(string pathPrefix = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Storage.Files.DataLake.DataLakeDirectoryClient GetDirectoryClient(string directoryName) { throw null; } @@ -546,6 +569,11 @@ public enum AccessControlType Group = 2, Mask = 4, } + public partial class ConcurrentAppendResult + { + public ConcurrentAppendResult() { } + public long CommittedBlockCount { get { throw null; } } + } public enum CopyStatus { Pending = 0, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs index e1cee30d2348..9ed7b8f9be54 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs @@ -1,5 +1,26 @@ namespace Azure.Storage.Files.DataLake { + public partial class DataLakeAppendFileClient : Azure.Storage.Files.DataLake.DataLakeFileClient + { + protected DataLakeAppendFileClient() { } + public DataLakeAppendFileClient(string connectionString, string fileSystemName, string filePath) { } + public DataLakeAppendFileClient(string connectionString, string fileSystemName, string filePath, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.AzureSasCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.AzureSasCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new System.Threading.Tasks.Task> CreateIfNotExistsAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public new Azure.Storage.Files.DataLake.DataLakeAppendFileClient WithCustomerProvidedKey(Azure.Storage.Files.DataLake.Models.DataLakeCustomerProvidedKey? customerProvidedKey) { throw null; } + } public partial class DataLakeClientOptions : Azure.Core.ClientOptions { public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_05_05) { } @@ -98,6 +119,7 @@ public DataLakeDirectoryClient(System.Uri directoryUri, Azure.Storage.StorageSha public override System.Uri GenerateUserDelegationSasUri(Azure.Storage.Sas.DataLakeSasPermissions permissions, System.DateTimeOffset expiresOn, Azure.Storage.Files.DataLake.Models.UserDelegationKey userDelegationKey, out string stringToSign) { throw null; } public override Azure.Response GetAccessControl(bool? userPrincipalName = default(bool?), Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public override System.Threading.Tasks.Task> GetAccessControlAsync(bool? userPrincipalName = default(bool?), Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Storage.Files.DataLake.DataLakeAppendFileClient GetAppendFileClient(string fileName) { throw null; } public virtual Azure.Storage.Files.DataLake.DataLakeFileClient GetFileClient(string fileName) { throw null; } public virtual Azure.Pageable GetPaths(bool recursive = false, bool userPrincipalName = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.AsyncPageable GetPathsAsync(bool recursive = false, bool userPrincipalName = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } @@ -317,6 +339,7 @@ public DataLakeFileSystemClient(System.Uri fileSystemUri, Azure.Storage.StorageS public virtual System.Uri GenerateUserDelegationSasUri(Azure.Storage.Sas.DataLakeSasBuilder builder, Azure.Storage.Files.DataLake.Models.UserDelegationKey userDelegationKey, out string stringToSign) { throw null; } public virtual Azure.Response GetAccessPolicy(Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual System.Threading.Tasks.Task> GetAccessPolicyAsync(Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Storage.Files.DataLake.DataLakeAppendFileClient GetAppendFileClient(string fileName) { throw null; } public virtual Azure.Pageable GetDeletedPaths(string pathPrefix = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.AsyncPageable GetDeletedPathsAsync(string pathPrefix = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Storage.Files.DataLake.DataLakeDirectoryClient GetDirectoryClient(string directoryName) { throw null; } @@ -546,6 +569,11 @@ public enum AccessControlType Group = 2, Mask = 4, } + public partial class ConcurrentAppendResult + { + public ConcurrentAppendResult() { } + public long CommittedBlockCount { get { throw null; } } + } public enum CopyStatus { Pending = 0, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index 7a1c3bd97cad..cb515350117a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -1,5 +1,26 @@ namespace Azure.Storage.Files.DataLake { + public partial class DataLakeAppendFileClient : Azure.Storage.Files.DataLake.DataLakeFileClient + { + protected DataLakeAppendFileClient() { } + public DataLakeAppendFileClient(string connectionString, string fileSystemName, string filePath) { } + public DataLakeAppendFileClient(string connectionString, string fileSystemName, string filePath, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.AzureSasCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.AzureSasCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } + public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual new System.Threading.Tasks.Task> CreateIfNotExistsAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public new Azure.Storage.Files.DataLake.DataLakeAppendFileClient WithCustomerProvidedKey(Azure.Storage.Files.DataLake.Models.DataLakeCustomerProvidedKey? customerProvidedKey) { throw null; } + } public partial class DataLakeClientOptions : Azure.Core.ClientOptions { public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_05_05) { } @@ -98,6 +119,7 @@ public DataLakeDirectoryClient(System.Uri directoryUri, Azure.Storage.StorageSha public override System.Uri GenerateUserDelegationSasUri(Azure.Storage.Sas.DataLakeSasPermissions permissions, System.DateTimeOffset expiresOn, Azure.Storage.Files.DataLake.Models.UserDelegationKey userDelegationKey, out string stringToSign) { throw null; } public override Azure.Response GetAccessControl(bool? userPrincipalName = default(bool?), Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public override System.Threading.Tasks.Task> GetAccessControlAsync(bool? userPrincipalName = default(bool?), Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Storage.Files.DataLake.DataLakeAppendFileClient GetAppendFileClient(string fileName) { throw null; } public virtual Azure.Storage.Files.DataLake.DataLakeFileClient GetFileClient(string fileName) { throw null; } public virtual Azure.Pageable GetPaths(bool recursive = false, bool userPrincipalName = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.AsyncPageable GetPathsAsync(bool recursive = false, bool userPrincipalName = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } @@ -317,6 +339,7 @@ public DataLakeFileSystemClient(System.Uri fileSystemUri, Azure.Storage.StorageS public virtual System.Uri GenerateUserDelegationSasUri(Azure.Storage.Sas.DataLakeSasBuilder builder, Azure.Storage.Files.DataLake.Models.UserDelegationKey userDelegationKey, out string stringToSign) { throw null; } public virtual Azure.Response GetAccessPolicy(Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual System.Threading.Tasks.Task> GetAccessPolicyAsync(Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Storage.Files.DataLake.DataLakeAppendFileClient GetAppendFileClient(string fileName) { throw null; } public virtual Azure.Pageable GetDeletedPaths(string pathPrefix = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.AsyncPageable GetDeletedPathsAsync(string pathPrefix = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Storage.Files.DataLake.DataLakeDirectoryClient GetDirectoryClient(string directoryName) { throw null; } @@ -546,6 +569,11 @@ public enum AccessControlType Group = 2, Mask = 4, } + public partial class ConcurrentAppendResult + { + public ConcurrentAppendResult() { } + public long CommittedBlockCount { get { throw null; } } + } public enum CopyStatus { Pending = 0, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 4a64b8398f65..c133602f4312 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_d74597f1e3" + "Tag": "net/storage/Azure.Storage.Files.DataLake_b3fc23739a" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs new file mode 100644 index 000000000000..aab7d2a59a5b --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs @@ -0,0 +1,858 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.Pipeline; +using Azure.Storage.Files.DataLake.Models; + +namespace Azure.Storage.Files.DataLake +{ + /// + /// The allows you to manipulate Azure Data Lake append blob-based files. + /// + public class DataLakeAppendFileClient : DataLakeFileClient + { + #region ctors + /// + /// Initializes a new instance of the + /// class for mocking. + /// + protected DataLakeAppendFileClient() + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + public DataLakeAppendFileClient(Uri fileUri) + : this(fileUri, (HttpPipelinePolicy)null, null, storageSharedKeyCredential: null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + /// + /// Optional that define the transport + /// pipeline policies for authentication, retries, etc., that are + /// applied to every request. + /// + public DataLakeAppendFileClient(Uri fileUri, DataLakeClientOptions options) + : this(fileUri, (HttpPipelinePolicy)null, options, storageSharedKeyCredential: null) + { + } + + /// + /// Initializes a new instance of the . + /// + /// + /// A connection string includes the authentication information + /// required for your application to access data in an Azure Storage + /// account at runtime. + /// + /// For more information, + /// + /// Configure Azure Storage connection strings + /// + /// + /// The name of the file system containing this path. + /// + /// + /// The path to the file. + /// + public DataLakeAppendFileClient( + string connectionString, + string fileSystemName, + string filePath) + : this(connectionString, fileSystemName, filePath, null) + { + } + + /// + /// Initializes a new instance of the . + /// + /// + /// A connection string includes the authentication information + /// required for your application to access data in an Azure Storage + /// account at runtime. + /// + /// For more information, + /// + /// Configure Azure Storage connection strings + /// + /// + /// The name of the file system containing this path. + /// + /// + /// The path to the file. + /// + /// + /// Optional client options that define the transport pipeline + /// policies for authentication, retries, etc., that are applied to + /// every request. + /// + public DataLakeAppendFileClient( + string connectionString, + string fileSystemName, + string filePath, + DataLakeClientOptions options) + : base(connectionString, fileSystemName, filePath, options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + /// + /// The shared key credential used to sign requests. + /// + public DataLakeAppendFileClient(Uri fileUri, StorageSharedKeyCredential credential) + : this(fileUri, credential.AsPolicy(), null, credential) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + /// + /// The shared key credential used to sign requests. + /// + /// + /// Optional that define the transport + /// pipeline policies for authentication, retries, etc., that are + /// applied to every request. + /// + public DataLakeAppendFileClient(Uri fileUri, StorageSharedKeyCredential credential, DataLakeClientOptions options) + : this(fileUri, credential.AsPolicy(), options, credential) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// Must not contain shared access signature, which should be passed in the second parameter. + /// + /// + /// The shared access signature credential used to sign requests. + /// + /// + /// This constructor should only be used when shared access signature needs to be updated during lifespan of this client. + /// + public DataLakeAppendFileClient(Uri fileUri, AzureSasCredential credential) + : this(fileUri, credential, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// Must not contain shared access signature, which should be passed in the second parameter. + /// + /// + /// The shared access signature credential used to sign requests. + /// + /// + /// Optional that define the transport + /// pipeline policies for authentication, retries, etc., that are + /// applied to every request. + /// + /// + /// This constructor should only be used when shared access signature needs to be updated during lifespan of this client. + /// + public DataLakeAppendFileClient + (Uri fileUri, + AzureSasCredential credential, + DataLakeClientOptions options) + : this(fileUri, credential.AsPolicy(fileUri), options, credential) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + /// + /// The token credential used to sign requests. + /// + public DataLakeAppendFileClient( + Uri fileUri, + TokenCredential credential) + : this(fileUri, credential, new DataLakeClientOptions()) + { + Errors.VerifyHttpsTokenAuth(fileUri); + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + /// + /// The token credential used to sign requests. + /// + /// + /// Optional that define the transport + /// pipeline policies for authentication, retries, etc., that are + /// applied to every request. + /// + public DataLakeAppendFileClient( + Uri fileUri, + TokenCredential credential, + DataLakeClientOptions options) + : this(fileUri, + credential.AsPolicy( + string.IsNullOrEmpty(options?.Audience?.ToString()) ? DataLakeAudience.DefaultAudience.CreateDefaultScope() : options.Audience.Value.CreateDefaultScope(), + options), + options, + credential) + { + Errors.VerifyHttpsTokenAuth(fileUri); + } + + /// + /// Initializes a new instance of the + /// class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + /// + /// An optional authentication policy used to sign requests. + /// + /// + /// Optional client options that define the transport pipeline + /// policies for authentication, retries, etc., that are applied to + /// every request. + /// + /// + /// The shared key credential used to sign requests. + /// + internal DataLakeAppendFileClient( + Uri fileUri, + HttpPipelinePolicy authentication, + DataLakeClientOptions options, + StorageSharedKeyCredential storageSharedKeyCredential) + : base( + fileUri, + authentication, + options, + storageSharedKeyCredential) + { + } + + /// + /// Initializes a new instance of the + /// class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + /// + /// An optional authentication policy used to sign requests. + /// + /// + /// Optional client options that define the transport pipeline + /// policies for authentication, retries, etc., that are applied to + /// every request. + /// + /// + /// The shared key credential used to sign requests. + /// + internal DataLakeAppendFileClient( + Uri fileUri, + HttpPipelinePolicy authentication, + DataLakeClientOptions options, + AzureSasCredential sasCredential) + : base(fileUri, + authentication, + options, + sasCredential: sasCredential) + { + } + + /// + /// Initializes a new instance of the + /// class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the + /// file. + /// + /// + /// An optional authentication policy used to sign requests. + /// + /// + /// Optional client options that define the transport pipeline + /// policies for authentication, retries, etc., that are applied to + /// every request. + /// + /// + /// Token credential. + /// + internal DataLakeAppendFileClient( + Uri fileUri, + HttpPipelinePolicy authentication, + DataLakeClientOptions options, + TokenCredential tokenCredential) + : base(fileUri, + authentication, + options, + tokenCredential: tokenCredential) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// A referencing the file that includes the + /// name of the account, the name of the file system, and the path of the file. + /// + /// + /// . + /// + internal DataLakeAppendFileClient( + Uri fileUri, + DataLakeClientConfiguration clientConfiguration) + : base(fileUri, clientConfiguration) + { + } + + internal DataLakeAppendFileClient( + Uri fileSystemUri, + string filePath, + DataLakeClientConfiguration clientConfiguration) + : base( + fileSystemUri, + filePath, + clientConfiguration) + { + } + #endregion + + /// + /// Initializes a new instance of the + /// class with an identical source but the specified + /// . + /// + /// + /// The customer provided key. + /// A new instance. + /// + /// Pass null to remove the customer provide key in the returned . + /// + public new DataLakeAppendFileClient WithCustomerProvidedKey(Models.DataLakeCustomerProvidedKey? customerProvidedKey) + { + DataLakeClientConfiguration newClientConfiguration = DataLakeClientConfiguration.DeepCopy(ClientConfiguration); + newClientConfiguration.CustomerProvidedKey = customerProvidedKey; + return new DataLakeAppendFileClient( + fileUri: Uri, + clientConfiguration: newClientConfiguration); + } + + #region Create + /// + /// The operation creates a file. + /// If the file already exists, it will be overwritten. If you don't intent to overwrite + /// an existing file, consider using the API. + /// + /// For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// newly created file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public new virtual Response Create( + DataLakePathCreateOptions options = default, + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeAppendFileClient)}.{nameof(Create)}"); + + try + { + scope.Start(); + + return base.CreateInternal( + resourceType: PathResourceType.File, + blobType: BlobType.Appendblob, + httpHeaders: options?.HttpHeaders, + metadata: options?.Metadata, + accessOptions: + (Permissions: options?.AccessOptions?.Permissions, + Umask: options?.AccessOptions?.Umask, + Owner: options?.AccessOptions?.Owner, + Group: options?.AccessOptions?.Group, + AccessControlList: options?.AccessOptions?.AccessControlList), + leaseId: options?.LeaseId, + leaseDuration: options?.LeaseDuration, + timeToExpire: options?.ScheduleDeletionOptions?.TimeToExpire, + expiresOn: options?.ScheduleDeletionOptions?.ExpiresOn, + encryptionContext: options?.EncryptionContext, + conditions: options?.Conditions, + async: false, + cancellationToken: cancellationToken) + .EnsureCompleted(); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The operation creates a file. + /// If the file already exists, it will be overwritten. If you don't intent to overwrite + /// an existing file, consider using the API. + /// + /// For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// newly created file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public new virtual async Task> CreateAsync( + DataLakePathCreateOptions options = default, + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeAppendFileClient)}.{nameof(Create)}"); + + try + { + scope.Start(); + + return await base.CreateInternal( + resourceType: PathResourceType.File, + blobType: BlobType.Appendblob, + httpHeaders: options?.HttpHeaders, + metadata: options?.Metadata, + accessOptions: + (Permissions: options?.AccessOptions?.Permissions, + Umask: options?.AccessOptions?.Umask, + Owner: options?.AccessOptions?.Owner, + Group: options?.AccessOptions?.Group, + AccessControlList: options?.AccessOptions?.AccessControlList), + leaseId: options?.LeaseId, + leaseDuration: options?.LeaseDuration, + timeToExpire: options?.ScheduleDeletionOptions?.TimeToExpire, + expiresOn: options?.ScheduleDeletionOptions?.ExpiresOn, + encryptionContext: options?.EncryptionContext, + conditions: options?.Conditions, + async: true, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + #endregion Create + + #region Create If Not Exists + /// + /// The operation creates a file. + /// If the file already exists, it is not changed. + /// + /// For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// newly created file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public new virtual Response CreateIfNotExists( + DataLakePathCreateOptions options = default, + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeAppendFileClient)}.{nameof(CreateIfNotExists)}"); + + try + { + scope.Start(); + + return base.CreateIfNotExistsInternal( + PathResourceType.File, + BlobType.Appendblob, + httpHeaders: options?.HttpHeaders, + metadata: options?.Metadata, + accessOptions: + (Permissions: options?.AccessOptions?.Permissions, + Umask: options?.AccessOptions?.Umask, + Owner: options?.AccessOptions?.Owner, + Group: options?.AccessOptions?.Group, + AccessControlList: options?.AccessOptions?.AccessControlList), + leaseId: options?.LeaseId, + leaseDuration: options?.LeaseDuration, + timeToExpire: options?.ScheduleDeletionOptions?.TimeToExpire, + expiresOn: options?.ScheduleDeletionOptions?.ExpiresOn, + encryptionContext: options?.EncryptionContext, + async: false, + cancellationToken: cancellationToken) + .EnsureCompleted(); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The operation creates a file. + /// If the file already exists, it is not changed. + /// + /// For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// newly created file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public new virtual async Task> CreateIfNotExistsAsync( + DataLakePathCreateOptions options = default, + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeAppendFileClient)}.{nameof(CreateIfNotExists)}"); + + try + { + scope.Start(); + + return await base.CreateIfNotExistsInternal( + PathResourceType.File, + BlobType.Appendblob, + httpHeaders: options?.HttpHeaders, + metadata: options?.Metadata, + accessOptions: + (Permissions: options?.AccessOptions?.Permissions, + Umask: options?.AccessOptions?.Umask, + Owner: options?.AccessOptions?.Owner, + Group: options?.AccessOptions?.Group, + AccessControlList: options?.AccessOptions?.AccessControlList), + leaseId: options?.LeaseId, + leaseDuration: options?.LeaseDuration, + timeToExpire: options?.ScheduleDeletionOptions?.TimeToExpire, + expiresOn: options?.ScheduleDeletionOptions?.ExpiresOn, + encryptionContext: options?.EncryptionContext, + async: true, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + #endregion Create If Not Exists + + #region Append + /// + /// The operation uploads data to be appended to a file. + /// Data can only be appended to a file. + /// + /// For more information, see + /// + /// Update Path. + /// + /// + /// A containing the content to upload. + /// + /// + /// Specifies if the file should be created if it doesn't already exist. + /// + /// + /// This hash is used to verify the integrity of the request content during transport. When this header is specified, + /// the storage service compares the hash of the content that has arrived with this header value. If the two hashes do not match, + /// the operation will fail with error code 400 (Bad Request). Note that this MD5 hash is not stored with the file. This header is + /// associated with the request content, and not with the stored content of the file itself. + /// + /// + /// Optional to provide + /// progress updates about data transfers. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the state + /// of the updated file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual Response Append( + Stream content, + bool createIfNotExists = default, + byte[] contentHash = default, + IProgress progressHandler = default, + CancellationToken cancellationToken = default) + => AppendInternal( + content, + createIfNotExists, + contentHash, + progressHandler, + async: false, + cancellationToken) + .EnsureCompleted(); + + /// + /// The operation uploads data to be appended to a file. Data can only be appended to a file. + /// + /// For more information, see + /// + /// Update Path. + /// + /// + /// A containing the content to upload. + /// + /// + /// Specifies if the file should be created if it doesn't already exist. + /// + /// + /// This hash is used to verify the integrity of the request content during transport. When this header is specified, + /// the storage service compares the hash of the content that has arrived with this header value. If the two hashes do not match, + /// the operation will fail with error code 400 (Bad Request). Note that this MD5 hash is not stored with the file. This header is + /// associated with the request content, and not with the stored content of the file itself. + /// + /// + /// Optional to provide + /// progress updates about data transfers. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the state + /// of the updated file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual async Task> AppendAsync( + Stream content, + bool createIfNotExists = default, + byte[] contentHash = default, + IProgress progressHandler = default, + CancellationToken cancellationToken = default) + => await AppendInternal( + content, + createIfNotExists, + contentHash, + progressHandler, + async: true, + cancellationToken) + .ConfigureAwait(false); + + /// + /// The operation uploads data to be appended to a file. Data can only be appended to a file. + /// To apply perviously uploaded data to a file, call Flush Data. + /// + /// For more information, see + /// + /// Update Path. + /// + /// + /// A containing the content to upload. + /// + /// + /// Specifies if the file should be created if it doesn't already exist. + /// + /// + /// This hash is used to verify the integrity of the request content during transport. When this header is specified, + /// the storage service compares the hash of the content that has arrived with this header value. If the two hashes do not match, + /// the operation will fail with error code 400 (Bad Request). Note that this MD5 hash is not stored with the file. This header is + /// associated with the request content, and not with the stored content of the file itself. + /// + /// + /// Optional to provide + /// progress updates about data transfers. + /// + /// + /// Whether to invoke the operation asynchronously. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the state + /// of the updated file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + internal virtual async Task> AppendInternal( + Stream content, + bool createIfNotExists, + byte[] contentHash, + IProgress progressHandler, + bool async, + CancellationToken cancellationToken) + { + using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(DataLakeFileClient))) + { + content = content?.WithNoDispose().WithProgress(progressHandler); + ClientConfiguration.Pipeline.LogMethodEnter( + nameof(DataLakeFileClient), + message: + $"{nameof(Uri)}: {Uri}"); + + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeAppendFileClient)}.{nameof(Append)}"); + try + { + scope.Start(); + Errors.VerifyStreamPosition(content, nameof(content)); + ResponseWithHeaders response; + + if (async) + { + response = await PathRestClient.ConcurrentAppendAsync( + body: content, + appendMode: createIfNotExists ? AppendMode.AutoCreate : null, + timeout: null, + contentLength: content?.Length - content?.Position ?? 0, + transactionalContentHash: contentHash, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + else + { + response = PathRestClient.ConcurrentAppend( + body: content, + appendMode: createIfNotExists ? AppendMode.AutoCreate : null, + timeout: null, + contentLength: content?.Length - content?.Position ?? 0, + transactionalContentHash: contentHash, + cancellationToken: cancellationToken); + } + + return Response.FromValue( + response.ToConcurrentAppendResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + ClientConfiguration.Pipeline.LogException(ex); + scope.Failed(ex); + throw; + } + finally + { + ClientConfiguration.Pipeline.LogMethodExit(nameof(DataLakeFileClient)); + scope.Dispose(); + } + } + } + #endregion Append + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeDirectoryClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeDirectoryClient.cs index 7b2ba5c7cee3..5dc63999668c 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeDirectoryClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeDirectoryClient.cs @@ -424,6 +424,20 @@ public virtual DataLakeFileClient GetFileClient(string fileName) $"{Path}/{fileName}", ClientConfiguration); + /// + /// Creates a new object by appending + /// to the end of . The + /// new uses the same request policy + /// pipeline as the . + /// + /// The name of the file. + /// A new instance. + public virtual DataLakeAppendFileClient GetAppendFileClient(string fileName) + => new DataLakeAppendFileClient( + Uri, + $"{Path}/{fileName}", + ClientConfiguration); + /// /// Creates a new object by appending /// to the end of . diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs index 77f5257c5692..1aa27255071b 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs @@ -990,6 +990,19 @@ internal static BlobContainerEncryptionScopeOptions ToBlobContainerEncryptionSco return new Blobs.Models.CustomerProvidedKey(dataLakeCustomerProvidedKey.Value.EncryptionKey); } + internal static ConcurrentAppendResult ToConcurrentAppendResult(this ResponseWithHeaders response) + { + if (response == null) + { + return null; + } + + return new ConcurrentAppendResult + { + CommittedBlockCount = response.Headers.CommittedBlockCount.GetValueOrDefault() + }; + } + #region ValidateConditionsNotPresent internal static void ValidateConditionsNotPresent( this RequestConditions requestConditions, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index e755faff2f5a..076e1f9e574d 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -5785,13 +5785,10 @@ private async Task OpenWriteInternal( { Response createResponse = await CreateInternal( resourceType: PathResourceType.File, + blobType: null, httpHeaders: default, metadata: default, - permissions: default, - umask: default, - owner: default, - group: default, - accessControlList: default, + accessOptions: default, leaseId: default, leaseDuration: default, timeToExpire: default, @@ -5832,13 +5829,10 @@ private async Task OpenWriteInternal( { Response createResponse = await CreateInternal( resourceType: PathResourceType.File, + blobType: default, httpHeaders: default, metadata: default, - permissions: default, - umask: default, - owner: default, - group: default, - accessControlList: default, + accessOptions: default, leaseId: default, leaseDuration: default, timeToExpire: default, @@ -5901,13 +5895,15 @@ internal static PartitionedUploader.Behavio InitializeDestination = async (args, async, cancellationToken) => await client.CreateInternal( resourceType: PathResourceType.File, + blobType: null, httpHeaders: args.HttpHeaders, metadata: args.Metadata, - permissions: args.Permissions, - umask: args.Umask, - owner: default, - group: default, - accessControlList: default, + accessOptions: + (Permissions: args.Permissions, + Umask: args.Umask, + Owner: default, + Group: default, + AccessControlList: default), leaseId: default, leaseDuration: default, timeToExpire: default, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileSystemClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileSystemClient.cs index 5a5e9ca8650c..d82292c682a2 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileSystemClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileSystemClient.cs @@ -586,6 +586,20 @@ public virtual DataLakeFileClient GetFileClient(string fileName) fileName, ClientConfiguration); + /// + /// Create a new object by appending + /// to the end of . The + /// new uses the same request policy + /// pipeline as the . + /// + /// The name of the directory. + /// A new instance. + public virtual DataLakeAppendFileClient GetAppendFileClient(string fileName) + => new DataLakeAppendFileClient( + Uri, + fileName, + ClientConfiguration); + /// /// Sets the various name fields if they are currently null. /// diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakePathClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakePathClient.cs index cd7d7e0ec75d..5869b39051ab 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakePathClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakePathClient.cs @@ -804,13 +804,15 @@ public virtual Response Create( { return CreateInternal( resourceType: resourceType, + blobType: null, httpHeaders: options?.HttpHeaders, metadata: options?.Metadata, - permissions: options?.AccessOptions?.Permissions, - umask: options?.AccessOptions?.Umask, - owner: options?.AccessOptions?.Owner, - group: options?.AccessOptions?.Group, - accessControlList: options?.AccessOptions?.AccessControlList, + accessOptions: + (Permissions: options?.AccessOptions?.Permissions, + Umask: options?.AccessOptions?.Umask, + Owner: options?.AccessOptions?.Owner, + Group: options?.AccessOptions?.Group, + AccessControlList: options?.AccessOptions?.AccessControlList), leaseId: options?.LeaseId, leaseDuration: options?.LeaseDuration, timeToExpire: options?.ScheduleDeletionOptions?.TimeToExpire, @@ -854,13 +856,15 @@ public virtual async Task> CreateAsync( { return await CreateInternal( resourceType: resourceType, + blobType: null, httpHeaders: options?.HttpHeaders, metadata: options?.Metadata, - permissions: options?.AccessOptions?.Permissions, - umask: options?.AccessOptions?.Umask, - owner: options?.AccessOptions?.Owner, - group: options?.AccessOptions?.Group, - accessControlList: options?.AccessOptions?.AccessControlList, + accessOptions: + (Permissions: options?.AccessOptions?.Permissions, + Umask: options?.AccessOptions?.Umask, + Owner: options?.AccessOptions?.Owner, + Group: options?.AccessOptions?.Group, + AccessControlList: options?.AccessOptions?.AccessControlList), leaseId: options?.LeaseId, leaseDuration: options?.LeaseDuration, timeToExpire: options?.ScheduleDeletionOptions?.TimeToExpire, @@ -935,13 +939,15 @@ public virtual Response Create( { return CreateInternal( resourceType: resourceType, + blobType: null, httpHeaders: httpHeaders, metadata: metadata, - permissions: permissions, - umask: umask, - owner: null, - group: null, - accessControlList: null, + accessOptions: + (Permissions: permissions, + Umask: umask, + Owner: null, + Group: null, + AccessControlList: null), leaseId: null, leaseDuration: null, timeToExpire: null, @@ -1016,13 +1022,15 @@ public virtual async Task> CreateAsync( { return await CreateInternal( resourceType: resourceType, + blobType: null, httpHeaders: httpHeaders, metadata: metadata, - permissions: permissions, - umask: umask, - owner: null, - group: null, - accessControlList: null, + accessOptions: + (Permissions: permissions, + Umask: umask, + Owner: null, + Group: null, + AccessControlList: null), leaseId: null, leaseDuration: null, timeToExpire: null, @@ -1035,7 +1043,7 @@ public virtual async Task> CreateAsync( } /// - /// The + /// The CreateInternal"/> /// operation creates a file or directory. /// /// For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. @@ -1043,6 +1051,9 @@ public virtual async Task> CreateAsync( /// /// Resource type of this path - file or directory. /// + /// + /// Blob type. + /// /// /// Optional standard HTTP header properties that can be set for the /// new file or directory. @@ -1050,29 +1061,8 @@ public virtual async Task> CreateAsync( /// /// Optional custom metadata to set for this file or directory. /// - /// - /// Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access - /// permissions for the file owner, the file owning group, and others. Each class may be granted read, - /// write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit - /// octal notation (e.g. 0766) are supported. - /// - /// - /// Optional and only valid if Hierarchical Namespace is enabled for the account. - /// When creating a file or directory and the parent folder does not have a default ACL, - /// the umask restricts the permissions of the file or directory to be created. The resulting - /// permission is given by p bitwise-and ^u, where p is the permission and u is the umask. For example, - /// if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is - /// 0777 for a directory and 0666 for a file. The default umask is 0027. The umask must be specified - /// in 4-digit octal notation (e.g. 0766). - /// - /// - /// The owner of the file or directory. - /// - /// - /// Optional. The owning group of the file or directory. - /// - /// - /// The POSIX access control list for the file or directory. + /// + /// Access options. /// /// /// Proposed LeaseId. @@ -1112,13 +1102,14 @@ public virtual async Task> CreateAsync( /// internal virtual async Task> CreateInternal( PathResourceType resourceType, + BlobType? blobType, PathHttpHeaders httpHeaders, Metadata metadata, - string permissions, - string umask, - string owner, - string group, - IList accessControlList, + (string Permissions, + string Umask, + string Owner, + string Group, + IList AccessControlList) accessOptions, string leaseId, TimeSpan? leaseDuration, TimeSpan? timeToExpire, @@ -1136,8 +1127,8 @@ internal virtual async Task> CreateInternal( $"{nameof(Uri)}: {Uri}\n" + $"{nameof(httpHeaders)}: {httpHeaders}\n" + $"{nameof(metadata)}: {metadata}\n" + - $"{nameof(permissions)}: {permissions}\n" + - $"{nameof(umask)}: {umask}\n"); + $"{nameof(accessOptions.Permissions)}: {accessOptions.Permissions}\n" + + $"{nameof(accessOptions.Umask)}: {accessOptions.Umask}\n"); DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakePathClient)}.{nameof(Create)}"); @@ -1200,6 +1191,7 @@ internal virtual async Task> CreateInternal( { response = await PathRestClient.CreateAsync( resource: resourceType, + blobType: blobType, cacheControl: httpHeaders?.CacheControl, contentEncoding: httpHeaders?.ContentEncoding, contentLanguage: httpHeaders?.ContentLanguage, @@ -1207,8 +1199,8 @@ internal virtual async Task> CreateInternal( contentType: httpHeaders?.ContentType, leaseId: conditions?.LeaseId, properties: BuildMetadataString(metadata), - permissions: permissions, - umask: umask, + permissions: accessOptions.Permissions, + umask: accessOptions.Umask, ifMatch: conditions?.IfMatch?.ToString(), ifNoneMatch: conditions?.IfNoneMatch?.ToString(), ifModifiedSince: conditions?.IfModifiedSince, @@ -1216,9 +1208,9 @@ internal virtual async Task> CreateInternal( encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, - owner: owner, - group: group, - acl: PathAccessControlExtensions.ToAccessControlListString(accessControlList), + owner: accessOptions.Owner, + group: accessOptions.Group, + acl: PathAccessControlExtensions.ToAccessControlListString(accessOptions.AccessControlList), proposedLeaseId: leaseId, leaseDuration: serviceLeaseDuration, expiryOptions: pathExpiryOptions, @@ -1231,6 +1223,7 @@ internal virtual async Task> CreateInternal( { response = PathRestClient.Create( resource: resourceType, + blobType: blobType, cacheControl: httpHeaders?.CacheControl, contentEncoding: httpHeaders?.ContentEncoding, contentLanguage: httpHeaders?.ContentLanguage, @@ -1238,8 +1231,8 @@ internal virtual async Task> CreateInternal( contentType: httpHeaders?.ContentType, leaseId: conditions?.LeaseId, properties: BuildMetadataString(metadata), - permissions: permissions, - umask: umask, + permissions: accessOptions.Permissions, + umask: accessOptions.Umask, ifMatch: conditions?.IfMatch?.ToString(), ifNoneMatch: conditions?.IfNoneMatch?.ToString(), ifModifiedSince: conditions?.IfModifiedSince, @@ -1247,9 +1240,9 @@ internal virtual async Task> CreateInternal( encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, - owner: owner, - group: group, - acl: PathAccessControlExtensions.ToAccessControlListString(accessControlList), + owner: accessOptions.Owner, + group: accessOptions.Group, + acl: PathAccessControlExtensions.ToAccessControlListString(accessOptions.AccessControlList), proposedLeaseId: leaseId, leaseDuration: serviceLeaseDuration, expiryOptions: pathExpiryOptions, @@ -1308,13 +1301,15 @@ public virtual Response CreateIfNotExists( CancellationToken cancellationToken = default) => CreateIfNotExistsInternal( resourceType: resourceType, + blobType: null, httpHeaders: options?.HttpHeaders, metadata: options?.Metadata, - permissions: options?.AccessOptions?.Permissions, - umask: options?.AccessOptions?.Umask, - owner: options?.AccessOptions?.Owner, - group: options?.AccessOptions?.Group, - accessControlList: options?.AccessOptions?.AccessControlList, + accessOptions: + (Permissions: options?.AccessOptions?.Permissions, + Umask: options?.AccessOptions?.Umask, + Owner: options?.AccessOptions?.Owner, + Group: options?.AccessOptions?.Group, + AccessControlList: options?.AccessOptions?.AccessControlList), leaseId: options?.LeaseId, leaseDuration: options?.LeaseDuration, timeToExpire: options?.ScheduleDeletionOptions?.TimeToExpire, @@ -1354,13 +1349,15 @@ public virtual async Task> CreateIfNotExistsAsync( CancellationToken cancellationToken = default) => await CreateIfNotExistsInternal( resourceType: resourceType, + blobType: null, httpHeaders: options?.HttpHeaders, metadata: options?.Metadata, - permissions: options?.AccessOptions?.Permissions, - umask: options?.AccessOptions?.Umask, - owner: options?.AccessOptions?.Owner, - group: options?.AccessOptions?.Group, - accessControlList: options?.AccessOptions?.AccessControlList, + accessOptions: + (Permissions: options?.AccessOptions?.Permissions, + Umask: options?.AccessOptions?.Umask, + Owner: options?.AccessOptions?.Owner, + Group: options?.AccessOptions?.Group, + AccessControlList: options?.AccessOptions?.AccessControlList), leaseId: options?.LeaseId, leaseDuration: options?.LeaseDuration, timeToExpire: options?.ScheduleDeletionOptions?.TimeToExpire, @@ -1425,13 +1422,15 @@ public virtual Response CreateIfNotExists( CancellationToken cancellationToken) => CreateIfNotExistsInternal( resourceType: resourceType, + blobType: null, httpHeaders: httpHeaders, metadata: metadata, - permissions: permissions, - umask: umask, - owner: null, - group: null, - accessControlList: null, + accessOptions: + (Permissions: permissions, + Umask: umask, + Owner: null, + Group: null, + AccessControlList: null), leaseId: null, leaseDuration: null, timeToExpire: null, @@ -1496,13 +1495,15 @@ public virtual async Task> CreateIfNotExistsAsync( CancellationToken cancellationToken) => await CreateIfNotExistsInternal( resourceType: resourceType, + blobType: null, httpHeaders: httpHeaders, metadata: metadata, - permissions: permissions, - umask: umask, - owner: null, - group: null, - accessControlList: null, + accessOptions: + (Permissions: permissions, + Umask: umask, + Owner: null, + Group: null, + AccessControlList: null), leaseId: null, leaseDuration: null, timeToExpire: null, @@ -1513,7 +1514,7 @@ public virtual async Task> CreateIfNotExistsAsync( .ConfigureAwait(false); /// - /// The + /// The /// operation creates a file or directory. If the file or directory already exists, it is not changed. /// /// For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. @@ -1521,6 +1522,9 @@ public virtual async Task> CreateIfNotExistsAsync( /// /// Resource type of this path - file or directory. /// + /// + /// Blob type. + /// /// /// Optional standard HTTP header properties that can be set for the /// new file or directory. @@ -1528,29 +1532,8 @@ public virtual async Task> CreateIfNotExistsAsync( /// /// Optional custom metadata to set for this file or directory. /// - /// - /// Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access - /// permissions for the file owner, the file owning group, and others. Each class may be granted read, - /// write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit - /// octal notation (e.g. 0766) are supported. - /// - /// - /// Optional and only valid if Hierarchical Namespace is enabled for the account. - /// When creating a file or directory and the parent folder does not have a default ACL, - /// the umask restricts the permissions of the file or directory to be created. The resulting - /// permission is given by p bitwise-and ^u, where p is the permission and u is the umask. For example, - /// if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is - /// 0777 for a directory and 0666 for a file. The default umask is 0027. The umask must be specified - /// in 4-digit octal notation (e.g. 0766). - /// - /// - /// The owner of the file or directory. - /// - /// - /// Optional. The owning group of the file or directory. - /// - /// - /// The POSIX access control list for the file or directory. + /// + /// Access options. /// /// /// Proposed LeaseId. @@ -1584,15 +1567,16 @@ public virtual async Task> CreateIfNotExistsAsync( /// A will be thrown if /// a failure occurs. /// - private async Task> CreateIfNotExistsInternal( + internal async Task> CreateIfNotExistsInternal( PathResourceType resourceType, + BlobType? blobType, PathHttpHeaders httpHeaders, Metadata metadata, - string permissions, - string umask, - string owner, - string group, - IList accessControlList, + (string Permissions, + string Umask, + string Owner, + string Group, + IList AccessControlList) accessOptions, string leaseId, TimeSpan? leaseDuration, TimeSpan? timeToExpire, @@ -1607,13 +1591,10 @@ private async Task> CreateIfNotExistsInternal( DataLakeRequestConditions conditions = new DataLakeRequestConditions { IfNoneMatch = new ETag(Constants.Wildcard) }; response = await CreateInternal( resourceType: resourceType, + blobType: blobType, httpHeaders: httpHeaders, metadata: metadata, - permissions: permissions, - umask: umask, - owner: owner, - group: group, - accessControlList: accessControlList, + accessOptions: accessOptions, leaseId: leaseId, leaseDuration: leaseDuration, timeToExpire: timeToExpire, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/FileSystemRestClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/FileSystemRestClient.cs index 4144d908b754..f470f8c67e2b 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/FileSystemRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/FileSystemRestClient.cs @@ -33,7 +33,7 @@ internal partial class FileSystemRestClient /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. /// The value must be "filesystem" for all filesystem operations. The default value is "filesystem". - /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". + /// Specifies the version of the operation to use for this request. The default value is "2025-05-05". /// , , , or is null. public FileSystemRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string resource, string version) { diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/Models/AppendMode.Serialization.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/Models/AppendMode.Serialization.cs new file mode 100644 index 000000000000..adc0b168f411 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/Models/AppendMode.Serialization.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; + +namespace Azure.Storage.Files.DataLake.Models +{ + internal static partial class AppendModeExtensions + { + public static string ToSerialString(this AppendMode value) => value switch + { + AppendMode.None => "none", + AppendMode.AutoCreate => "autoCreate", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown AppendMode value.") + }; + + public static AppendMode ToAppendMode(this string value) + { + if (StringComparer.OrdinalIgnoreCase.Equals(value, "none")) return AppendMode.None; + if (StringComparer.OrdinalIgnoreCase.Equals(value, "autoCreate")) return AppendMode.AutoCreate; + throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown AppendMode value."); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/Models/BlobType.Serialization.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/Models/BlobType.Serialization.cs new file mode 100644 index 000000000000..fd38a0f6b1cb --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/Models/BlobType.Serialization.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; + +namespace Azure.Storage.Files.DataLake.Models +{ + internal static partial class BlobTypeExtensions + { + public static string ToSerialString(this BlobType value) => value switch + { + BlobType.None => "none", + BlobType.Appendblob => "appendblob", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown BlobType value.") + }; + + public static BlobType ToBlobType(this string value) + { + if (StringComparer.OrdinalIgnoreCase.Equals(value, "none")) return BlobType.None; + if (StringComparer.OrdinalIgnoreCase.Equals(value, "appendblob")) return BlobType.Appendblob; + throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown BlobType value."); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathConcurrentAppendHeaders.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathConcurrentAppendHeaders.cs new file mode 100644 index 000000000000..f3850082b383 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathConcurrentAppendHeaders.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; +using Azure.Core; + +namespace Azure.Storage.Files.DataLake +{ + internal partial class PathConcurrentAppendHeaders + { + private readonly Response _response; + public PathConcurrentAppendHeaders(Response response) + { + _response = response; + } + /// The version of the REST protocol used to process the request. + public string Version => _response.Headers.TryGetValue("x-ms-version", out string value) ? value : null; + /// The number of blocks that have been committed to this blob. + public long? CommittedBlockCount => _response.Headers.TryGetValue("x-ms-committed-block-count", out long? value) ? value : null; + /// Fast Path session data. + public string FastPathSessionData => _response.Headers.TryGetValue("x-ms-fastpath-session-data", out string value) ? value : null; + /// A UTC date/time value generated by the service that indicates the time at which the Fast Path data will expire. + public DateTimeOffset? FastPathSessionDataExpiry => _response.Headers.TryGetValue("x-ms-fastpath-session-expiry-time", out DateTimeOffset? value) ? value : null; + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs index d328c3079de6..9ad093dad82a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs @@ -30,7 +30,7 @@ internal partial class PathRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". + /// Specifies the version of the operation to use for this request. The default value is "2025-05-05". /// The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. /// , , or is null. public PathRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version, int? xMsLeaseDuration = null) @@ -42,7 +42,7 @@ public PathRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline _xMsLeaseDuration = xMsLeaseDuration; } - internal HttpMessage CreateCreateRequest(int? timeout, PathResourceType? resource, string continuation, PathRenameMode? mode, string cacheControl, string contentEncoding, string contentLanguage, string contentDisposition, string contentType, string renameSource, string leaseId, string sourceLeaseId, string properties, string permissions, string umask, string ifMatch, string ifNoneMatch, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string sourceIfMatch, string sourceIfNoneMatch, DateTimeOffset? sourceIfModifiedSince, DateTimeOffset? sourceIfUnmodifiedSince, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string owner, string group, string acl, string proposedLeaseId, long? leaseDuration, PathExpiryOptions? expiryOptions, string expiresOn, string encryptionContext) + internal HttpMessage CreateCreateRequest(int? timeout, PathResourceType? resource, string continuation, BlobType? blobType, PathRenameMode? mode, string cacheControl, string contentEncoding, string contentLanguage, string contentDisposition, string contentType, string renameSource, string leaseId, string sourceLeaseId, string properties, string permissions, string umask, string ifMatch, string ifNoneMatch, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string sourceIfMatch, string sourceIfNoneMatch, DateTimeOffset? sourceIfModifiedSince, DateTimeOffset? sourceIfUnmodifiedSince, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string owner, string group, string acl, string proposedLeaseId, long? leaseDuration, PathExpiryOptions? expiryOptions, string expiresOn, string encryptionContext) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -61,6 +61,10 @@ internal HttpMessage CreateCreateRequest(int? timeout, PathResourceType? resourc { uri.AppendQuery("continuation", continuation, true); } + if (blobType != null) + { + uri.AppendQuery("blobtype", blobType.Value.ToSerialString(), true); + } if (mode != null) { uri.AppendQuery("mode", mode.Value.ToSerialString(), true); @@ -195,6 +199,7 @@ internal HttpMessage CreateCreateRequest(int? timeout, PathResourceType? resourc /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. /// Required only for Create File and Create Directory. The value must be "file" or "directory". /// Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory. + /// Optional. Indicates concurrent append mode. /// Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be "legacy" or "posix", and the default value will be "posix". /// Optional. Sets the blob's cache control. If specified, this property is stored with the blob and returned with a read request. /// Optional. Sets the blob's content encoding. If specified, this property is stored with the blob and returned with a read request. @@ -228,9 +233,9 @@ internal HttpMessage CreateCreateRequest(int? timeout, PathResourceType? resourc /// Specifies the encryption context to set on the file. /// The cancellation token to use. /// Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: "*". - public async Task> CreateAsync(int? timeout = null, PathResourceType? resource = null, string continuation = null, PathRenameMode? mode = null, string cacheControl = null, string contentEncoding = null, string contentLanguage = null, string contentDisposition = null, string contentType = null, string renameSource = null, string leaseId = null, string sourceLeaseId = null, string properties = null, string permissions = null, string umask = null, string ifMatch = null, string ifNoneMatch = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string sourceIfMatch = null, string sourceIfNoneMatch = null, DateTimeOffset? sourceIfModifiedSince = null, DateTimeOffset? sourceIfUnmodifiedSince = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string owner = null, string group = null, string acl = null, string proposedLeaseId = null, long? leaseDuration = null, PathExpiryOptions? expiryOptions = null, string expiresOn = null, string encryptionContext = null, CancellationToken cancellationToken = default) + public async Task> CreateAsync(int? timeout = null, PathResourceType? resource = null, string continuation = null, BlobType? blobType = null, PathRenameMode? mode = null, string cacheControl = null, string contentEncoding = null, string contentLanguage = null, string contentDisposition = null, string contentType = null, string renameSource = null, string leaseId = null, string sourceLeaseId = null, string properties = null, string permissions = null, string umask = null, string ifMatch = null, string ifNoneMatch = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string sourceIfMatch = null, string sourceIfNoneMatch = null, DateTimeOffset? sourceIfModifiedSince = null, DateTimeOffset? sourceIfUnmodifiedSince = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string owner = null, string group = null, string acl = null, string proposedLeaseId = null, long? leaseDuration = null, PathExpiryOptions? expiryOptions = null, string expiresOn = null, string encryptionContext = null, CancellationToken cancellationToken = default) { - using var message = CreateCreateRequest(timeout, resource, continuation, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, contentType, renameSource, leaseId, sourceLeaseId, properties, permissions, umask, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, sourceIfMatch, sourceIfNoneMatch, sourceIfModifiedSince, sourceIfUnmodifiedSince, encryptionKey, encryptionKeySha256, encryptionAlgorithm, owner, group, acl, proposedLeaseId, leaseDuration, expiryOptions, expiresOn, encryptionContext); + using var message = CreateCreateRequest(timeout, resource, continuation, blobType, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, contentType, renameSource, leaseId, sourceLeaseId, properties, permissions, umask, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, sourceIfMatch, sourceIfNoneMatch, sourceIfModifiedSince, sourceIfUnmodifiedSince, encryptionKey, encryptionKeySha256, encryptionAlgorithm, owner, group, acl, proposedLeaseId, leaseDuration, expiryOptions, expiresOn, encryptionContext); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new PathCreateHeaders(message.Response); switch (message.Response.Status) @@ -246,6 +251,7 @@ public async Task> CreateAsync(int? timeo /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. /// Required only for Create File and Create Directory. The value must be "file" or "directory". /// Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory. + /// Optional. Indicates concurrent append mode. /// Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be "legacy" or "posix", and the default value will be "posix". /// Optional. Sets the blob's cache control. If specified, this property is stored with the blob and returned with a read request. /// Optional. Sets the blob's content encoding. If specified, this property is stored with the blob and returned with a read request. @@ -279,9 +285,9 @@ public async Task> CreateAsync(int? timeo /// Specifies the encryption context to set on the file. /// The cancellation token to use. /// Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: "*". - public ResponseWithHeaders Create(int? timeout = null, PathResourceType? resource = null, string continuation = null, PathRenameMode? mode = null, string cacheControl = null, string contentEncoding = null, string contentLanguage = null, string contentDisposition = null, string contentType = null, string renameSource = null, string leaseId = null, string sourceLeaseId = null, string properties = null, string permissions = null, string umask = null, string ifMatch = null, string ifNoneMatch = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string sourceIfMatch = null, string sourceIfNoneMatch = null, DateTimeOffset? sourceIfModifiedSince = null, DateTimeOffset? sourceIfUnmodifiedSince = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string owner = null, string group = null, string acl = null, string proposedLeaseId = null, long? leaseDuration = null, PathExpiryOptions? expiryOptions = null, string expiresOn = null, string encryptionContext = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders Create(int? timeout = null, PathResourceType? resource = null, string continuation = null, BlobType? blobType = null, PathRenameMode? mode = null, string cacheControl = null, string contentEncoding = null, string contentLanguage = null, string contentDisposition = null, string contentType = null, string renameSource = null, string leaseId = null, string sourceLeaseId = null, string properties = null, string permissions = null, string umask = null, string ifMatch = null, string ifNoneMatch = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string sourceIfMatch = null, string sourceIfNoneMatch = null, DateTimeOffset? sourceIfModifiedSince = null, DateTimeOffset? sourceIfUnmodifiedSince = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string owner = null, string group = null, string acl = null, string proposedLeaseId = null, long? leaseDuration = null, PathExpiryOptions? expiryOptions = null, string expiresOn = null, string encryptionContext = null, CancellationToken cancellationToken = default) { - using var message = CreateCreateRequest(timeout, resource, continuation, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, contentType, renameSource, leaseId, sourceLeaseId, properties, permissions, umask, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, sourceIfMatch, sourceIfNoneMatch, sourceIfModifiedSince, sourceIfUnmodifiedSince, encryptionKey, encryptionKeySha256, encryptionAlgorithm, owner, group, acl, proposedLeaseId, leaseDuration, expiryOptions, expiresOn, encryptionContext); + using var message = CreateCreateRequest(timeout, resource, continuation, blobType, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, contentType, renameSource, leaseId, sourceLeaseId, properties, permissions, umask, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, sourceIfMatch, sourceIfNoneMatch, sourceIfModifiedSince, sourceIfUnmodifiedSince, encryptionKey, encryptionKeySha256, encryptionAlgorithm, owner, group, acl, proposedLeaseId, leaseDuration, expiryOptions, expiresOn, encryptionContext); _pipeline.Send(message, cancellationToken); var headers = new PathCreateHeaders(message.Response); switch (message.Response.Status) @@ -1597,5 +1603,103 @@ public ResponseWithHeaders Undelete(int? timeout = null, st throw new RequestFailedException(message.Response); } } + + internal HttpMessage CreateConcurrentAppendRequest(Stream body, int? timeout, AppendMode? appendMode, long? contentLength, byte[] transactionalContentHash, string fastPathSessionData, long? conditionalAppendPosition) + { + var message = _pipeline.CreateMessage(); + var request = message.Request; + request.Method = RequestMethod.Patch; + var uri = new RawRequestUriBuilder(); + uri.AppendRaw(_url, false); + uri.AppendQuery("action", "concurrentAppend", true); + if (timeout != null) + { + uri.AppendQuery("timeout", timeout.Value, true); + } + if (appendMode != null) + { + uri.AppendQuery("appendmode", appendMode.Value.ToSerialString(), true); + } + request.Uri = uri; + request.Headers.Add("x-ms-version", _version); + if (fastPathSessionData != null) + { + request.Headers.Add("x-ms-fastpath-session-data", fastPathSessionData); + } + if (conditionalAppendPosition != null) + { + request.Headers.Add("x-ms-blob-condition-appendpos", conditionalAppendPosition.Value); + } + request.Headers.Add("Accept", "application/json"); + if (contentLength != null) + { + request.Headers.Add("Content-Length", contentLength.Value); + } + if (transactionalContentHash != null) + { + request.Headers.Add("Content-MD5", transactionalContentHash, "D"); + } + request.Headers.Add("Content-Type", "application/octet-stream"); + request.Content = RequestContent.Create(body); + return message; + } + + /// Appends data to the file. + /// Initial data. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. + /// Optional. Indicates concurrent append mode. + /// Required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". Must be the length of the request content in bytes for "Append Data". + /// Specify the transactional md5 for the body, to be validated by the service. + /// The to use. + /// Optional conditional header, used only for the Append Block operation. A number indicating the byte offset to compare. Append Block will succeed only if the append position is equal to this number. If it is not, the request will fail with the AppendPositionConditionNotMet error (HTTP status code 412 - Precondition Failed). + /// The cancellation token to use. + /// is null. + public async Task> ConcurrentAppendAsync(Stream body, int? timeout = null, AppendMode? appendMode = null, long? contentLength = null, byte[] transactionalContentHash = null, string fastPathSessionData = null, long? conditionalAppendPosition = null, CancellationToken cancellationToken = default) + { + if (body == null) + { + throw new ArgumentNullException(nameof(body)); + } + + using var message = CreateConcurrentAppendRequest(body, timeout, appendMode, contentLength, transactionalContentHash, fastPathSessionData, conditionalAppendPosition); + await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); + var headers = new PathConcurrentAppendHeaders(message.Response); + switch (message.Response.Status) + { + case 202: + return ResponseWithHeaders.FromValue(headers, message.Response); + default: + throw new RequestFailedException(message.Response); + } + } + + /// Appends data to the file. + /// Initial data. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations">Setting Timeouts for Blob Service Operations.</a>. + /// Optional. Indicates concurrent append mode. + /// Required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". Must be the length of the request content in bytes for "Append Data". + /// Specify the transactional md5 for the body, to be validated by the service. + /// The to use. + /// Optional conditional header, used only for the Append Block operation. A number indicating the byte offset to compare. Append Block will succeed only if the append position is equal to this number. If it is not, the request will fail with the AppendPositionConditionNotMet error (HTTP status code 412 - Precondition Failed). + /// The cancellation token to use. + /// is null. + public ResponseWithHeaders ConcurrentAppend(Stream body, int? timeout = null, AppendMode? appendMode = null, long? contentLength = null, byte[] transactionalContentHash = null, string fastPathSessionData = null, long? conditionalAppendPosition = null, CancellationToken cancellationToken = default) + { + if (body == null) + { + throw new ArgumentNullException(nameof(body)); + } + + using var message = CreateConcurrentAppendRequest(body, timeout, appendMode, contentLength, transactionalContentHash, fastPathSessionData, conditionalAppendPosition); + _pipeline.Send(message, cancellationToken); + var headers = new PathConcurrentAppendHeaders(message.Response); + switch (message.Response.Status) + { + case 202: + return ResponseWithHeaders.FromValue(headers, message.Response); + default: + throw new RequestFailedException(message.Response); + } + } } } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/ServiceRestClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/ServiceRestClient.cs index b00fa12238f4..b57fd17b550b 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/ServiceRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/ServiceRestClient.cs @@ -28,7 +28,7 @@ internal partial class ServiceRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". + /// Specifies the version of the operation to use for this request. The default value is "2025-05-05". /// , , or is null. public ServiceRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version) { diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/ConcurrentAppendResult.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/ConcurrentAppendResult.cs new file mode 100644 index 000000000000..20da8775d64d --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/ConcurrentAppendResult.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Files.DataLake.Models +{ + /// + /// The result of the Concurrent Append operation. + /// + public class ConcurrentAppendResult + { + /// + /// The number of blocks that have been commited to this Append File. + /// + public long CommittedBlockCount { get; internal set; } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/Internal/AppendMode.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/Internal/AppendMode.cs new file mode 100644 index 000000000000..a3eac46e3561 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/Internal/AppendMode.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Files.DataLake.Models +{ + /// The AppendMode. + internal enum AppendMode + { + /// none. + None, + /// autoCreate. + AutoCreate + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/Internal/BlobType.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/Internal/BlobType.cs new file mode 100644 index 000000000000..a7f4bad27ca8 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/Internal/BlobType.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +namespace Azure.Storage.Files.DataLake.Models +{ + /// The BlobType. + internal enum BlobType + { + /// none. + None, + /// appendblob. + Appendblob + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md index ec9675a014f7..2b8108810e6c 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/a936baeb45003f1d31ce855084b2e54365af78af/specification/storage/data-plane/Azure.Storage.Files.DataLake/stable/2025-01-05/DataLakeStorage.json + - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/e70f2deacb9fff31a320171f89c04083625fed6d/specification/storage/data-plane/Azure.Storage.Files.DataLake/stable/2025-05-05/DataLakeStorage.json generation1-convenience-client: true modelerfour: seal-single-value-enum-by-default: true diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs new file mode 100644 index 000000000000..d61c117f4a9b --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs @@ -0,0 +1,671 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection.Metadata; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Azure.Core.TestFramework; +using Azure.Storage.Files.DataLake.Models; +using Azure.Storage.Test; +using Azure.Storage.Tests.Shared; +using NUnit.Framework; + +namespace Azure.Storage.Files.DataLake.Tests +{ + public class AppendFileClientTests : PathTestBase + { + public AppendFileClientTests(bool async, DataLakeClientOptions.ServiceVersion serviceVersion) + : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) + { + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync(bool createIfNotExists) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + + // Act + Response response; + if (createIfNotExists) + { + response = await file.CreateIfNotExistsAsync(); + } + else + { + response = await file.CreateAsync(); + } + + // Assert + AssertValidStoragePathInfo(response.Value); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_Error() + { + // Arrange + DataLakeServiceClient service = DataLakeClientBuilder.GetServiceClient_Hns(); + DataLakeFileSystemClient fileSystem = InstrumentClient(service.GetFileSystemClient(GetNewFileSystemName())); + DataLakeAppendFileClient file = InstrumentClient(fileSystem.GetAppendFileClient(GetNewFileName())); + + // Act + await TestHelper.AssertExpectedExceptionAsync( + fileSystem.CreateDirectoryAsync(GetNewDirectoryName()), + e => Assert.AreEqual("FilesystemNotFound", e.ErrorCode)); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_HttpHeaders(bool createIfNotExists) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + PathHttpHeaders headers = new PathHttpHeaders + { + ContentType = ContentType, + ContentEncoding = ContentEncoding, + ContentLanguage = ContentLanguage, + ContentDisposition = ContentDisposition, + CacheControl = CacheControl + }; + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + HttpHeaders = headers, + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response response = await file.GetPropertiesAsync(); + Assert.AreEqual(ContentType, response.Value.ContentType); + // TODO service bug + //Assert.AreEqual(ContentEncoding, response.Value.ContentEncoding); + Assert.AreEqual(ContentLanguage, response.Value.ContentLanguage); + Assert.AreEqual(ContentDisposition, response.Value.ContentDisposition); + Assert.AreEqual(CacheControl, response.Value.CacheControl); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_Metadata(bool createIfNotExists) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + // Arrange + IDictionary metadata = BuildMetadata(); + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + Metadata = metadata, + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response getPropertiesResponse = await file.GetPropertiesAsync(); + AssertMetadataEquality(metadata, getPropertiesResponse.Value.Metadata, isDirectory: false); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_PermissionAndUmask(bool createIfNotExists) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + AccessOptions = new DataLakeAccessOptions + { + Permissions = "0777", + Umask = "0057" + } + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response response = await file.GetAccessControlAsync(); + AssertPathPermissionsEquality(PathPermissions.ParseSymbolicPermissions("rwx-w----"), response.Value.Permissions); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_Owner(bool createIfNotExists) + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + string owner = Recording.Random.NewGuid().ToString(); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + AccessOptions = new DataLakeAccessOptions + { + Owner = owner, + } + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response response = await file.GetAccessControlAsync(); + Assert.AreEqual(owner, response.Value.Owner); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_Group(bool createIfNotExists) + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + string group = Recording.Random.NewGuid().ToString(); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + AccessOptions = new DataLakeAccessOptions + { + Group = group, + } + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response response = await file.GetAccessControlAsync(); + Assert.AreEqual(group, response.Value.Group); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_Acl(bool createIfNotExists) + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + AccessOptions = new DataLakeAccessOptions + { + AccessControlList = AccessControlList + } + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response response = await file.GetAccessControlAsync(); + AssertAccessControlListEquality(AccessControlList, response.Value.AccessControlList.ToList()); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_Lease(bool createIfNotExists) + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + string leaseId = Recording.Random.NewGuid().ToString(); + TimeSpan duration = TimeSpan.FromSeconds(15); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + LeaseId = leaseId, + LeaseDuration = duration, + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response propertiesResponse = await file.GetPropertiesAsync(); + Assert.AreEqual(DataLakeLeaseStatus.Locked, propertiesResponse.Value.LeaseStatus); + Assert.AreEqual(DataLakeLeaseState.Leased, propertiesResponse.Value.LeaseState); + Assert.AreEqual(DataLakeLeaseDuration.Fixed, propertiesResponse.Value.LeaseDuration); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [RetryOnException(5, typeof(AssertionException))] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_RelativeExpiry(bool createIfNotExists) + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + ScheduleDeletionOptions = new DataLakePathScheduleDeletionOptions(timeToExpire: new TimeSpan(hours: 1, minutes: 0, seconds: 0)) + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response propertiesResponse = await file.GetPropertiesAsync(); + DateTimeOffset expectedExpiryTime = propertiesResponse.Value.CreatedOn.AddHours(1); + + // The expiry time and creation time can sometimes differ by about a second. + Assert.AreEqual(expectedExpiryTime.Year, propertiesResponse.Value.ExpiresOn.Year); + Assert.AreEqual(expectedExpiryTime.Month, propertiesResponse.Value.ExpiresOn.Month); + Assert.AreEqual(expectedExpiryTime.Day, propertiesResponse.Value.ExpiresOn.Day); + Assert.AreEqual(expectedExpiryTime.Hour, propertiesResponse.Value.ExpiresOn.Hour); + Assert.AreEqual(expectedExpiryTime.Minute, propertiesResponse.Value.ExpiresOn.Minute); + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_AbsoluteExpiry(bool createIfNotExists) + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + ScheduleDeletionOptions = new DataLakePathScheduleDeletionOptions(expiresOn: new DateTimeOffset(2100, 1, 1, 0, 0, 0, 0, TimeSpan.Zero)) + }; + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(options: options); + } + else + { + await file.CreateAsync(options: options); + } + + // Assert + Response propertiesResponse = await file.GetPropertiesAsync(); + Assert.AreEqual(options.ScheduleDeletionOptions.ExpiresOn, propertiesResponse.Value.ExpiresOn); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_Conditions() + { + var garbageLeaseId = GetGarbageLeaseId(); + foreach (AccessConditionParameters parameters in Conditions_Data) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + // Arrange + // This file is intentionally created twice + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + await file.CreateAsync(); + + parameters.Match = await SetupPathMatchCondition(file, parameters.Match); + parameters.LeaseId = await SetupPathLeaseCondition(file, parameters.LeaseId, garbageLeaseId); + + DataLakeRequestConditions conditions = BuildDataLakeRequestConditions( + parameters: parameters, + lease: true); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + Conditions = conditions + }; + + // Act + Response response = await file.CreateAsync(options); + + // Assert + Assert.IsNotNull(response.GetRawResponse().Headers.RequestId); + } + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateAsync_ConditionsFail() + { + var garbageLeaseId = GetGarbageLeaseId(); + foreach (AccessConditionParameters parameters in GetConditionsFail_Data(garbageLeaseId)) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + // Arrange + // This file is intentionally created twice + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + await file.CreateAsync(); + parameters.NoneMatch = await SetupPathMatchCondition(file, parameters.NoneMatch); + DataLakeRequestConditions conditions = BuildDataLakeRequestConditions( + parameters: parameters, + lease: true); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + Conditions = conditions + }; + + // Act + await TestHelper.AssertExpectedExceptionAsync( + file.CreateAsync(options), + e => { }); + } + } + + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_10_02)] + public async Task CreateAsync_CPK(bool createIfNotExists) + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + DataLakeCustomerProvidedKey customerProvidedKey = GetCustomerProvidedKey(); + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName()).WithCustomerProvidedKey(customerProvidedKey)); + + // Act + if (createIfNotExists) + { + await file.CreateIfNotExistsAsync(); + } + else + { + await file.CreateAsync(); + } + + // Assert + Response response = await file.GetPropertiesAsync(); + Assert.IsTrue(response.Value.IsServerEncrypted); + Assert.AreEqual(customerProvidedKey.EncryptionKeyHash, response.Value.EncryptionKeySha256); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2021_04_10)] + public async Task Create_EncryptionContext() + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + + string encryptionContext = "encryptionContext"; + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + EncryptionContext = encryptionContext + }; + + // Act + await file.CreateAsync(options); + + // Assert. We are also going to test GetProperties(), Read(), and GetPaths() with this test. + Response pathPropertiesResponse = await file.GetPropertiesAsync(); + Assert.AreEqual(encryptionContext, pathPropertiesResponse.Value.EncryptionContext); + + Response readResponse = await file.ReadAsync(); + Assert.AreEqual(encryptionContext, readResponse.Value.Properties.EncryptionContext); + + AsyncPageable getPathsResponse = test.FileSystem.GetPathsAsync(recursive: true); + IList paths = await getPathsResponse.ToListAsync(); + + Assert.AreEqual(2, paths.Count); + Assert.AreEqual(encryptionContext, paths[1].EncryptionContext); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateIfNotExistsAsync_NotExists() + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + + // Act + Response response = await file.CreateIfNotExistsAsync(); + + // Assert + Assert.IsNotNull(response.Value.ETag); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateIfNotExistsAsync_Exists() + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + await file.CreateAsync(); + + // Act + Response response = await file.CreateIfNotExistsAsync(); + + // Assert + Assert.IsNull(response); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task CreateIfNotExistsAsync_Error() + { + // Arrange + await using DisposingFileSystem test = await GetNewFileSystem(); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + DataLakeAppendFileClient file = InstrumentClient(directory.GetAppendFileClient(GetNewFileName())); + DataLakeAppendFileClient unauthorizedFile = InstrumentClient(new DataLakeAppendFileClient(file.Uri, GetOptions())); + + // Act + await TestHelper.AssertExpectedExceptionAsync( + unauthorizedFile.CreateIfNotExistsAsync(), + e => Assert.AreEqual("NoAuthenticationInformation", e.ErrorCode)); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_12_06)] + public async Task AppendAsync() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(test.FileSystem.GetAppendFileClient(GetNewFileName())); + await file.CreateAsync(); + var data = GetRandomBuffer(Constants.KB); + using Stream stream = new MemoryStream(data); + + // Act + Response concurrentAppendResponse = await file.AppendAsync(stream); + + // Assert + Assert.AreEqual(1, concurrentAppendResponse.Value.CommittedBlockCount); + Response downloadResponse = await file.ReadAsync(); + MemoryStream actual = new MemoryStream(); + await downloadResponse.Value.Content.CopyToAsync(actual); + TestHelper.AssertSequenceEqual(data, actual.ToArray()); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task AppendAsync_Error() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(test.FileSystem.GetAppendFileClient(GetNewFileName())); + var data = GetRandomBuffer(Constants.KB); + + // Act + using Stream stream = new MemoryStream(data); + await TestHelper.AssertExpectedExceptionAsync( + file.AppendAsync(stream, 0), + e => Assert.AreEqual("PathNotFound", e.ErrorCode)); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task AppendAsync_ContentHash() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(test.FileSystem.GetAppendFileClient(GetNewFileName())); + await file.CreateAsync(); + byte[] data = GetRandomBuffer(Constants.KB); + byte[] contentHash = MD5.Create().ComputeHash(data); + + // Act + Stream stream = new MemoryStream(data); + await file.AppendAsync(stream, contentHash: contentHash); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task AppendAsync_ProgressReporting() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(test.FileSystem.GetAppendFileClient(GetNewFileName())); + await file.CreateAsync(); + var data = GetRandomBuffer(Constants.KB); + TestProgress progress = new TestProgress(); + using Stream stream = new MemoryStream(data); + + // Act + await file.AppendAsync(stream, progressHandler: progress); + + // Assert + Assert.IsFalse(progress.List.Count == 0); + + Assert.AreEqual(Constants.KB, progress.List[progress.List.Count - 1]); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)] + public async Task AppendAsync_CreateIfNotExists() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(test.FileSystem.GetAppendFileClient(GetNewFileName())); + var data = GetRandomBuffer(Constants.KB); + using Stream stream = new MemoryStream(data); + + // Act + await file.AppendAsync(stream, createIfNotExists: true); + + // Assert + Response downloadResponse = await file.ReadAsync(); + MemoryStream actual = new MemoryStream(); + await downloadResponse.Value.Content.CopyToAsync(actual); + TestHelper.AssertSequenceEqual(data, actual.ToArray()); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakePartitionedUploaderTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakePartitionedUploaderTests.cs index 799cfbc12cc6..75bd1ada964f 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakePartitionedUploaderTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakePartitionedUploaderTests.cs @@ -285,13 +285,10 @@ private void SetupInternalStaging(Mock clientMock, AppendSin clientMock.Setup( c => c.CreateInternal( IsAny(), - s_pathHttpHeaders, - default, - s_permissions, - s_umask, - default, default, + s_pathHttpHeaders, default, + IsAny<(string, string, string, string, IList)>(), default, default, default, @@ -300,7 +297,7 @@ private void SetupInternalStaging(Mock clientMock, AppendSin s_conditions, _async, s_cancellationToken - )).Returns, string, string, string, string, IList, string, TimeSpan?, TimeSpan?, DateTimeOffset?, string, DataLakeRequestConditions, bool, CancellationToken>(sink.CreateInternal); + )).Returns, (string, string, string, string, IList), string, TimeSpan?, TimeSpan?, DateTimeOffset?, string, DataLakeRequestConditions, bool, CancellationToken>(sink.CreateInternal); clientMock.Setup( c => c.AppendInternal( @@ -355,13 +352,14 @@ public AppendSink(bool saveBytes = true) public async Task> CreateInternal( PathResourceType type, + BlobType? blobType, PathHttpHeaders httpHeaders, IDictionary metadata, - string permissions, - string umask, - string owner, - string group, - IList accessControlList, + (string Permissions, + string Umask, + string Owner, + string Group, + IList AccessControlList) accessOptions, string leaseId, TimeSpan? leaseDuration, TimeSpan? timeToExpire, From 6bcefc9f5fddbae16de9593549a0271fa9c1fe2e Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Fri, 29 Sep 2023 15:36:46 -0500 Subject: [PATCH 2/4] Fast Path --- .../Azure.Storage.Files.DataLake.net6.0.cs | 6 ++-- .../Azure.Storage.Files.DataLake.net8.0.cs | 6 ++-- ...e.Storage.Files.DataLake.netstandard2.0.cs | 6 ++-- .../Azure.Storage.Files.DataLake/assets.json | 2 +- .../src/DataLakeAppendFileClient.cs | 16 +++++++++ .../src/DataLakeExtensions.cs | 4 ++- .../src/Models/ConcurrentAppendResult.cs | 12 +++++++ .../tests/AppendFileClientTests.cs | 34 ++++++++++++++++++- 8 files changed, 77 insertions(+), 9 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs index cb515350117a..40cb1e8641fa 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs @@ -13,8 +13,8 @@ public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential c public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } - public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } @@ -573,6 +573,8 @@ public partial class ConcurrentAppendResult { public ConcurrentAppendResult() { } public long CommittedBlockCount { get { throw null; } } + public string FastPathSessionData { get { throw null; } } + public System.DateTimeOffset? FastPathSessionDataExpiresOn { get { throw null; } } } public enum CopyStatus { diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs index 9ed7b8f9be54..70834a7bb512 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs @@ -13,8 +13,8 @@ public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential c public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } - public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } @@ -573,6 +573,8 @@ public partial class ConcurrentAppendResult { public ConcurrentAppendResult() { } public long CommittedBlockCount { get { throw null; } } + public string FastPathSessionData { get { throw null; } } + public System.DateTimeOffset? FastPathSessionDataExpiresOn { get { throw null; } } } public enum CopyStatus { diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index cb515350117a..40cb1e8641fa 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -13,8 +13,8 @@ public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential c public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } - public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } @@ -573,6 +573,8 @@ public partial class ConcurrentAppendResult { public ConcurrentAppendResult() { } public long CommittedBlockCount { get { throw null; } } + public string FastPathSessionData { get { throw null; } } + public System.DateTimeOffset? FastPathSessionDataExpiresOn { get { throw null; } } } public enum CopyStatus { diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index c133602f4312..328f14c3df85 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_b3fc23739a" + "Tag": "net/storage/Azure.Storage.Files.DataLake_b496ca5717" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs index aab7d2a59a5b..cf38a375cda6 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs @@ -675,6 +675,9 @@ internal DataLakeAppendFileClient( /// Optional to provide /// progress updates about data transfers. /// + /// + /// Fast path session data. + /// /// /// Optional to propagate /// notifications that the operation should be cancelled. @@ -692,12 +695,14 @@ public virtual Response Append( bool createIfNotExists = default, byte[] contentHash = default, IProgress progressHandler = default, + string fastPathSessionData = default, CancellationToken cancellationToken = default) => AppendInternal( content, createIfNotExists, contentHash, progressHandler, + fastPathSessionData, async: false, cancellationToken) .EnsureCompleted(); @@ -725,6 +730,9 @@ public virtual Response Append( /// Optional to provide /// progress updates about data transfers. /// + /// + /// Fast path session data. + /// /// /// Optional to propagate /// notifications that the operation should be cancelled. @@ -742,12 +750,14 @@ public virtual async Task> AppendAsync( bool createIfNotExists = default, byte[] contentHash = default, IProgress progressHandler = default, + string fastPathSessionData = default, CancellationToken cancellationToken = default) => await AppendInternal( content, createIfNotExists, contentHash, progressHandler, + fastPathSessionData, async: true, cancellationToken) .ConfigureAwait(false); @@ -776,6 +786,9 @@ public virtual async Task> AppendAsync( /// Optional to provide /// progress updates about data transfers. /// + /// + /// Fast path session data. + /// /// /// Whether to invoke the operation asynchronously. /// @@ -796,6 +809,7 @@ internal virtual async Task> AppendInternal( bool createIfNotExists, byte[] contentHash, IProgress progressHandler, + string fastPathSessionData, bool async, CancellationToken cancellationToken) { @@ -822,6 +836,7 @@ internal virtual async Task> AppendInternal( timeout: null, contentLength: content?.Length - content?.Position ?? 0, transactionalContentHash: contentHash, + fastPathSessionData: fastPathSessionData, cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -833,6 +848,7 @@ internal virtual async Task> AppendInternal( timeout: null, contentLength: content?.Length - content?.Position ?? 0, transactionalContentHash: contentHash, + fastPathSessionData: fastPathSessionData, cancellationToken: cancellationToken); } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs index 1aa27255071b..c757b293d3c8 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs @@ -999,7 +999,9 @@ internal static ConcurrentAppendResult ToConcurrentAppendResult(this ResponseWit return new ConcurrentAppendResult { - CommittedBlockCount = response.Headers.CommittedBlockCount.GetValueOrDefault() + CommittedBlockCount = response.Headers.CommittedBlockCount.GetValueOrDefault(), + FastPathSessionData = response.Headers.FastPathSessionData, + FastPathSessionDataExpiresOn = response.Headers.FastPathSessionDataExpiry }; } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/ConcurrentAppendResult.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/ConcurrentAppendResult.cs index 20da8775d64d..9d63318080f4 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/ConcurrentAppendResult.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/ConcurrentAppendResult.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; + namespace Azure.Storage.Files.DataLake.Models { /// @@ -12,5 +14,15 @@ public class ConcurrentAppendResult /// The number of blocks that have been commited to this Append File. /// public long CommittedBlockCount { get; internal set; } + + /// + /// Fast Path session data. + /// + public string FastPathSessionData { get; internal set; } + + /// + /// Fast Path session data expires on. + /// + public DateTimeOffset? FastPathSessionDataExpiresOn { get; internal set; } } } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs index d61c117f4a9b..31df790f6736 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs @@ -484,7 +484,6 @@ public async Task CreateAsync_CPK(bool createIfNotExists) // Assert Response response = await file.GetPropertiesAsync(); Assert.IsTrue(response.Value.IsServerEncrypted); - Assert.AreEqual(customerProvidedKey.EncryptionKeyHash, response.Value.EncryptionKeySha256); } [RecordedTest] @@ -667,5 +666,38 @@ public async Task AppendAsync_CreateIfNotExists() await downloadResponse.Value.Content.CopyToAsync(actual); TestHelper.AssertSequenceEqual(data, actual.ToArray()); } + + [Test] + [PlaybackOnly("This feature is not enabled in prod")] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2021_08_06)] + public async Task AppendAsync_FastPath() + { + DataLakeServiceClient service = GetServiceClient_OAuth(); + await using DisposingFileSystem test = await GetNewFileSystem(service); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(test.FileSystem.GetAppendFileClient(GetNewFileName())); + await file.CreateAsync(); + var data = GetRandomBuffer(Constants.KB); + using Stream stream = new MemoryStream(data); + string fastPathSessionData = "refresh"; + + // Act + Response concurrentAppendResponse = await file.AppendAsync( + content: stream, + fastPathSessionData: fastPathSessionData); + + // Assert + Assert.IsNotNull(concurrentAppendResponse.Value.FastPathSessionData); + Assert.IsNotNull(concurrentAppendResponse.Value.FastPathSessionDataExpiresOn); + + using Stream stream2 = new MemoryStream(data); + concurrentAppendResponse = await file.AppendAsync( + content: stream2, + fastPathSessionData: concurrentAppendResponse.Value.FastPathSessionData); + + Assert.IsNull(concurrentAppendResponse.Value.FastPathSessionData); + Assert.IsNull(concurrentAppendResponse.Value.FastPathSessionDataExpiresOn); + } } } From 3571fc00fc97f33459a32acf272dc3be6e28e038 Mon Sep 17 00:00:00 2001 From: Sean McCullough Date: Wed, 18 Dec 2024 14:41:47 -0600 Subject: [PATCH 3/4] Added support for Concurrent Append x-ms-conditional-appendpos --- .../Azure.Storage.Files.DataLake.net6.0.cs | 4 +-- .../Azure.Storage.Files.DataLake.net8.0.cs | 4 +-- ...e.Storage.Files.DataLake.netstandard2.0.cs | 4 +-- .../Azure.Storage.Files.DataLake/assets.json | 2 +- .../src/DataLakeAppendFileClient.cs | 19 ++++++++++ .../src/Generated/PathRestClient.cs | 2 +- .../src/autorest.md | 2 +- .../tests/AppendFileClientTests.cs | 35 +++++++++++++++++++ 8 files changed, 63 insertions(+), 9 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs index 40cb1e8641fa..9f05c7dfb911 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs @@ -13,8 +13,8 @@ public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential c public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } - public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, long? ifAppendPositionEqual = default(long?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, long? ifAppendPositionEqual = default(long?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs index 70834a7bb512..ecab396a2520 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs @@ -13,8 +13,8 @@ public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential c public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } - public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, long? ifAppendPositionEqual = default(long?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, long? ifAppendPositionEqual = default(long?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index 40cb1e8641fa..9f05c7dfb911 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -13,8 +13,8 @@ public DataLakeAppendFileClient(System.Uri fileUri, Azure.Core.TokenCredential c public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential) { } public DataLakeAppendFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Files.DataLake.DataLakeClientOptions options) { } - public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } - public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response Append(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, long? ifAppendPositionEqual = default(long?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> AppendAsync(System.IO.Stream content, bool createIfNotExists = false, byte[] contentHash = null, System.IProgress progressHandler = null, string fastPathSessionData = null, long? ifAppendPositionEqual = default(long?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response Create(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new System.Threading.Tasks.Task> CreateAsync(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual new Azure.Response CreateIfNotExists(Azure.Storage.Files.DataLake.Models.DataLakePathCreateOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 328f14c3df85..2d6983d1c5bb 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_b496ca5717" + "Tag": "net/storage/Azure.Storage.Files.DataLake_5d1b0e31ef" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs index cf38a375cda6..cbd969a5c4c5 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeAppendFileClient.cs @@ -678,6 +678,10 @@ internal DataLakeAppendFileClient( /// /// Fast path session data. /// + /// + /// IfAppendPositionEqual ensures that the AppendBlock operation + /// succeeds only if the append position is equal to a value. + /// /// /// Optional to propagate /// notifications that the operation should be cancelled. @@ -696,6 +700,7 @@ public virtual Response Append( byte[] contentHash = default, IProgress progressHandler = default, string fastPathSessionData = default, + long? ifAppendPositionEqual = default, CancellationToken cancellationToken = default) => AppendInternal( content, @@ -703,6 +708,7 @@ public virtual Response Append( contentHash, progressHandler, fastPathSessionData, + ifAppendPositionEqual, async: false, cancellationToken) .EnsureCompleted(); @@ -733,6 +739,10 @@ public virtual Response Append( /// /// Fast path session data. /// + /// + /// IfAppendPositionEqual ensures that the AppendBlock operation + /// succeeds only if the append position is equal to a value. + /// /// /// Optional to propagate /// notifications that the operation should be cancelled. @@ -751,6 +761,7 @@ public virtual async Task> AppendAsync( byte[] contentHash = default, IProgress progressHandler = default, string fastPathSessionData = default, + long? ifAppendPositionEqual = default, CancellationToken cancellationToken = default) => await AppendInternal( content, @@ -758,6 +769,7 @@ public virtual async Task> AppendAsync( contentHash, progressHandler, fastPathSessionData, + ifAppendPositionEqual, async: true, cancellationToken) .ConfigureAwait(false); @@ -789,6 +801,10 @@ public virtual async Task> AppendAsync( /// /// Fast path session data. /// + /// + /// IfAppendPositionEqual ensures that the AppendBlock operation + /// succeeds only if the append position is equal to a value. + /// /// /// Whether to invoke the operation asynchronously. /// @@ -810,6 +826,7 @@ internal virtual async Task> AppendInternal( byte[] contentHash, IProgress progressHandler, string fastPathSessionData, + long? ifAppendPositionEqual, bool async, CancellationToken cancellationToken) { @@ -837,6 +854,7 @@ internal virtual async Task> AppendInternal( contentLength: content?.Length - content?.Position ?? 0, transactionalContentHash: contentHash, fastPathSessionData: fastPathSessionData, + conditionalAppendPosition: ifAppendPositionEqual, cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -849,6 +867,7 @@ internal virtual async Task> AppendInternal( contentLength: content?.Length - content?.Position ?? 0, transactionalContentHash: contentHash, fastPathSessionData: fastPathSessionData, + conditionalAppendPosition: ifAppendPositionEqual, cancellationToken: cancellationToken); } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs index 9ad093dad82a..7b8a8b577f99 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs @@ -1628,7 +1628,7 @@ internal HttpMessage CreateConcurrentAppendRequest(Stream body, int? timeout, Ap } if (conditionalAppendPosition != null) { - request.Headers.Add("x-ms-blob-condition-appendpos", conditionalAppendPosition.Value); + request.Headers.Add("x-ms-conditional-appendpos", conditionalAppendPosition.Value); } request.Headers.Add("Accept", "application/json"); if (contentLength != null) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md index 2b8108810e6c..3148cfe5522a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/e70f2deacb9fff31a320171f89c04083625fed6d/specification/storage/data-plane/Azure.Storage.Files.DataLake/stable/2025-05-05/DataLakeStorage.json + - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/130071d5927b1dbda12ee3d2b85ee9cacddf1eea/specification/storage/data-plane/Azure.Storage.Files.DataLake/stable/2025-05-05/DataLakeStorage.json generation1-convenience-client: true modelerfour: seal-single-value-enum-by-default: true diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs index 31df790f6736..41253dfa06d8 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs @@ -699,5 +699,40 @@ public async Task AppendAsync_FastPath() Assert.IsNull(concurrentAppendResponse.Value.FastPathSessionData); Assert.IsNull(concurrentAppendResponse.Value.FastPathSessionDataExpiresOn); } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_12_06)] + [TestCase(true)] + [TestCase(false)] + public async Task AppendAsync_IfAppendPositionEqual(bool appendPositionEqual) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + DataLakeAppendFileClient file = InstrumentClient(test.FileSystem.GetAppendFileClient(GetNewFileName())); + await file.CreateAsync(); + var data = GetRandomBuffer(Constants.KB); + using Stream stream = new MemoryStream(data); + + long? appendPosition = 0; + + if (!appendPositionEqual) + { + appendPosition = 1; + } + + // Act + if (appendPositionEqual) + { + Response concurrentAppendResponse = await file.AppendAsync(stream, ifAppendPositionEqual: appendPosition); + Assert.AreEqual(1, concurrentAppendResponse.Value.CommittedBlockCount); + } + else + { + await TestHelper.AssertExpectedExceptionAsync( + file.AppendAsync(stream, ifAppendPositionEqual: appendPosition), + e => Assert.AreEqual("InvalidAppendPosition", e.ErrorCode)); + } + } } } From 0812b3c6749aad381c54ce4bbc3300853fecc09b Mon Sep 17 00:00:00 2001 From: Sean McCullough Date: Wed, 18 Dec 2024 14:58:25 -0600 Subject: [PATCH 4/4] Fixed CI --- .../Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs index 41253dfa06d8..aead492178d6 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/AppendFileClientTests.cs @@ -668,7 +668,7 @@ public async Task AppendAsync_CreateIfNotExists() } [Test] - [PlaybackOnly("This feature is not enabled in prod")] + [Ignore("This feature is not enabled in prod")] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2021_08_06)] public async Task AppendAsync_FastPath() {