From 33c69536b4677a7ffa16497b173993d9283b477d Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Tue, 8 Oct 2024 20:12:32 -0500 Subject: [PATCH 01/10] Added STG 97 x-ms-version (#46487) --- .../tests/ChangeFeedTestBase.cs | 1 + .../Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs | 1 + .../api/Azure.Storage.Blobs.netstandard2.0.cs | 1 + .../api/Azure.Storage.Blobs.netstandard2.1.cs | 1 + sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs | 7 ++++++- .../tests/BlobsClientTestFixtureAttribute.cs | 1 + .../api/Azure.Storage.Common.net6.0.cs | 2 +- .../api/Azure.Storage.Common.netstandard2.0.cs | 2 +- sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs | 2 +- .../src/Shared/StorageVersionExtensions.cs | 5 ++++- sdk/storage/Azure.Storage.Common/tests/CommonTestBase.cs | 5 +++-- .../api/Azure.Storage.Files.DataLake.net6.0.cs | 1 + .../api/Azure.Storage.Files.DataLake.netstandard2.0.cs | 1 + .../src/DataLakeClientOptions.cs | 7 ++++++- .../tests/DataLakeClientTestFixtureAttribute.cs | 1 + .../api/Azure.Storage.Files.Shares.net6.0.cs | 1 + .../api/Azure.Storage.Files.Shares.netstandard2.0.cs | 1 + .../Azure.Storage.Files.Shares/src/ShareClientOptions.cs | 7 ++++++- .../tests/ShareClientTestFixtureAttribute.cs | 1 + .../api/Azure.Storage.Queues.net6.0.cs | 1 + .../api/Azure.Storage.Queues.netstandard2.0.cs | 1 + .../api/Azure.Storage.Queues.netstandard2.1.cs | 1 + sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs | 7 ++++++- .../tests/QueueClientTestFixtureAttribute.cs | 1 + 24 files changed, 49 insertions(+), 10 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/ChangeFeedTestBase.cs b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/ChangeFeedTestBase.cs index d65ba1264ce66..1edae8044c6aa 100644 --- a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/ChangeFeedTestBase.cs +++ b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/ChangeFeedTestBase.cs @@ -33,6 +33,7 @@ namespace Azure.Storage.Blobs.ChangeFeed.Tests BlobClientOptions.ServiceVersion.V2024_08_04, BlobClientOptions.ServiceVersion.V2024_11_04, BlobClientOptions.ServiceVersion.V2025_01_05, + BlobClientOptions.ServiceVersion.V2025_05_05, StorageVersionExtensions.LatestVersion, StorageVersionExtensions.MaxVersion, RecordingServiceVersion = StorageVersionExtensions.MaxVersion, diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs index d93de39ce28c0..8707ec2108684 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs @@ -88,6 +88,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class BlobContainerClient diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index d93de39ce28c0..8707ec2108684 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -88,6 +88,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class BlobContainerClient diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs index d93de39ce28c0..8707ec2108684 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs @@ -88,6 +88,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class BlobContainerClient diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs index b16cefc83a535..0f90cb42066d2 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs @@ -156,7 +156,12 @@ public enum ServiceVersion /// /// The 2025-01-05 service version. /// - V2025_01_05 = 25 + V2025_01_05 = 25, + + /// + /// The 2025-05-05 service version. + /// + V2025_05_05 = 26 #pragma warning restore CA1707 // Identifiers should not contain underscores } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobsClientTestFixtureAttribute.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobsClientTestFixtureAttribute.cs index d0372ab20cf47..b5fa207562666 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobsClientTestFixtureAttribute.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobsClientTestFixtureAttribute.cs @@ -36,6 +36,7 @@ public BlobsClientTestFixtureAttribute(params object[] additionalParameters) BlobClientOptions.ServiceVersion.V2024_08_04, BlobClientOptions.ServiceVersion.V2024_11_04, BlobClientOptions.ServiceVersion.V2025_01_05, + BlobClientOptions.ServiceVersion.V2025_05_05, StorageVersionExtensions.LatestVersion, StorageVersionExtensions.MaxVersion }, diff --git a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.net6.0.cs b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.net6.0.cs index 121838723ee4f..aebd7cfb6a16d 100644 --- a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.net6.0.cs +++ b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.net6.0.cs @@ -183,7 +183,7 @@ public enum SasProtocol } public partial class SasQueryParameters { - public const string DefaultSasVersion = "2025-01-05"; + public const string DefaultSasVersion = "2025-05-05"; protected SasQueryParameters() { } protected SasQueryParameters(System.Collections.Generic.IDictionary values) { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] diff --git a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs index 9b59550e809d0..73d1004bbb02a 100644 --- a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs @@ -182,7 +182,7 @@ public enum SasProtocol } public partial class SasQueryParameters { - public const string DefaultSasVersion = "2025-01-05"; + public const string DefaultSasVersion = "2025-05-05"; protected SasQueryParameters() { } protected SasQueryParameters(System.Collections.Generic.IDictionary values) { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index 3e00882188fba..c16ed90ead55b 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -25,7 +25,7 @@ internal static class Constants /// Gets the default service version to use when building shared access /// signatures. /// - public const string DefaultSasVersion = "2025-01-05"; + public const string DefaultSasVersion = "2025-05-05"; /// /// Max download range size while requesting a transactional hash. diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs index 44c0973ea9be1..3b4b0a4cafd92 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs @@ -56,7 +56,7 @@ internal static class StorageVersionExtensions /// internal const ServiceVersion MaxVersion = #if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK || BlobDataMovementSDK || ShareDataMovementSDK - ServiceVersion.V2025_01_05; + ServiceVersion.V2025_05_05; #else ERROR_STORAGE_SERVICE_NOT_DEFINED; #endif @@ -95,6 +95,7 @@ public static string ToVersionString(this ServiceVersion version) => ServiceVersion.V2024_08_04 => "2024-08-04", ServiceVersion.V2024_11_04 => "2024-11-04", ServiceVersion.V2025_01_05 => "2025-01-05", + ServiceVersion.V2025_05_05 => "2025-05-05", #endif _ => throw Errors.VersionNotSupported(nameof(version)) }; @@ -158,6 +159,8 @@ public static Azure.Storage.Blobs.BlobClientOptions.ServiceVersion AsBlobsVersio Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2024_11_04, Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_01_05 => Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05, + Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_05_05 => + Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05, _ => throw Errors.VersionNotSupported(nameof(version)) }; #endif diff --git a/sdk/storage/Azure.Storage.Common/tests/CommonTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/CommonTestBase.cs index 5694c805d3550..80d95bab47932 100644 --- a/sdk/storage/Azure.Storage.Common/tests/CommonTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/CommonTestBase.cs @@ -35,8 +35,9 @@ namespace Azure.Storage.Test BlobClientOptions.ServiceVersion.V2024_08_04, BlobClientOptions.ServiceVersion.V2024_11_04, BlobClientOptions.ServiceVersion.V2025_01_05, - RecordingServiceVersion = BlobClientOptions.ServiceVersion.V2025_01_05, - LiveServiceVersions = new object[] { BlobClientOptions.ServiceVersion.V2024_11_04, })] + BlobClientOptions.ServiceVersion.V2025_05_05, + RecordingServiceVersion = BlobClientOptions.ServiceVersion.V2025_05_05, + LiveServiceVersions = new object[] { BlobClientOptions.ServiceVersion.V2025_01_05, })] public abstract class CommonTestBase : StorageTestBase { protected readonly BlobClientOptions.ServiceVersion _serviceVersion; diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs index c5b8a7798a0cf..a09b75e4ef259 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs @@ -36,6 +36,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class DataLakeDirectoryClient : Azure.Storage.Files.DataLake.DataLakePathClient diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index c5b8a7798a0cf..a09b75e4ef259 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -36,6 +36,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class DataLakeDirectoryClient : Azure.Storage.Files.DataLake.DataLakePathClient diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeClientOptions.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeClientOptions.cs index 5f8fd0849ba0f..5fcb4ca03f06d 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeClientOptions.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeClientOptions.cs @@ -156,7 +156,12 @@ public enum ServiceVersion /// /// The 2025-01-05 service version. /// - V2025_01_05 = 25 + V2025_01_05 = 25, + + /// + /// The 2025-05-05 service version. + /// + V2025_05_05 = 26 #pragma warning restore CA1707 // Identifiers should not contain underscores } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeClientTestFixtureAttribute.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeClientTestFixtureAttribute.cs index eab0498c5dfcc..8d1b0227c208a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeClientTestFixtureAttribute.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeClientTestFixtureAttribute.cs @@ -34,6 +34,7 @@ public DataLakeClientTestFixtureAttribute() DataLakeClientOptions.ServiceVersion.V2024_08_04, DataLakeClientOptions.ServiceVersion.V2024_11_04, DataLakeClientOptions.ServiceVersion.V2025_01_05, + DataLakeClientOptions.ServiceVersion.V2025_05_05, StorageVersionExtensions.LatestVersion, StorageVersionExtensions.MaxVersion) { diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs index b1b355dda471c..430c0b16d2ebe 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs @@ -149,6 +149,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class ShareDirectoryClient diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs index b1b355dda471c..430c0b16d2ebe 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs @@ -149,6 +149,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class ShareDirectoryClient diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareClientOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareClientOptions.cs index 30c5ab3b05155..e536e194a80d7 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareClientOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareClientOptions.cs @@ -153,7 +153,12 @@ public enum ServiceVersion /// /// The 2025-01-05 service version. /// - V2025_01_05 = 25 + V2025_01_05 = 25, + + /// + /// The 2025-05-05 service version. + /// + V2025_05_05 = 26 #pragma warning restore CA1707 // Identifiers should not contain underscores } diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareClientTestFixtureAttribute.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareClientTestFixtureAttribute.cs index 7d078484201bc..7daa231516799 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareClientTestFixtureAttribute.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareClientTestFixtureAttribute.cs @@ -38,6 +38,7 @@ public ShareClientTestFixtureAttribute(params object[] additionalParameters) ShareClientOptions.ServiceVersion.V2024_08_04, ShareClientOptions.ServiceVersion.V2024_11_04, ShareClientOptions.ServiceVersion.V2025_01_05, + ShareClientOptions.ServiceVersion.V2025_05_05, StorageVersionExtensions.LatestVersion, StorageVersionExtensions.MaxVersion }, diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net6.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net6.0.cs index 9f440eb3639d7..3bcbeb0dc9fb1 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net6.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net6.0.cs @@ -108,6 +108,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class QueueMessageDecodingFailedEventArgs : Azure.SyncAsyncEventArgs diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index 9f440eb3639d7..3bcbeb0dc9fb1 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -108,6 +108,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class QueueMessageDecodingFailedEventArgs : Azure.SyncAsyncEventArgs diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.1.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.1.cs index 9f440eb3639d7..3bcbeb0dc9fb1 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.1.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.1.cs @@ -108,6 +108,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class QueueMessageDecodingFailedEventArgs : Azure.SyncAsyncEventArgs diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs index 9468a41cc15e5..14bdcc581726e 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs @@ -159,7 +159,12 @@ public enum ServiceVersion /// /// The 2025-01-05 service version. /// - V2025_01_05 = 25 + V2025_01_05 = 25, + + /// + /// The 2025-05-05 service version. + /// + V2025_05_05 = 26 #pragma warning restore CA1707 // Identifiers should not contain underscores } diff --git a/sdk/storage/Azure.Storage.Queues/tests/QueueClientTestFixtureAttribute.cs b/sdk/storage/Azure.Storage.Queues/tests/QueueClientTestFixtureAttribute.cs index b053e71cdf051..ef4a9a5e0bde5 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/QueueClientTestFixtureAttribute.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/QueueClientTestFixtureAttribute.cs @@ -37,6 +37,7 @@ public QueueClientTestFixtureAttribute(params object[] additionalParameters) QueueClientOptions.ServiceVersion.V2024_08_04, QueueClientOptions.ServiceVersion.V2024_11_04, QueueClientOptions.ServiceVersion.V2025_01_05, + QueueClientOptions.ServiceVersion.V2025_05_05, StorageVersionExtensions.LatestVersion, StorageVersionExtensions.MaxVersion }, From f93ed21438ef4ffdcf32f9e3b46535a39d779cc2 Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:39:55 -0800 Subject: [PATCH 02/10] Content Validation STG 97 (#46489) --- ...e.Storage.Blobs.Batch.Samples.Tests.csproj | 1 + .../Azure.Storage.Blobs.Batch.Tests.csproj | 3 +- ...rage.Blobs.ChangeFeed.Samples.Tests.csproj | 3 +- ...zure.Storage.Blobs.ChangeFeed.Tests.csproj | 3 +- .../api/Azure.Storage.Blobs.net6.0.cs | 3 + .../api/Azure.Storage.Blobs.netstandard2.0.cs | 3 + .../api/Azure.Storage.Blobs.netstandard2.1.cs | 3 + sdk/storage/Azure.Storage.Blobs/assets.json | 2 +- .../Azure.Storage.Blobs.Samples.Tests.csproj | 1 + .../src/AppendBlobClient.cs | 45 +- .../src/Azure.Storage.Blobs.csproj | 7 + .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 110 +++- .../src/BlobClientOptions.cs | 2 + .../src/BlobClientSideDecryptor.cs | 2 +- .../src/BlockBlobClient.cs | 92 ++- .../src/Models/BlobDownloadDetails.cs | 8 + .../src/Models/BlobDownloadInfo.cs | 10 + .../src/Models/BlobDownloadStreamingResult.cs | 8 + .../Azure.Storage.Blobs/src/PageBlobClient.cs | 49 +- .../src/PartitionedDownloader.cs | 95 +-- .../Azure.Storage.Blobs/src/autorest.md | 4 +- .../tests/Azure.Storage.Blobs.Tests.csproj | 3 + .../BlobBaseClientTransferValidationTests.cs | 114 ++-- .../tests/ClientSideEncryptionTests.cs | 2 +- .../tests/PartitionedDownloaderTests.cs | 2 +- .../Azure.Storage.Common.Samples.Tests.csproj | 1 + .../src/Shared/ChecksumExtensions.cs | 22 + .../src/Shared/Constants.cs | 9 + .../src/Shared/ContentRange.cs | 18 +- .../src/Shared/ContentRangeExtensions.cs | 14 + .../src/Shared/Errors.Clients.cs | 10 + .../Azure.Storage.Common/src/Shared/Errors.cs | 19 + .../src/Shared/LazyLoadingReadOnlyStream.cs | 40 +- .../src/Shared/PooledMemoryStream.cs | 2 +- .../src/Shared/StorageCrc64Composer.cs | 48 +- .../StorageRequestValidationPipelinePolicy.cs | 29 + .../src/Shared/StreamExtensions.cs | 22 +- .../src/Shared/StructuredMessage.cs | 244 ++++++++ ...tructuredMessageDecodingRetriableStream.cs | 264 +++++++++ .../Shared/StructuredMessageDecodingStream.cs | 542 +++++++++++++++++ .../Shared/StructuredMessageEncodingStream.cs | 545 ++++++++++++++++++ ...redMessagePrecalculatedCrcWrapperStream.cs | 451 +++++++++++++++ .../TransferValidationOptionsExtensions.cs | 7 - .../tests/Azure.Storage.Common.Tests.csproj | 9 + .../tests/Shared/FaultyStream.cs | 13 +- .../Shared/ObserveStructuredMessagePolicy.cs | 85 +++ .../tests/Shared/RequestExtensions.cs | 27 + .../Shared/TamperStreamContentsPolicy.cs | 11 +- .../Shared/TransferValidationTestBase.cs | 325 ++++++++--- ...uredMessageDecodingRetriableStreamTests.cs | 246 ++++++++ .../StructuredMessageDecodingStreamTests.cs | 323 +++++++++++ .../StructuredMessageEncodingStreamTests.cs | 271 +++++++++ .../tests/StructuredMessageHelper.cs | 68 +++ .../StructuredMessageStreamRoundtripTests.cs | 127 ++++ .../tests/StructuredMessageTests.cs | 114 ++++ ...ge.DataMovement.Blobs.Samples.Tests.csproj | 1 + .../Azure.Storage.DataMovement.Blobs.csproj | 1 + .../src/DataMovementBlobsExtensions.cs | 4 +- ...re.Storage.DataMovement.Blobs.Tests.csproj | 5 + ...taMovement.Blobs.Files.Shares.Tests.csproj | 1 + ...Movement.Files.Shares.Samples.Tests.csproj | 3 +- .../src/DataMovementSharesExtensions.cs | 4 +- ...age.DataMovement.Files.Shares.Tests.csproj | 1 + .../src/Azure.Storage.DataMovement.csproj | 2 +- .../Azure.Storage.DataMovement.Tests.csproj | 1 + .../Azure.Storage.Files.DataLake/assets.json | 2 +- ...torage.Files.DataLake.Samples.Tests.csproj | 1 + .../src/Azure.Storage.Files.DataLake.csproj | 5 + .../src/DataLakeFileClient.cs | 43 +- .../src/autorest.md | 4 +- .../Azure.Storage.Files.DataLake.Tests.csproj | 3 + ...taLakeFileClientTransferValidationTests.cs | 5 +- .../api/Azure.Storage.Files.Shares.net6.0.cs | 1 + ...ure.Storage.Files.Shares.netstandard2.0.cs | 1 + .../Azure.Storage.Files.Shares/assets.json | 2 +- ....Storage.Files.Shares.Samples.Tests.csproj | 1 + .../src/Azure.Storage.Files.Shares.csproj | 8 +- .../src/Models/ShareFileDownloadInfo.cs | 6 + .../src/ShareErrors.cs | 15 - .../src/ShareFileClient.cs | 165 ++++-- .../src/autorest.md | 4 +- .../Azure.Storage.Files.Shares.Tests.csproj | 1 + .../ShareFileClientTransferValidationTests.cs | 42 +- .../Azure.Storage.Queues.Samples.Tests.csproj | 1 + .../tests/Azure.Storage.Queues.Tests.csproj | 1 + 85 files changed, 4428 insertions(+), 387 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ContentRangeExtensions.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj index 3dea34a02b7ea..6009a5336b8b9 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj @@ -17,6 +17,7 @@ + PreserveNewest diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj index 2b77907e9aaac..286ab317256bf 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj @@ -23,6 +23,7 @@ + PreserveNewest @@ -42,4 +43,4 @@ - \ No newline at end of file + diff --git a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/samples/Azure.Storage.Blobs.ChangeFeed.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/samples/Azure.Storage.Blobs.ChangeFeed.Samples.Tests.csproj index 7711cae537db6..6f8fcaf6528b3 100644 --- a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/samples/Azure.Storage.Blobs.ChangeFeed.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/samples/Azure.Storage.Blobs.ChangeFeed.Samples.Tests.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks) Microsoft Azure.Storage.Blobs.ChangeFeed client library samples @@ -14,6 +14,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj index 9682ab15ecd60..8cf13cd60744f 100644 --- a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj @@ -17,6 +17,7 @@ + @@ -28,4 +29,4 @@ PreserveNewest - \ No newline at end of file + diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs index 18170fd30ce7a..d0e0ceb1f36b7 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs @@ -523,6 +523,7 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } @@ -568,6 +569,7 @@ internal BlobDownloadInfo() { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public string ContentType { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } + public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadOptions @@ -589,6 +591,7 @@ public partial class BlobDownloadStreamingResult : System.IDisposable internal BlobDownloadStreamingResult() { } public System.IO.Stream Content { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } + public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadToOptions diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 8707ec2108684..785d09b1b8b8b 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -523,6 +523,7 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } @@ -568,6 +569,7 @@ internal BlobDownloadInfo() { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public string ContentType { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } + public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadOptions @@ -589,6 +591,7 @@ public partial class BlobDownloadStreamingResult : System.IDisposable internal BlobDownloadStreamingResult() { } public System.IO.Stream Content { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } + public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadToOptions diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs index 8707ec2108684..785d09b1b8b8b 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs @@ -523,6 +523,7 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } @@ -568,6 +569,7 @@ internal BlobDownloadInfo() { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public string ContentType { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } + public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadOptions @@ -589,6 +591,7 @@ public partial class BlobDownloadStreamingResult : System.IDisposable internal BlobDownloadStreamingResult() { } public System.IO.Stream Content { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } + public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadToOptions diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index 0facb33e2a026..1994292f7b658 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_5c382dfb14" + "Tag": "net/storage/Azure.Storage.Blobs_c5174c4663" } diff --git a/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj index 77fd767c3486c..568dd6cba9516 100644 --- a/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj @@ -16,6 +16,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index e70d5e02c82d7..9a110cf8eb13a 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -1242,14 +1242,39 @@ internal async Task> AppendBlockInternal( BlobErrors.VerifyHttpsCustomerProvidedKey(Uri, ClientConfiguration.CustomerProvidedKey); Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content.WithNoDispose().WithProgress(progressHandler); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + ClientSideEncryption == null) // don't allow feature combination + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + contentLength = (content?.Length - content?.Position) ?? 0; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); + contentLength = (content?.Length - content?.Position) ?? 0; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content.WithNoDispose().WithProgress(progressHandler); + } ResponseWithHeaders response; @@ -1267,6 +1292,8 @@ internal async Task> AppendBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, ifModifiedSince: conditions?.IfModifiedSince, ifUnmodifiedSince: conditions?.IfUnmodifiedSince, ifMatch: conditions?.IfMatch?.ToString(), @@ -1289,6 +1316,8 @@ internal async Task> AppendBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, ifModifiedSince: conditions?.IfModifiedSince, ifUnmodifiedSince: conditions?.IfUnmodifiedSince, ifMatch: conditions?.IfMatch?.ToString(), diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index 79c36fd3da840..e2e54aa39cc84 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -54,6 +54,8 @@ + + @@ -93,6 +95,11 @@ + + + + + diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 5186b550b74e4..7b88735731f1b 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1031,6 +1031,7 @@ private async Task> DownloadInternal( ContentHash = blobDownloadDetails.ContentHash, ContentLength = blobDownloadDetails.ContentLength, ContentType = blobDownloadDetails.ContentType, + ExpectTrailingDetails = blobDownloadStreamingResult.ExpectTrailingDetails, }, response.GetRawResponse()); } #endregion @@ -1547,30 +1548,52 @@ internal virtual async ValueTask> Download // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - Stream stream = RetriableStream.Create( - response.Value.Content, - startOffset => - StartDownloadAsync( - range, - conditionsWithEtag, - validationOptions, - startOffset, - async, - cancellationToken) - .EnsureCompleted() - .Value.Content, - async startOffset => - (await StartDownloadAsync( - range, - conditionsWithEtag, - validationOptions, - startOffset, - async, - cancellationToken) - .ConfigureAwait(false)) - .Value.Content, - ClientConfiguration.Pipeline.ResponseClassifier, - Constants.MaxReliabilityRetries); + ValueTask> Factory(long offset, bool async, CancellationToken cancellationToken) + => StartDownloadAsync( + range, + conditionsWithEtag, + validationOptions, + offset, + async, + cancellationToken); + async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)> StructuredMessageFactory( + long offset, bool async, CancellationToken cancellationToken) + { + Response result = await Factory(offset, async, cancellationToken).ConfigureAwait(false); + return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.Details.ContentLength); + } + Stream stream; + if (response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) + { + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( + response.Value.Content, response.Value.Details.ContentLength); + stream = new StructuredMessageDecodingRetriableStream( + decodingStream, + decodedData, + StructuredMessage.Flags.StorageCrc64, + startOffset => StructuredMessageFactory(startOffset, async: false, cancellationToken) + .EnsureCompleted(), + async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) + .ConfigureAwait(false), + decodedData => + { + response.Value.Details.ContentCrc = new byte[StructuredMessage.Crc64Length]; + decodedData.Crc.WriteCrc64(response.Value.Details.ContentCrc); + }, + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); + } + else + { + stream = RetriableStream.Create( + response.Value.Content, + startOffset => Factory(startOffset, async: false, cancellationToken) + .EnsureCompleted().Value.Content, + async startOffset => (await Factory(startOffset, async: true, cancellationToken) + .ConfigureAwait(false)).Value.Content, + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); + } stream = stream.WithProgress(progressHandler); @@ -1578,7 +1601,11 @@ internal virtual async ValueTask> Download * Buffer response stream and ensure it matches the transactional checksum if any. * Storage will not return a checksum for payload >4MB, so this buffer is capped similarly. * Checksum validation is opt-in, so this buffer is part of that opt-in. */ - if (validationOptions != default && validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && validationOptions.AutoValidateChecksum) + if (validationOptions != default && + validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && + validationOptions.AutoValidateChecksum && + // structured message decoding does the validation for us + !response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) { // safe-buffer; transactional hash download limit well below maxInt var readDestStream = new MemoryStream((int)response.Value.Details.ContentLength); @@ -1649,8 +1676,8 @@ await ContentHasher.AssertResponseHashMatchInternal( /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded blob. contains + /// A describing the + /// downloaded blob. contains /// the blob's data. /// /// @@ -1689,13 +1716,29 @@ private async ValueTask> StartDownloadAsyn operationName: nameof(BlobBaseClient.Download), parameterName: nameof(conditions)); + bool? rangeGetContentMD5 = null; + bool? rangeGetContentCRC64 = null; + string structuredBodyType = null; + switch (validationOptions?.ChecksumAlgorithm.ResolveAuto()) + { + case StorageChecksumAlgorithm.MD5: + rangeGetContentMD5 = true; + break; + case StorageChecksumAlgorithm.StorageCrc64: + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + break; + default: + break; + } + if (async) { response = await BlobRestClient.DownloadAsync( range: pageRange?.ToString(), leaseId: conditions?.LeaseId, - rangeGetContentMD5: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, - rangeGetContentCRC64: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? true : null, + rangeGetContentMD5: rangeGetContentMD5, + rangeGetContentCRC64: rangeGetContentCRC64, + structuredBodyType: structuredBodyType, encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, @@ -1712,8 +1755,9 @@ private async ValueTask> StartDownloadAsyn response = BlobRestClient.Download( range: pageRange?.ToString(), leaseId: conditions?.LeaseId, - rangeGetContentMD5: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, - rangeGetContentCRC64: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? true : null, + rangeGetContentMD5: rangeGetContentMD5, + rangeGetContentCRC64: rangeGetContentCRC64, + structuredBodyType: structuredBodyType, encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, @@ -1729,9 +1773,11 @@ private async ValueTask> StartDownloadAsyn long length = response.IsUnavailable() ? 0 : response.Headers.ContentLength ?? 0; ClientConfiguration.Pipeline.LogTrace($"Response: {response.GetRawResponse().Status}, ContentLength: {length}"); - return Response.FromValue( + Response result = Response.FromValue( response.ToBlobDownloadStreamingResult(), response.GetRawResponse()); + result.Value.ExpectTrailingDetails = structuredBodyType != null; + return result; } #endregion diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs index 0f90cb42066d2..e8def313432dc 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs @@ -323,6 +323,8 @@ private void AddHeadersAndQueryParameters() Diagnostics.LoggedHeaderNames.Add("x-ms-encryption-key-sha256"); Diagnostics.LoggedHeaderNames.Add("x-ms-copy-source-error-code"); Diagnostics.LoggedHeaderNames.Add("x-ms-copy-source-status-code"); + Diagnostics.LoggedHeaderNames.Add("x-ms-structured-body"); + Diagnostics.LoggedHeaderNames.Add("x-ms-structured-content-length"); Diagnostics.LoggedQueryParameters.Add("comp"); Diagnostics.LoggedQueryParameters.Add("maxresults"); diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs index 0d03c057fee52..2ab39be2784cc 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs @@ -187,7 +187,7 @@ private static bool CanIgnorePadding(ContentRange? contentRange) // did we request the last block? // end is inclusive/0-index, so end = n and size = n+1 means we requested the last block - if (contentRange.Value.Size - contentRange.Value.End == 1) + if (contentRange.Value.TotalResourceLength - contentRange.Value.End == 1) { return false; } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index f9269ce6cb893..0d65fae7b5011 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -875,14 +875,35 @@ internal virtual async Task> UploadInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content?.WithNoDispose().WithProgress(progressHandler); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (content != null && + validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + ClientSideEncryption == null) // don't allow feature combination + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64); + contentLength = content.Length - content.Position; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content.WithNoDispose().WithProgress(progressHandler); + } ResponseWithHeaders response; @@ -921,6 +942,8 @@ internal virtual async Task> UploadInternal( legalHold: legalHold, transactionalContentMD5: hashResult?.MD5AsArray, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -953,6 +976,8 @@ internal virtual async Task> UploadInternal( legalHold: legalHold, transactionalContentMD5: hashResult?.MD5AsArray, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, cancellationToken: cancellationToken); } @@ -1305,14 +1330,39 @@ internal virtual async Task> StageBlockInternal( Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content.WithNoDispose().WithProgress(progressHandler); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + ClientSideEncryption == null) // don't allow feature combination + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + contentLength = (content?.Length - content?.Position) ?? 0; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); + contentLength = (content?.Length - content?.Position) ?? 0; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content.WithNoDispose().WithProgress(progressHandler); + } ResponseWithHeaders response; @@ -1320,7 +1370,7 @@ internal virtual async Task> StageBlockInternal( { response = await BlockBlobRestClient.StageBlockAsync( blockId: base64BlockId, - contentLength: (content?.Length - content?.Position) ?? 0, + contentLength: contentLength, body: content, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, transactionalContentMD5: hashResult?.MD5AsArray, @@ -1329,6 +1379,8 @@ internal virtual async Task> StageBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -1336,7 +1388,7 @@ internal virtual async Task> StageBlockInternal( { response = BlockBlobRestClient.StageBlock( blockId: base64BlockId, - contentLength: (content?.Length - content?.Position) ?? 0, + contentLength: contentLength, body: content, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, transactionalContentMD5: hashResult?.MD5AsArray, @@ -1345,6 +1397,8 @@ internal virtual async Task> StageBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, cancellationToken: cancellationToken); } @@ -2791,7 +2845,7 @@ internal async Task OpenWriteInternal( immutabilityPolicy: default, legalHold: default, progressHandler: default, - transferValidationOverride: default, + transferValidationOverride: new() { ChecksumAlgorithm = StorageChecksumAlgorithm.None }, operationName: default, async: async, cancellationToken: cancellationToken) diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs index bc119822cdc12..0490ec239798e 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs @@ -34,6 +34,14 @@ public class BlobDownloadDetails public byte[] ContentHash { get; internal set; } #pragma warning restore CA1819 // Properties should not return arrays + /// + /// When requested using , this value contains the CRC for the download blob range. + /// This value may only become populated once the network stream is fully consumed. If this instance is accessed through + /// , the network stream has already been consumed. Otherwise, consume the content stream before + /// checking this value. + /// + public byte[] ContentCrc { get; internal set; } + /// /// Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob. /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs index e034573b54b3a..b42801e36ab55 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs @@ -4,6 +4,8 @@ using System; using System.ComponentModel; using System.IO; +using System.Threading.Tasks; +using Azure.Core; using Azure.Storage.Shared; namespace Azure.Storage.Blobs.Models @@ -49,6 +51,14 @@ public class BlobDownloadInfo : IDisposable, IDownloadedContent /// public BlobDownloadDetails Details { get; internal set; } + /// + /// Indicates some contents of are mixed into the response stream. + /// They will not be set until has been fully consumed. These details + /// will be extracted from the content stream by the library before the calling code can + /// encounter them. + /// + public bool ExpectTrailingDetails { get; internal set; } + /// /// Constructor. /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs index 4fbada6e67aad..9b7d4d4e00dad 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs @@ -24,6 +24,14 @@ internal BlobDownloadStreamingResult() { } /// public Stream Content { get; internal set; } + /// + /// Indicates some contents of are mixed into the response stream. + /// They will not be set until has been fully consumed. These details + /// will be extracted from the content stream by the library before the calling code can + /// encounter them. + /// + public bool ExpectTrailingDetails { get; internal set; } + /// /// Disposes the by calling Dispose on the underlying stream. /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs index fa575e41b8ebe..7038897531fbb 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs @@ -1363,15 +1363,42 @@ internal async Task> UploadPagesInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content?.WithNoDispose().WithProgress(progressHandler); - HttpRange range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + HttpRange range; + if (validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + ClientSideEncryption == null) // don't allow feature combination + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + contentLength = (content?.Length - content?.Position) ?? 0; + range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content?.WithNoDispose().WithProgress(progressHandler); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); + contentLength = (content?.Length - content?.Position) ?? 0; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content?.WithNoDispose().WithProgress(progressHandler); + range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); + } ResponseWithHeaders response; @@ -1388,6 +1415,8 @@ internal async Task> UploadPagesInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, ifSequenceNumberLessThanOrEqualTo: conditions?.IfSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan: conditions?.IfSequenceNumberLessThan, ifSequenceNumberEqualTo: conditions?.IfSequenceNumberEqual, @@ -1412,6 +1441,8 @@ internal async Task> UploadPagesInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, ifSequenceNumberLessThanOrEqualTo: conditions?.IfSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan: conditions?.IfSequenceNumberLessThan, ifSequenceNumberEqualTo: conditions?.IfSequenceNumberEqual, diff --git a/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs b/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs index 361594561ed7e..601ee2ac9d398 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs @@ -23,6 +23,8 @@ internal class PartitionedDownloader private const string _operationName = nameof(BlobBaseClient) + "." + nameof(BlobBaseClient.DownloadTo); private const string _innerOperationName = nameof(BlobBaseClient) + "." + nameof(BlobBaseClient.DownloadStreaming); + private const int Crc64Len = Constants.StorageCrc64SizeInBytes; + /// /// The client used to download the blob. /// @@ -49,6 +51,7 @@ internal class PartitionedDownloader /// private readonly StorageChecksumAlgorithm _validationAlgorithm; private readonly int _checksumSize; + // TODO disabling master crc temporarily. segment CRCs still handled. private bool UseMasterCrc => _validationAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64; private StorageCrc64HashAlgorithm _masterCrcCalculator = null; @@ -202,20 +205,31 @@ public async Task DownloadToInternal( } // Destination wrapped in master crc step if needed (must wait until after encryption wrap check) - Memory composedCrc = default; + byte[] composedCrcBuf = default; if (UseMasterCrc) { _masterCrcCalculator = StorageCrc64HashAlgorithm.Create(); destination = ChecksumCalculatingStream.GetWriteStream(destination, _masterCrcCalculator.Append); - disposables.Add(_arrayPool.RentAsMemoryDisposable( - Constants.StorageCrc64SizeInBytes, out composedCrc)); - composedCrc.Span.Clear(); + disposables.Add(_arrayPool.RentDisposable(Crc64Len, out composedCrcBuf)); + composedCrcBuf.Clear(); } // If the first segment was the entire blob, we'll copy that to // the output stream and finish now - long initialLength = initialResponse.Value.Details.ContentLength; - long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange); + long initialLength; + long totalLength; + // Get blob content length downloaded from content range when available to handle transit encoding + if (string.IsNullOrWhiteSpace(initialResponse.Value.Details.ContentRange)) + { + initialLength = initialResponse.Value.Details.ContentLength; + totalLength = 0; + } + else + { + ContentRange recievedRange = ContentRange.Parse(initialResponse.Value.Details.ContentRange); + initialLength = recievedRange.GetRangeLength(); + totalLength = recievedRange.TotalResourceLength.Value; + } if (initialLength == totalLength) { await HandleOneShotDownload(initialResponse, destination, async, cancellationToken) @@ -240,15 +254,16 @@ await HandleOneShotDownload(initialResponse, destination, async, cancellationTok } else { - using (_arrayPool.RentAsMemoryDisposable(_checksumSize, out Memory partitionChecksum)) + using (_arrayPool.RentDisposable(_checksumSize, out byte[] partitionChecksum)) { - await CopyToInternal(initialResponse, destination, partitionChecksum, async, cancellationToken).ConfigureAwait(false); + await CopyToInternal(initialResponse, destination, new(partitionChecksum, 0, _checksumSize), async, cancellationToken).ConfigureAwait(false); if (UseMasterCrc) { StorageCrc64Composer.Compose( - (composedCrc.ToArray(), 0L), - (partitionChecksum.ToArray(), initialResponse.Value.Details.ContentLength) - ).CopyTo(composedCrc); + (composedCrcBuf, 0L), + (partitionChecksum, initialResponse.Value.Details.ContentRange.GetContentRangeLengthOrDefault() + ?? initialResponse.Value.Details.ContentLength) + ).AsSpan(0, Crc64Len).CopyTo(composedCrcBuf); } } } @@ -287,15 +302,16 @@ await HandleOneShotDownload(initialResponse, destination, async, cancellationTok else { Response result = await responseValueTask.ConfigureAwait(false); - using (_arrayPool.RentAsMemoryDisposable(_checksumSize, out Memory partitionChecksum)) + using (_arrayPool.RentDisposable(_checksumSize, out byte[] partitionChecksum)) { - await CopyToInternal(result, destination, partitionChecksum, async, cancellationToken).ConfigureAwait(false); + await CopyToInternal(result, destination, new(partitionChecksum, 0, _checksumSize), async, cancellationToken).ConfigureAwait(false); if (UseMasterCrc) { StorageCrc64Composer.Compose( - (composedCrc.ToArray(), 0L), - (partitionChecksum.ToArray(), result.Value.Details.ContentLength) - ).CopyTo(composedCrc); + (composedCrcBuf, 0L), + (partitionChecksum, result.Value.Details.ContentRange.GetContentRangeLengthOrDefault() + ?? result.Value.Details.ContentLength) + ).AsSpan(0, Crc64Len).CopyTo(composedCrcBuf); } } } @@ -311,7 +327,7 @@ await HandleOneShotDownload(initialResponse, destination, async, cancellationTok } #pragma warning restore AZC0110 // DO NOT use await keyword in possibly synchronous scope. - await FinalizeDownloadInternal(destination, composedCrc, async, cancellationToken) + await FinalizeDownloadInternal(destination, composedCrcBuf?.AsMemory(0, Crc64Len) ?? default, async, cancellationToken) .ConfigureAwait(false); return initialResponse.GetRawResponse(); @@ -329,7 +345,7 @@ async Task ConsumeQueuedTask() // CopyToAsync causes ConsumeQueuedTask to wait until the // download is complete - using (_arrayPool.RentAsMemoryDisposable(_checksumSize, out Memory partitionChecksum)) + using (_arrayPool.RentDisposable(_checksumSize, out byte[] partitionChecksum)) { await CopyToInternal( response, @@ -338,13 +354,14 @@ await CopyToInternal( async, cancellationToken) .ConfigureAwait(false); - if (UseMasterCrc) - { - StorageCrc64Composer.Compose( - (composedCrc.ToArray(), 0L), - (partitionChecksum.ToArray(), response.Value.Details.ContentLength) - ).CopyTo(composedCrc); - } + if (UseMasterCrc) + { + StorageCrc64Composer.Compose( + (composedCrcBuf, 0L), + (partitionChecksum, response.Value.Details.ContentRange.GetContentRangeLengthOrDefault() + ?? response.Value.Details.ContentLength) + ).AsSpan(0, Crc64Len).CopyTo(composedCrcBuf); + } } } } @@ -391,7 +408,7 @@ await FinalizeDownloadInternal(destination, partitionChecksum, async, cancellati private async Task FinalizeDownloadInternal( Stream destination, - Memory composedCrc, + ReadOnlyMemory composedCrc, bool async, CancellationToken cancellationToken) { @@ -407,20 +424,6 @@ private async Task FinalizeDownloadInternal( } } - private static long ParseRangeTotalLength(string range) - { - if (range == null) - { - return 0; - } - int lengthSeparator = range.IndexOf("/", StringComparison.InvariantCultureIgnoreCase); - if (lengthSeparator == -1) - { - throw BlobErrors.ParsingFullHttpRangeFailed(range); - } - return long.Parse(range.Substring(lengthSeparator + 1), CultureInfo.InvariantCulture); - } - private async Task CopyToInternal( Response response, Stream destination, @@ -429,7 +432,9 @@ private async Task CopyToInternal( CancellationToken cancellationToken) { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); - using IHasher hasher = ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm); + // if structured message, this crc is validated in the decoding process. don't decode it here. + bool structuredMessage = response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader); + using IHasher hasher = structuredMessage ? null : ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm); using Stream rawSource = response.Value.Content; using Stream source = hasher != null ? ChecksumCalculatingStream.GetReadStream(rawSource, hasher.AppendHash) @@ -441,7 +446,13 @@ await source.CopyToInternal( cancellationToken) .ConfigureAwait(false); - if (hasher != null) + // with structured message, the message integrity will already be validated, + // but we can still get the checksum out of the response object + if (structuredMessage) + { + response.Value.Details.ContentCrc?.CopyTo(checksumBuffer.Span); + } + else if (hasher != null) { hasher.GetFinalHash(checksumBuffer.Span); (ReadOnlyMemory checksum, StorageChecksumAlgorithm _) diff --git a/sdk/storage/Azure.Storage.Blobs/src/autorest.md b/sdk/storage/Azure.Storage.Blobs/src/autorest.md index 6c18c66066ebd..72e1adb821b41 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/autorest.md +++ b/sdk/storage/Azure.Storage.Blobs/src/autorest.md @@ -34,7 +34,7 @@ directive: if (property.includes('/{containerName}/{blob}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ContainerName") && false == param['$ref'].endsWith("#/parameters/Blob"))}); - } + } else if (property.includes('/{containerName}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ContainerName"))}); @@ -158,7 +158,7 @@ directive: var newName = property.replace('/{containerName}/{blob}', ''); $[newName] = $[oldName]; delete $[oldName]; - } + } else if (property.includes('/{containerName}')) { var oldName = property; diff --git a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj index 62c7b6d17e63e..1c3856c83b64e 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj @@ -6,6 +6,9 @@ Microsoft Azure.Storage.Blobs client library tests false + + BlobSDK + diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs index 73d11612f1d8c..3ec448e6d1ed0 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; +using System.Buffers; using System.IO; using System.Threading.Tasks; using Azure.Core.TestFramework; @@ -37,7 +39,10 @@ protected override async Task> GetDispo StorageChecksumAlgorithm uploadAlgorithm = StorageChecksumAlgorithm.None, StorageChecksumAlgorithm downloadAlgorithm = StorageChecksumAlgorithm.None) { - var disposingContainer = await ClientBuilder.GetTestContainerAsync(service: service, containerName: containerName); + var disposingContainer = await ClientBuilder.GetTestContainerAsync( + service: service, + containerName: containerName, + publicAccessType: PublicAccessType.None); disposingContainer.Container.ClientConfiguration.TransferValidation.Upload.ChecksumAlgorithm = uploadAlgorithm; disposingContainer.Container.ClientConfiguration.TransferValidation.Download.ChecksumAlgorithm = downloadAlgorithm; @@ -91,57 +96,96 @@ public override void TestAutoResolve() } #region Added Tests - [TestCaseSource("GetValidationAlgorithms")] - public async Task ExpectedDownloadStreamingStreamTypeReturned(StorageChecksumAlgorithm algorithm) + [Test] + public virtual async Task OlderServiceVersionThrowsOnStructuredMessage() { - await using var test = await GetDisposingContainerAsync(); + // use service version before structured message was introduced + await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( + service: ClientBuilder.GetServiceClient_SharedKey( + InstrumentClientOptions(new BlobClientOptions(BlobClientOptions.ServiceVersion.V2024_11_04))), + publicAccessType: PublicAccessType.None); // Arrange - var data = GetRandomBuffer(Constants.KB); - BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName())); - using (var stream = new MemoryStream(data)) + const int dataLength = Constants.KB; + var data = GetRandomBuffer(dataLength); + + var resourceName = GetNewResourceName(); + var blob = InstrumentClient(disposingContainer.Container.GetBlobClient(GetNewResourceName())); + await blob.UploadAsync(BinaryData.FromBytes(data)); + + var validationOptions = new DownloadTransferValidationOptions { - await blob.UploadAsync(stream); - } - // don't make options instance at all for no hash request - DownloadTransferValidationOptions transferValidation = algorithm == StorageChecksumAlgorithm.None - ? default - : new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; + ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 + }; + AsyncTestDelegate operation = async () => await (await blob.DownloadStreamingAsync( + new BlobDownloadOptions + { + Range = new HttpRange(length: Constants.StructuredMessage.MaxDownloadCrcWithHeader + 1), + TransferValidation = validationOptions, + })).Value.Content.CopyToAsync(Stream.Null); + Assert.That(operation, Throws.TypeOf()); + } + + [Test] + public async Task StructuredMessagePopulatesCrcDownloadStreaming() + { + await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( + publicAccessType: PublicAccessType.None); + + const int dataLength = Constants.KB; + byte[] data = GetRandomBuffer(dataLength); + byte[] dataCrc = new byte[8]; + StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); + + var blob = disposingContainer.Container.GetBlobClient(GetNewResourceName()); + await blob.UploadAsync(BinaryData.FromBytes(data)); - // Act - Response response = await blob.DownloadStreamingAsync(new BlobDownloadOptions + Response response = await blob.DownloadStreamingAsync(new() { - TransferValidation = transferValidation, - Range = new HttpRange(length: data.Length) + TransferValidation = new DownloadTransferValidationOptions + { + ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 + } }); - // Assert - // validated stream is buffered - Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType()); + // crc is not present until response stream is consumed + Assert.That(response.Value.Details.ContentCrc, Is.Null); + + byte[] downloadedData; + using (MemoryStream ms = new()) + { + await response.Value.Content.CopyToAsync(ms); + downloadedData = ms.ToArray(); + } + + Assert.That(response.Value.Details.ContentCrc, Is.EqualTo(dataCrc)); + Assert.That(downloadedData, Is.EqualTo(data)); } [Test] - public async Task ExpectedDownloadStreamingStreamTypeReturned_None() + public async Task StructuredMessagePopulatesCrcDownloadContent() { - await using var test = await GetDisposingContainerAsync(); + await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( + publicAccessType: PublicAccessType.None); - // Arrange - var data = GetRandomBuffer(Constants.KB); - BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName())); - using (var stream = new MemoryStream(data)) - { - await blob.UploadAsync(stream); - } + const int dataLength = Constants.KB; + byte[] data = GetRandomBuffer(dataLength); + byte[] dataCrc = new byte[8]; + StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); + + var blob = disposingContainer.Container.GetBlobClient(GetNewResourceName()); + await blob.UploadAsync(BinaryData.FromBytes(data)); - // Act - Response response = await blob.DownloadStreamingAsync(new BlobDownloadOptions + Response response = await blob.DownloadContentAsync(new BlobDownloadOptions() { - Range = new HttpRange(length: data.Length) + TransferValidation = new DownloadTransferValidationOptions + { + ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 + } }); - // Assert - // unvalidated stream type is private; just check we didn't get back a buffered stream - Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType()); + Assert.That(response.Value.Details.ContentCrc, Is.EqualTo(dataCrc)); + Assert.That(response.Value.Content.ToArray(), Is.EqualTo(data)); } #endregion } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs index 20ad6f6afb4b1..9b0fe99f6cb98 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -1358,7 +1358,7 @@ public void CanParseLargeContentRange() { long compareValue = (long)Int32.MaxValue + 1; //Increase max int32 by one ContentRange contentRange = ContentRange.Parse($"bytes 0 {compareValue} {compareValue}"); - Assert.AreEqual((long)Int32.MaxValue + 1, contentRange.Size); + Assert.AreEqual((long)Int32.MaxValue + 1, contentRange.TotalResourceLength); Assert.AreEqual(0, contentRange.Start); Assert.AreEqual((long)Int32.MaxValue + 1, contentRange.End); } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs index d8d4756a510c1..af408264c5bfa 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs @@ -305,7 +305,7 @@ public Response GetStream(HttpRange range, BlobRequ ContentHash = new byte[] { 1, 2, 3 }, LastModified = DateTimeOffset.Now, Metadata = new Dictionary() { { "meta", "data" } }, - ContentRange = $"bytes {range.Offset}-{range.Offset + contentLength}/{_length}", + ContentRange = $"bytes {range.Offset}-{Math.Max(1, range.Offset + contentLength - 1)}/{_length}", ETag = s_etag, ContentEncoding = "test", CacheControl = "test", diff --git a/sdk/storage/Azure.Storage.Common/samples/Azure.Storage.Common.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Common/samples/Azure.Storage.Common.Samples.Tests.csproj index 7d454aeaa0af2..aeca4497a8770 100644 --- a/sdk/storage/Azure.Storage.Common/samples/Azure.Storage.Common.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Common/samples/Azure.Storage.Common.Samples.Tests.csproj @@ -19,6 +19,7 @@ + PreserveNewest diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs new file mode 100644 index 0000000000000..48304640eee43 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; + +namespace Azure.Storage; + +internal static class ChecksumExtensions +{ + public static void WriteCrc64(this ulong crc, Span dest) + => BinaryPrimitives.WriteUInt64LittleEndian(dest, crc); + + public static bool TryWriteCrc64(this ulong crc, Span dest) + => BinaryPrimitives.TryWriteUInt64LittleEndian(dest, crc); + + public static ulong ReadCrc64(this ReadOnlySpan crc) + => BinaryPrimitives.ReadUInt64LittleEndian(crc); + + public static bool TryReadCrc64(this ReadOnlySpan crc, out ulong value) + => BinaryPrimitives.TryReadUInt64LittleEndian(crc, out value); +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index c16ed90ead55b..48f93c8422fb4 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -657,6 +657,15 @@ internal static class AccountResources internal static readonly int[] PathStylePorts = { 10000, 10001, 10002, 10003, 10004, 10100, 10101, 10102, 10103, 10104, 11000, 11001, 11002, 11003, 11004, 11100, 11101, 11102, 11103, 11104 }; } + internal static class StructuredMessage + { + public const string StructuredMessageHeader = "x-ms-structured-body"; + public const string StructuredContentLength = "x-ms-structured-content-length"; + public const string CrcStructuredMessage = "XSM/1.0; properties=crc64"; + public const int DefaultSegmentContentLength = 4 * MB; + public const int MaxDownloadCrcWithHeader = 4 * MB; + } + internal static class ClientSideEncryption { public const string HttpMessagePropertyKeyV1 = "Azure.Storage.StorageTelemetryPolicy.ClientSideEncryption.V1"; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ContentRange.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ContentRange.cs index f656382efad2b..cb3b0a7bee189 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ContentRange.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ContentRange.cs @@ -82,20 +82,20 @@ public RangeUnit(string value) public long? End { get; } /// - /// Size of this range, measured in this instance's . + /// Size of the entire resource this range is from, measured in this instance's . /// - public long? Size { get; } + public long? TotalResourceLength { get; } /// /// Unit this range is measured in. Generally "bytes". /// public RangeUnit Unit { get; } - public ContentRange(RangeUnit unit, long? start, long? end, long? size) + public ContentRange(RangeUnit unit, long? start, long? end, long? totalResourceLength) { Start = start; End = end; - Size = size; + TotalResourceLength = totalResourceLength; Unit = unit; } @@ -113,7 +113,7 @@ public static ContentRange Parse(string headerValue) string unit = default; long? start = default; long? end = default; - long? size = default; + long? resourceSize = default; try { @@ -136,10 +136,10 @@ public static ContentRange Parse(string headerValue) var rawSize = tokens[blobSizeIndex]; if (rawSize != WildcardMarker) { - size = long.Parse(rawSize, CultureInfo.InvariantCulture); + resourceSize = long.Parse(rawSize, CultureInfo.InvariantCulture); } - return new ContentRange(unit, start, end, size); + return new ContentRange(unit, start, end, resourceSize); } catch (IndexOutOfRangeException) { @@ -165,7 +165,7 @@ public static HttpRange ToHttpRange(ContentRange contentRange) /// /// Indicates whether this instance and a specified are equal /// - public bool Equals(ContentRange other) => (other.Start == Start) && (other.End == End) && (other.Unit == Unit) && (other.Size == Size); + public bool Equals(ContentRange other) => (other.Start == Start) && (other.End == End) && (other.Unit == Unit) && (other.TotalResourceLength == TotalResourceLength); /// /// Determines if two values are the same. @@ -185,6 +185,6 @@ public static HttpRange ToHttpRange(ContentRange contentRange) /// [EditorBrowsable(EditorBrowsableState.Never)] - public override int GetHashCode() => HashCodeBuilder.Combine(Start, End, Size, Unit.GetHashCode()); + public override int GetHashCode() => HashCodeBuilder.Combine(Start, End, TotalResourceLength, Unit.GetHashCode()); } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ContentRangeExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ContentRangeExtensions.cs new file mode 100644 index 0000000000000..160a69b19a9c8 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ContentRangeExtensions.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography; + +internal static class ContentRangeExtensions +{ + public static long? GetContentRangeLengthOrDefault(this string contentRange) + => string.IsNullOrWhiteSpace(contentRange) + ? default : ContentRange.Parse(contentRange).GetRangeLength(); + + public static long GetRangeLength(this ContentRange contentRange) + => contentRange.End.Value - contentRange.Start.Value + 1; +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs index 2a5fe38668104..867607e551e6a 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs @@ -3,6 +3,7 @@ using System; using System.Globalization; +using System.IO; using System.Linq; using System.Security.Authentication; using System.Xml.Serialization; @@ -105,9 +106,18 @@ public static ArgumentException VersionNotSupported(string paramName) public static RequestFailedException ClientRequestIdMismatch(Response response, string echo, string original) => new RequestFailedException(response.Status, $"Response x-ms-client-request-id '{echo}' does not match the original expected request id, '{original}'.", null); + public static InvalidDataException StructuredMessageNotAcknowledgedGET(Response response) + => new InvalidDataException($"Response does not acknowledge structured message was requested. Unknown data structure in response body."); + + public static InvalidDataException StructuredMessageNotAcknowledgedPUT(Response response) + => new InvalidDataException($"Response does not acknowledge structured message was sent. Unexpected data may have been persisted to storage."); + public static ArgumentException TransactionalHashingNotSupportedWithClientSideEncryption() => new ArgumentException("Client-side encryption and transactional hashing are not supported at the same time."); + public static InvalidDataException ExpectedStructuredMessage() + => new InvalidDataException($"Expected {Constants.StructuredMessage.StructuredMessageHeader} in response, but found none."); + public static void VerifyHttpsTokenAuth(Uri uri) { if (uri.Scheme != Constants.Https) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs index 6b89a59011d51..e3372665928c1 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs @@ -72,6 +72,9 @@ public static ArgumentException CannotDeferTransactionalHashVerification() public static ArgumentException CannotInitializeWriteStreamWithData() => new ArgumentException("Initialized buffer for StorageWriteStream must be empty."); + public static InvalidDataException InvalidStructuredMessage(string optionalMessage = default) + => new InvalidDataException(("Invalid structured message data. " + optionalMessage ?? "").Trim()); + internal static void VerifyStreamPosition(Stream stream, string streamName) { if (stream != null && stream.CanSeek && stream.Length > 0 && stream.Position >= stream.Length) @@ -80,6 +83,22 @@ internal static void VerifyStreamPosition(Stream stream, string streamName) } } + internal static void AssertBufferMinimumSize(ReadOnlySpan buffer, int minSize, string paramName) + { + if (buffer.Length < minSize) + { + throw new ArgumentException($"Expected buffer Length of at least {minSize} bytes. Got {buffer.Length}.", paramName); + } + } + + internal static void AssertBufferExactSize(ReadOnlySpan buffer, int size, string paramName) + { + if (buffer.Length != size) + { + throw new ArgumentException($"Expected buffer Length of exactly {size} bytes. Got {buffer.Length}.", paramName); + } + } + public static void ThrowIfParamNull(object obj, string paramName) { if (obj == null) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs index c3e9c641c3fea..fe2db427bef02 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs @@ -249,41 +249,9 @@ private async Task DownloadInternal(bool async, CancellationToken cancellat response = await _downloadInternalFunc(range, _validationOptions, async, cancellationToken).ConfigureAwait(false); using Stream networkStream = response.Value.Content; - - // The number of bytes we just downloaded. - long downloadSize = GetResponseRange(response.GetRawResponse()).Length.Value; - - // The number of bytes we copied in the last loop. - int copiedBytes; - - // Bytes we have copied so far. - int totalCopiedBytes = 0; - - // Bytes remaining to copy. It is save to truncate the long because we asked for a max of int _buffer size bytes. - int remainingBytes = (int)downloadSize; - - do - { - if (async) - { - copiedBytes = await networkStream.ReadAsync( - buffer: _buffer, - offset: totalCopiedBytes, - count: remainingBytes, - cancellationToken: cancellationToken).ConfigureAwait(false); - } - else - { - copiedBytes = networkStream.Read( - buffer: _buffer, - offset: totalCopiedBytes, - count: remainingBytes); - } - - totalCopiedBytes += copiedBytes; - remainingBytes -= copiedBytes; - } - while (copiedBytes != 0); + // use stream copy to ensure consumption of any trailing metadata (e.g. structured message) + // allow buffer limits to catch the error of data size mismatch + int totalCopiedBytes = (int) await networkStream.CopyToInternal(new MemoryStream(_buffer), async, cancellationToken).ConfigureAwait((false)); _bufferPosition = 0; _bufferLength = totalCopiedBytes; @@ -291,7 +259,7 @@ private async Task DownloadInternal(bool async, CancellationToken cancellat // if we deferred transactional hash validation on download, validate now // currently we always defer but that may change - if (_validationOptions != default && _validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && !_validationOptions.AutoValidateChecksum) + if (_validationOptions != default && _validationOptions.ChecksumAlgorithm == StorageChecksumAlgorithm.MD5 && !_validationOptions.AutoValidateChecksum) // TODO better condition { ContentHasher.AssertResponseHashMatch(_buffer, _bufferPosition, _bufferLength, _validationOptions.ChecksumAlgorithm, response.GetRawResponse()); } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs index 3e218d18a90af..6070329d10d3d 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs @@ -251,7 +251,7 @@ public override int Read(byte[] buffer, int offset, int count) Length - Position, bufferCount - (Position - offsetOfBuffer), count - read); - Array.Copy(currentBuffer, Position - offsetOfBuffer, buffer, read, toCopy); + Array.Copy(currentBuffer, Position - offsetOfBuffer, buffer, offset + read, toCopy); read += toCopy; Position += toCopy; } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs index ab6b76d78a87e..307ff23b21144 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs @@ -12,22 +12,52 @@ namespace Azure.Storage /// internal static class StorageCrc64Composer { - public static Memory Compose(params (byte[] Crc64, long OriginalDataLength)[] partitions) + public static byte[] Compose(params (byte[] Crc64, long OriginalDataLength)[] partitions) + => Compose(partitions.AsEnumerable()); + + public static byte[] Compose(IEnumerable<(byte[] Crc64, long OriginalDataLength)> partitions) { - return Compose(partitions.AsEnumerable()); + ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64, 0), tup.OriginalDataLength))); + return BitConverter.GetBytes(result); } - public static Memory Compose(IEnumerable<(byte[] Crc64, long OriginalDataLength)> partitions) + public static byte[] Compose(params (ReadOnlyMemory Crc64, long OriginalDataLength)[] partitions) + => Compose(partitions.AsEnumerable()); + + public static byte[] Compose(IEnumerable<(ReadOnlyMemory Crc64, long OriginalDataLength)> partitions) { - ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64, 0), tup.OriginalDataLength))); - return new Memory(BitConverter.GetBytes(result)); +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64.Span), tup.OriginalDataLength))); +#else + ulong result = Compose(partitions.Select(tup => (System.BitConverter.ToUInt64(tup.Crc64.ToArray(), 0), tup.OriginalDataLength))); +#endif + return BitConverter.GetBytes(result); } + public static byte[] Compose( + ReadOnlySpan leftCrc64, long leftOriginalDataLength, + ReadOnlySpan rightCrc64, long rightOriginalDataLength) + { +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + ulong result = Compose( + (BitConverter.ToUInt64(leftCrc64), leftOriginalDataLength), + (BitConverter.ToUInt64(rightCrc64), rightOriginalDataLength)); +#else + ulong result = Compose( + (BitConverter.ToUInt64(leftCrc64.ToArray(), 0), leftOriginalDataLength), + (BitConverter.ToUInt64(rightCrc64.ToArray(), 0), rightOriginalDataLength)); +#endif + return BitConverter.GetBytes(result); + } + + public static ulong Compose(params (ulong Crc64, long OriginalDataLength)[] partitions) + => Compose(partitions.AsEnumerable()); + public static ulong Compose(IEnumerable<(ulong Crc64, long OriginalDataLength)> partitions) { ulong composedCrc = 0; long composedDataLength = 0; - foreach (var tup in partitions) + foreach ((ulong crc64, long originalDataLength) in partitions) { composedCrc = StorageCrc64Calculator.Concatenate( uInitialCrcAB: 0, @@ -35,9 +65,9 @@ public static ulong Compose(IEnumerable<(ulong Crc64, long OriginalDataLength)> uFinalCrcA: composedCrc, uSizeA: (ulong) composedDataLength, uInitialCrcB: 0, - uFinalCrcB: tup.Crc64, - uSizeB: (ulong)tup.OriginalDataLength); - composedDataLength += tup.OriginalDataLength; + uFinalCrcB: crc64, + uSizeB: (ulong)originalDataLength); + composedDataLength += originalDataLength; } return composedCrc; } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs index 0cef4f4d8d4ed..9f4ddb5249e82 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs @@ -33,6 +33,35 @@ public override void OnReceivedResponse(HttpMessage message) { throw Errors.ClientRequestIdMismatch(message.Response, echo.First(), original); } + + if (message.Request.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader) && + message.Request.Headers.Contains(Constants.StructuredMessage.StructuredContentLength)) + { + AssertStructuredMessageAcknowledgedPUT(message); + } + else if (message.Request.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) + { + AssertStructuredMessageAcknowledgedGET(message); + } + } + + private static void AssertStructuredMessageAcknowledgedPUT(HttpMessage message) + { + if (!message.Response.IsError && + !message.Response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) + { + throw Errors.StructuredMessageNotAcknowledgedPUT(message.Response); + } + } + + private static void AssertStructuredMessageAcknowledgedGET(HttpMessage message) + { + if (!message.Response.IsError && + !(message.Response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader) && + message.Response.Headers.Contains(Constants.StructuredMessage.StructuredContentLength))) + { + throw Errors.StructuredMessageNotAcknowledgedGET(message.Response); + } } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs index 31f121d414ea4..c8803ecf421e7 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; +using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -48,7 +50,7 @@ public static async Task WriteInternal( } } - public static Task CopyToInternal( + public static Task CopyToInternal( this Stream src, Stream dest, bool async, @@ -79,21 +81,33 @@ public static Task CopyToInternal( /// Cancellation token for the operation. /// /// - public static async Task CopyToInternal( + public static async Task CopyToInternal( this Stream src, Stream dest, int bufferSize, bool async, CancellationToken cancellationToken) { + using IDisposable _ = ArrayPool.Shared.RentDisposable(bufferSize, out byte[] buffer); + long totalRead = 0; + int read; if (async) { - await src.CopyToAsync(dest, bufferSize, cancellationToken).ConfigureAwait(false); + while (0 < (read = await src.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false))) + { + totalRead += read; + await dest.WriteAsync(buffer, 0, read, cancellationToken).ConfigureAwait(false); + } } else { - src.CopyTo(dest, bufferSize); + while (0 < (read = src.Read(buffer, 0, buffer.Length))) + { + totalRead += read; + dest.Write(buffer, 0, read); + } } + return totalRead; } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs new file mode 100644 index 0000000000000..a0a46837797b9 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.IO; +using Azure.Storage.Common; + +namespace Azure.Storage.Shared; + +internal static class StructuredMessage +{ + public const int Crc64Length = 8; + + [Flags] + public enum Flags + { + None = 0, + StorageCrc64 = 1, + } + + public static class V1_0 + { + public const byte MessageVersionByte = 1; + + public const int StreamHeaderLength = 13; + public const int StreamHeaderVersionOffset = 0; + public const int StreamHeaderMessageLengthOffset = 1; + public const int StreamHeaderFlagsOffset = 9; + public const int StreamHeaderSegmentCountOffset = 11; + + public const int SegmentHeaderLength = 10; + public const int SegmentHeaderNumOffset = 0; + public const int SegmentHeaderContentLengthOffset = 2; + + #region Stream Header + public static void ReadStreamHeader( + ReadOnlySpan buffer, + out long messageLength, + out Flags flags, + out int totalSegments) + { + Errors.AssertBufferExactSize(buffer, 13, nameof(buffer)); + if (buffer[StreamHeaderVersionOffset] != 1) + { + throw new InvalidDataException("Unrecognized version of structured message."); + } + messageLength = (long)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(StreamHeaderMessageLengthOffset, 8)); + flags = (Flags)BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(StreamHeaderFlagsOffset, 2)); + totalSegments = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(StreamHeaderSegmentCountOffset, 2)); + } + + public static int WriteStreamHeader( + Span buffer, + long messageLength, + Flags flags, + int totalSegments) + { + const int versionOffset = 0; + const int messageLengthOffset = 1; + const int flagsOffset = 9; + const int numSegmentsOffset = 11; + + Errors.AssertBufferMinimumSize(buffer, StreamHeaderLength, nameof(buffer)); + + buffer[versionOffset] = MessageVersionByte; + BinaryPrimitives.WriteUInt64LittleEndian(buffer.Slice(messageLengthOffset, 8), (ulong)messageLength); + BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(flagsOffset, 2), (ushort)flags); + BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(numSegmentsOffset, 2), (ushort)totalSegments); + + return StreamHeaderLength; + } + + /// + /// Gets stream header in a buffer rented from the provided ArrayPool. + /// + /// + /// Disposable to return the buffer to the pool. + /// + public static IDisposable GetStreamHeaderBytes( + ArrayPool pool, + out Memory bytes, + long messageLength, + Flags flags, + int totalSegments) + { + Argument.AssertNotNull(pool, nameof(pool)); + IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); + WriteStreamHeader(bytes.Span, messageLength, flags, totalSegments); + return disposable; + } + #endregion + + #region StreamFooter + public static int GetStreamFooterSize(Flags flags) + => flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; + + public static void ReadStreamFooter( + ReadOnlySpan buffer, + Flags flags, + out ulong crc64) + { + int expectedBufferSize = GetSegmentFooterSize(flags); + Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); + + crc64 = flags.HasFlag(Flags.StorageCrc64) ? buffer.ReadCrc64() : default; + } + + public static int WriteStreamFooter(Span buffer, ReadOnlySpan crc64 = default) + { + int requiredSpace = 0; + if (!crc64.IsEmpty) + { + Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); + requiredSpace += Crc64Length; + } + + Errors.AssertBufferMinimumSize(buffer, requiredSpace, nameof(buffer)); + int offset = 0; + if (!crc64.IsEmpty) + { + crc64.CopyTo(buffer.Slice(offset, Crc64Length)); + offset += Crc64Length; + } + + return offset; + } + + /// + /// Gets stream header in a buffer rented from the provided ArrayPool. + /// + /// + /// Disposable to return the buffer to the pool. + /// + public static IDisposable GetStreamFooterBytes( + ArrayPool pool, + out Memory bytes, + ReadOnlySpan crc64 = default) + { + Argument.AssertNotNull(pool, nameof(pool)); + IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); + WriteStreamFooter(bytes.Span, crc64); + return disposable; + } + #endregion + + #region SegmentHeader + public static void ReadSegmentHeader( + ReadOnlySpan buffer, + out int segmentNum, + out long contentLength) + { + Errors.AssertBufferExactSize(buffer, 10, nameof(buffer)); + segmentNum = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(0, 2)); + contentLength = (long)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(2, 8)); + } + + public static int WriteSegmentHeader(Span buffer, int segmentNum, long segmentLength) + { + const int segmentNumOffset = 0; + const int segmentLengthOffset = 2; + + Errors.AssertBufferMinimumSize(buffer, SegmentHeaderLength, nameof(buffer)); + + BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(segmentNumOffset, 2), (ushort)segmentNum); + BinaryPrimitives.WriteUInt64LittleEndian(buffer.Slice(segmentLengthOffset, 8), (ulong)segmentLength); + + return SegmentHeaderLength; + } + + /// + /// Gets segment header in a buffer rented from the provided ArrayPool. + /// + /// + /// Disposable to return the buffer to the pool. + /// + public static IDisposable GetSegmentHeaderBytes( + ArrayPool pool, + out Memory bytes, + int segmentNum, + long segmentLength) + { + Argument.AssertNotNull(pool, nameof(pool)); + IDisposable disposable = pool.RentAsMemoryDisposable(SegmentHeaderLength, out bytes); + WriteSegmentHeader(bytes.Span, segmentNum, segmentLength); + return disposable; + } + #endregion + + #region SegmentFooter + public static int GetSegmentFooterSize(Flags flags) + => flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; + + public static void ReadSegmentFooter( + ReadOnlySpan buffer, + Flags flags, + out ulong crc64) + { + int expectedBufferSize = GetSegmentFooterSize(flags); + Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); + + crc64 = flags.HasFlag(Flags.StorageCrc64) ? buffer.ReadCrc64() : default; + } + + public static int WriteSegmentFooter(Span buffer, ReadOnlySpan crc64 = default) + { + int requiredSpace = 0; + if (!crc64.IsEmpty) + { + Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); + requiredSpace += Crc64Length; + } + + Errors.AssertBufferMinimumSize(buffer, requiredSpace, nameof(buffer)); + int offset = 0; + if (!crc64.IsEmpty) + { + crc64.CopyTo(buffer.Slice(offset, Crc64Length)); + offset += Crc64Length; + } + + return offset; + } + + /// + /// Gets stream header in a buffer rented from the provided ArrayPool. + /// + /// + /// Disposable to return the buffer to the pool. + /// + public static IDisposable GetSegmentFooterBytes( + ArrayPool pool, + out Memory bytes, + ReadOnlySpan crc64 = default) + { + Argument.AssertNotNull(pool, nameof(pool)); + IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); + WriteSegmentFooter(bytes.Span, crc64); + return disposable; + } + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs new file mode 100644 index 0000000000000..22dfaef259972 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.Pipeline; + +namespace Azure.Storage.Shared; + +internal class StructuredMessageDecodingRetriableStream : Stream +{ + public class DecodedData + { + public ulong Crc { get; set; } + } + + private readonly Stream _innerRetriable; + private long _decodedBytesRead; + + private readonly StructuredMessage.Flags _expectedFlags; + private readonly List _decodedDatas; + private readonly Action _onComplete; + + private StorageCrc64HashAlgorithm _totalContentCrc; + + private readonly Func _decodingStreamFactory; + private readonly Func> _decodingAsyncStreamFactory; + + public StructuredMessageDecodingRetriableStream( + Stream initialDecodingStream, + StructuredMessageDecodingStream.RawDecodedData initialDecodedData, + StructuredMessage.Flags expectedFlags, + Func decodingStreamFactory, + Func> decodingAsyncStreamFactory, + Action onComplete, + ResponseClassifier responseClassifier, + int maxRetries) + { + _decodingStreamFactory = decodingStreamFactory; + _decodingAsyncStreamFactory = decodingAsyncStreamFactory; + _innerRetriable = RetriableStream.Create(initialDecodingStream, StreamFactory, StreamFactoryAsync, responseClassifier, maxRetries); + _decodedDatas = new() { initialDecodedData }; + _expectedFlags = expectedFlags; + _onComplete = onComplete; + + if (expectedFlags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + _totalContentCrc = StorageCrc64HashAlgorithm.Create(); + } + } + + private Stream StreamFactory(long _) + { + long offset = _decodedDatas.SelectMany(d => d.SegmentCrcs).Select(s => s.SegmentLen).Sum(); + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = _decodingStreamFactory(offset); + _decodedDatas.Add(decodedData); + FastForwardInternal(decodingStream, _decodedBytesRead - offset, false).EnsureCompleted(); + return decodingStream; + } + + private async ValueTask StreamFactoryAsync(long _) + { + long offset = _decodedDatas.SelectMany(d => d.SegmentCrcs).Select(s => s.SegmentLen).Sum(); + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = await _decodingAsyncStreamFactory(offset).ConfigureAwait(false); + _decodedDatas.Add(decodedData); + await FastForwardInternal(decodingStream, _decodedBytesRead - offset, true).ConfigureAwait(false); + return decodingStream; + } + + private static async ValueTask FastForwardInternal(Stream stream, long bytes, bool async) + { + using (ArrayPool.Shared.RentDisposable(4 * Constants.KB, out byte[] buffer)) + { + if (async) + { + while (bytes > 0) + { + bytes -= await stream.ReadAsync(buffer, 0, (int)Math.Min(bytes, buffer.Length)).ConfigureAwait(false); + } + } + else + { + while (bytes > 0) + { + bytes -= stream.Read(buffer, 0, (int)Math.Min(bytes, buffer.Length)); + } + } + } + } + + protected override void Dispose(bool disposing) + { + _decodedDatas.Clear(); + _innerRetriable.Dispose(); + } + + private void OnCompleted() + { + DecodedData final = new(); + if (_totalContentCrc != null) + { + final.Crc = ValidateCrc(); + } + _onComplete?.Invoke(final); + } + + private ulong ValidateCrc() + { + using IDisposable _ = ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf); + Span calculatedBytes = new(buf, 0, StructuredMessage.Crc64Length); + _totalContentCrc.GetCurrentHash(calculatedBytes); + ulong calculated = BinaryPrimitives.ReadUInt64LittleEndian(calculatedBytes); + + ulong reported = _decodedDatas.Count == 1 + ? _decodedDatas.First().TotalCrc.Value + : StorageCrc64Composer.Compose(_decodedDatas.SelectMany(d => d.SegmentCrcs)); + + if (calculated != reported) + { + Span reportedBytes = new(buf, calculatedBytes.Length, StructuredMessage.Crc64Length); + BinaryPrimitives.WriteUInt64LittleEndian(reportedBytes, reported); + throw Errors.ChecksumMismatch(calculatedBytes, reportedBytes); + } + + return calculated; + } + + #region Read + public override int Read(byte[] buffer, int offset, int count) + { + int read = _innerRetriable.Read(buffer, offset, count); + _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } + else + { + _totalContentCrc?.Append(new ReadOnlySpan(buffer, offset, read)); + } + return read; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int read = await _innerRetriable.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } + else + { + _totalContentCrc?.Append(new ReadOnlySpan(buffer, offset, read)); + } + return read; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override int Read(Span buffer) + { + int read = _innerRetriable.Read(buffer); + _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } + else + { + _totalContentCrc?.Append(buffer.Slice(0, read)); + } + return read; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int read = await _innerRetriable.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } + else + { + _totalContentCrc?.Append(buffer.Span.Slice(0, read)); + } + return read; + } +#endif + + public override int ReadByte() + { + int val = _innerRetriable.ReadByte(); + _decodedBytesRead += 1; + if (val == -1) + { + OnCompleted(); + } + return val; + } + + public override int EndRead(IAsyncResult asyncResult) + { + int read = _innerRetriable.EndRead(asyncResult); + _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } + return read; + } + #endregion + + #region Passthru + public override bool CanRead => _innerRetriable.CanRead; + + public override bool CanSeek => _innerRetriable.CanSeek; + + public override bool CanWrite => _innerRetriable.CanWrite; + + public override bool CanTimeout => _innerRetriable.CanTimeout; + + public override long Length => _innerRetriable.Length; + + public override long Position { get => _innerRetriable.Position; set => _innerRetriable.Position = value; } + + public override void Flush() => _innerRetriable.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => _innerRetriable.FlushAsync(cancellationToken); + + public override long Seek(long offset, SeekOrigin origin) => _innerRetriable.Seek(offset, origin); + + public override void SetLength(long value) => _innerRetriable.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) => _innerRetriable.Write(buffer, offset, count); + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => _innerRetriable.WriteAsync(buffer, offset, count, cancellationToken); + + public override void WriteByte(byte value) => _innerRetriable.WriteByte(value); + + public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback callback, object state) => _innerRetriable.BeginWrite(buffer, offset, count, callback, state); + + public override void EndWrite(IAsyncResult asyncResult) => _innerRetriable.EndWrite(asyncResult); + + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback callback, object state) => _innerRetriable.BeginRead(buffer, offset, count, callback, state); + + public override int ReadTimeout { get => _innerRetriable.ReadTimeout; set => _innerRetriable.ReadTimeout = value; } + + public override int WriteTimeout { get => _innerRetriable.WriteTimeout; set => _innerRetriable.WriteTimeout = value; } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override void Write(ReadOnlySpan buffer) => _innerRetriable.Write(buffer); + + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => _innerRetriable.WriteAsync(buffer, cancellationToken); +#endif + #endregion +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs new file mode 100644 index 0000000000000..e6b193ae18260 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -0,0 +1,542 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Common; + +namespace Azure.Storage.Shared; + +/// +/// Decodes a structured message stream as the data is read. +/// +/// +/// Wraps the inner stream in a , which avoids using its internal +/// buffer if individual Read() calls are larger than it. This ensures one of the three scenarios +/// +/// +/// Read buffer >= stream buffer: +/// There is enough space in the read buffer for inline metadata to be safely +/// extracted in only one read to the true inner stream. +/// +/// +/// Read buffer < next inline metadata: +/// The stream buffer has been activated, and we can read multiple small times from the inner stream +/// without multi-reading the real stream, even when partway through an existing stream buffer. +/// +/// +/// Else: +/// Same as #1, but also the already-allocated stream buffer has been used to slightly improve +/// resource churn when reading inner stream. +/// +/// +/// +internal class StructuredMessageDecodingStream : Stream +{ + internal class RawDecodedData + { + public long? InnerStreamLength { get; set; } + public int? TotalSegments { get; set; } + public StructuredMessage.Flags? Flags { get; set; } + public List<(ulong SegmentCrc, long SegmentLen)> SegmentCrcs { get; } = new(); + public ulong? TotalCrc { get; set; } + public bool DecodeCompleted { get; set; } + } + + private enum SMRegion + { + StreamHeader, + StreamFooter, + SegmentHeader, + SegmentFooter, + SegmentContent, + } + + private readonly Stream _innerBufferedStream; + + private byte[] _metadataBuffer = ArrayPool.Shared.Rent(Constants.KB); + private int _metadataBufferOffset = 0; + private int _metadataBufferLength = 0; + + private int _streamHeaderLength; + private int _streamFooterLength; + private int _segmentHeaderLength; + private int _segmentFooterLength; + + private long? _expectedInnerStreamLength; + + private bool _disposed; + + private readonly RawDecodedData _decodedData; + private StorageCrc64HashAlgorithm _totalContentCrc; + private StorageCrc64HashAlgorithm _segmentCrc; + + private readonly bool _validateChecksums; + + public override bool CanRead => true; + + public override bool CanWrite => false; + + public override bool CanSeek => false; + + public override bool CanTimeout => _innerBufferedStream.CanTimeout; + + public override int ReadTimeout => _innerBufferedStream.ReadTimeout; + + public override int WriteTimeout => _innerBufferedStream.WriteTimeout; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public static (Stream DecodedStream, RawDecodedData DecodedData) WrapStream( + Stream innerStream, + long? expextedStreamLength = default) + { + RawDecodedData data = new(); + return (new StructuredMessageDecodingStream(innerStream, data, expextedStreamLength), data); + } + + private StructuredMessageDecodingStream( + Stream innerStream, + RawDecodedData decodedData, + long? expectedStreamLength) + { + Argument.AssertNotNull(innerStream, nameof(innerStream)); + Argument.AssertNotNull(decodedData, nameof(decodedData)); + + _expectedInnerStreamLength = expectedStreamLength; + _innerBufferedStream = new BufferedStream(innerStream); + _decodedData = decodedData; + + // Assumes stream will be structured message 1.0. Will validate this when consuming stream. + _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; + _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; + + _validateChecksums = true; + } + + #region Write + public override void Flush() => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + #endregion + + #region Read + public override int Read(byte[] buf, int offset, int count) + { + int decodedRead; + int read; + do + { + read = _innerBufferedStream.Read(buf, offset, count); + _innerStreamConsumed += read; + decodedRead = Decode(new Span(buf, offset, read)); + } while (decodedRead <= 0 && read > 0); + + if (read <= 0) + { + AssertDecodeFinished(); + } + + return decodedRead; + } + + public override async Task ReadAsync(byte[] buf, int offset, int count, CancellationToken cancellationToken) + { + int decodedRead; + int read; + do + { + read = await _innerBufferedStream.ReadAsync(buf, offset, count, cancellationToken).ConfigureAwait(false); + _innerStreamConsumed += read; + decodedRead = Decode(new Span(buf, offset, read)); + } while (decodedRead <= 0 && read > 0); + + if (read <= 0) + { + AssertDecodeFinished(); + } + + return decodedRead; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override int Read(Span buf) + { + int decodedRead; + int read; + do + { + read = _innerBufferedStream.Read(buf); + _innerStreamConsumed += read; + decodedRead = Decode(buf.Slice(0, read)); + } while (decodedRead <= 0 && read > 0); + + if (read <= 0) + { + AssertDecodeFinished(); + } + + return decodedRead; + } + + public override async ValueTask ReadAsync(Memory buf, CancellationToken cancellationToken = default) + { + int decodedRead; + int read; + do + { + read = await _innerBufferedStream.ReadAsync(buf).ConfigureAwait(false); + _innerStreamConsumed += read; + decodedRead = Decode(buf.Slice(0, read).Span); + } while (decodedRead <= 0 && read > 0); + + if (read <= 0) + { + AssertDecodeFinished(); + } + + return decodedRead; + } +#endif + + private void AssertDecodeFinished() + { + if (_streamFooterLength > 0 && !_decodedData.DecodeCompleted) + { + throw Errors.InvalidStructuredMessage("Premature end of stream."); + } + _decodedData.DecodeCompleted = true; + } + + private long _innerStreamConsumed = 0; + private long _decodedContentConsumed = 0; + private SMRegion _currentRegion = SMRegion.StreamHeader; + private int _currentSegmentNum = 0; + private long _currentSegmentContentLength; + private long _currentSegmentContentRemaining; + private long CurrentRegionLength => _currentRegion switch + { + SMRegion.StreamHeader => _streamHeaderLength, + SMRegion.StreamFooter => _streamFooterLength, + SMRegion.SegmentHeader => _segmentHeaderLength, + SMRegion.SegmentFooter => _segmentFooterLength, + SMRegion.SegmentContent => _currentSegmentContentLength, + _ => 0, + }; + + /// + /// Decodes given bytes in place. Decoding based on internal stream position info. + /// Decoded data size will be less than or equal to encoded data length. + /// + /// + /// Length of the decoded data in . + /// + private int Decode(Span buffer) + { + if (buffer.IsEmpty) + { + return 0; + } + List<(int Offset, int Count)> gaps = new(); + + int bufferConsumed = ProcessMetadataBuffer(buffer); + + if (bufferConsumed > 0) + { + gaps.Add((0, bufferConsumed)); + } + + while (bufferConsumed < buffer.Length) + { + if (_currentRegion == SMRegion.SegmentContent) + { + int read = (int)Math.Min(buffer.Length - bufferConsumed, _currentSegmentContentRemaining); + _totalContentCrc?.Append(buffer.Slice(bufferConsumed, read)); + _segmentCrc?.Append(buffer.Slice(bufferConsumed, read)); + bufferConsumed += read; + _decodedContentConsumed += read; + _currentSegmentContentRemaining -= read; + if (_currentSegmentContentRemaining == 0) + { + _currentRegion = SMRegion.SegmentFooter; + } + } + else if (buffer.Length - bufferConsumed < CurrentRegionLength) + { + SavePartialMetadata(buffer.Slice(bufferConsumed)); + gaps.Add((bufferConsumed, buffer.Length - bufferConsumed)); + bufferConsumed = buffer.Length; + } + else + { + int processed = _currentRegion switch + { + SMRegion.StreamHeader => ProcessStreamHeader(buffer.Slice(bufferConsumed)), + SMRegion.StreamFooter => ProcessStreamFooter(buffer.Slice(bufferConsumed)), + SMRegion.SegmentHeader => ProcessSegmentHeader(buffer.Slice(bufferConsumed)), + SMRegion.SegmentFooter => ProcessSegmentFooter(buffer.Slice(bufferConsumed)), + _ => 0, + }; + // TODO surface error if processed is 0 + gaps.Add((bufferConsumed, processed)); + bufferConsumed += processed; + } + } + + if (gaps.Count == 0) + { + return buffer.Length; + } + + // gaps is already sorted by offset due to how it was assembled + int gap = 0; + for (int i = gaps.First().Offset; i < buffer.Length; i++) + { + if (gaps.Count > 0 && gaps.First().Offset == i) + { + int count = gaps.First().Count; + gap += count; + i += count - 1; + gaps.RemoveAt(0); + } + else + { + buffer[i - gap] = buffer[i]; + } + } + return buffer.Length - gap; + } + + /// + /// Processes metadata in the internal buffer, if any. Appends any necessary data + /// from the append buffer to complete metadata. + /// + /// + /// Bytes consumed from . + /// + private int ProcessMetadataBuffer(ReadOnlySpan append) + { + if (_metadataBufferLength == 0) + { + return 0; + } + if (_currentRegion == SMRegion.SegmentContent) + { + return 0; + } + int appended = 0; + if (_metadataBufferLength < CurrentRegionLength && append.Length > 0) + { + appended = Math.Min((int)CurrentRegionLength - _metadataBufferLength, append.Length); + SavePartialMetadata(append.Slice(0, appended)); + } + if (_metadataBufferLength == CurrentRegionLength) + { + Span metadata = new(_metadataBuffer, _metadataBufferOffset, (int)CurrentRegionLength); + switch (_currentRegion) + { + case SMRegion.StreamHeader: + ProcessStreamHeader(metadata); + break; + case SMRegion.StreamFooter: + ProcessStreamFooter(metadata); + break; + case SMRegion.SegmentHeader: + ProcessSegmentHeader(metadata); + break; + case SMRegion.SegmentFooter: + ProcessSegmentFooter(metadata); + break; + } + _metadataBufferOffset = 0; + _metadataBufferLength = 0; + } + return appended; + } + + private void SavePartialMetadata(ReadOnlySpan span) + { + // safety array resize w/ArrayPool + if (_metadataBufferLength + span.Length > _metadataBuffer.Length) + { + ResizeMetadataBuffer(2 * (_metadataBufferLength + span.Length)); + } + + // realign any existing content if necessary + if (_metadataBufferLength != 0 && _metadataBufferOffset != 0) + { + // don't use Array.Copy() to move elements in the same array + for (int i = 0; i < _metadataBufferLength; i++) + { + _metadataBuffer[i] = _metadataBuffer[i + _metadataBufferOffset]; + } + _metadataBufferOffset = 0; + } + + span.CopyTo(new Span(_metadataBuffer, _metadataBufferOffset + _metadataBufferLength, span.Length)); + _metadataBufferLength += span.Length; + } + + private int ProcessStreamHeader(ReadOnlySpan span) + { + StructuredMessage.V1_0.ReadStreamHeader( + span.Slice(0, _streamHeaderLength), + out long streamLength, + out StructuredMessage.Flags flags, + out int totalSegments); + + _decodedData.InnerStreamLength = streamLength; + _decodedData.Flags = flags; + _decodedData.TotalSegments = totalSegments; + + if (_expectedInnerStreamLength.HasValue && _expectedInnerStreamLength.Value != streamLength) + { + throw Errors.InvalidStructuredMessage("Unexpected message size."); + } + + if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + _segmentFooterLength = StructuredMessage.Crc64Length; + _streamFooterLength = StructuredMessage.Crc64Length; + if (_validateChecksums) + { + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + _totalContentCrc = StorageCrc64HashAlgorithm.Create(); + } + } + _currentRegion = SMRegion.SegmentHeader; + return _streamHeaderLength; + } + + private int ProcessStreamFooter(ReadOnlySpan span) + { + int footerLen = StructuredMessage.V1_0.GetStreamFooterSize(_decodedData.Flags.Value); + StructuredMessage.V1_0.ReadStreamFooter( + span.Slice(0, footerLen), + _decodedData.Flags.Value, + out ulong reportedCrc); + if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + if (_validateChecksums) + { + ValidateCrc64(_totalContentCrc, reportedCrc); + } + _decodedData.TotalCrc = reportedCrc; + } + + if (_innerStreamConsumed != _decodedData.InnerStreamLength) + { + throw Errors.InvalidStructuredMessage("Unexpected message size."); + } + if (_currentSegmentNum != _decodedData.TotalSegments) + { + throw Errors.InvalidStructuredMessage("Missing expected message segments."); + } + + _decodedData.DecodeCompleted = true; + return footerLen; + } + + private int ProcessSegmentHeader(ReadOnlySpan span) + { + StructuredMessage.V1_0.ReadSegmentHeader( + span.Slice(0, _segmentHeaderLength), + out int newSegNum, + out _currentSegmentContentLength); + _currentSegmentContentRemaining = _currentSegmentContentLength; + if (newSegNum != _currentSegmentNum + 1) + { + throw Errors.InvalidStructuredMessage("Unexpected segment number in structured message."); + } + _currentSegmentNum = newSegNum; + _currentRegion = SMRegion.SegmentContent; + return _segmentHeaderLength; + } + + private int ProcessSegmentFooter(ReadOnlySpan span) + { + int footerLen = StructuredMessage.V1_0.GetSegmentFooterSize(_decodedData.Flags.Value); + StructuredMessage.V1_0.ReadSegmentFooter( + span.Slice(0, footerLen), + _decodedData.Flags.Value, + out ulong reportedCrc); + if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + if (_validateChecksums) + { + ValidateCrc64(_segmentCrc, reportedCrc); + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + } + _decodedData.SegmentCrcs.Add((reportedCrc, _currentSegmentContentLength)); + } + _currentRegion = _currentSegmentNum == _decodedData.TotalSegments ? SMRegion.StreamFooter : SMRegion.SegmentHeader; + return footerLen; + } + + private static void ValidateCrc64(StorageCrc64HashAlgorithm calculation, ulong reported) + { + using IDisposable _ = ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf); + Span calculatedBytes = new(buf, 0, StructuredMessage.Crc64Length); + Span reportedBytes = new(buf, calculatedBytes.Length, StructuredMessage.Crc64Length); + calculation.GetCurrentHash(calculatedBytes); + reported.WriteCrc64(reportedBytes); + if (!calculatedBytes.SequenceEqual(reportedBytes)) + { + throw Errors.ChecksumMismatch(calculatedBytes, reportedBytes); + } + } + #endregion + + public override long Seek(long offset, SeekOrigin origin) + => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (_disposed) + { + return; + } + + if (disposing) + { + _innerBufferedStream.Dispose(); + _disposed = true; + } + } + + private void ResizeMetadataBuffer(int newSize) + { + byte[] newBuf = ArrayPool.Shared.Rent(newSize); + Array.Copy(_metadataBuffer, _metadataBufferOffset, newBuf, 0, _metadataBufferLength); + ArrayPool.Shared.Return(_metadataBuffer); + _metadataBuffer = newBuf; + } + + private void AlignMetadataBuffer() + { + if (_metadataBufferOffset != 0 && _metadataBufferLength != 0) + { + for (int i = 0; i < _metadataBufferLength; i++) + { + _metadataBuffer[i] = _metadataBuffer[_metadataBufferOffset + i]; + } + _metadataBufferOffset = 0; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs new file mode 100644 index 0000000000000..cb0ef340155ec --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs @@ -0,0 +1,545 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Pipeline; +using Azure.Storage.Common; + +namespace Azure.Storage.Shared; + +internal class StructuredMessageEncodingStream : Stream +{ + private readonly Stream _innerStream; + + private readonly int _streamHeaderLength; + private readonly int _streamFooterLength; + private readonly int _segmentHeaderLength; + private readonly int _segmentFooterLength; + private readonly int _segmentContentLength; + + private readonly StructuredMessage.Flags _flags; + private bool _disposed; + + private bool UseCrcSegment => _flags.HasFlag(StructuredMessage.Flags.StorageCrc64); + private readonly StorageCrc64HashAlgorithm _totalCrc; + private StorageCrc64HashAlgorithm _segmentCrc; + private readonly byte[] _segmentCrcs; + private int _latestSegmentCrcd = 0; + + #region Segments + /// + /// Gets the 1-indexed segment number the underlying stream is currently positioned in. + /// 1-indexed to match segment labelling as specified by SM spec. + /// + private int CurrentInnerSegment => (int)Math.Floor(_innerStream.Position / (float)_segmentContentLength) + 1; + + /// + /// Gets the 1-indexed segment number the encoded data stream is currently positioned in. + /// 1-indexed to match segment labelling as specified by SM spec. + /// + private int CurrentEncodingSegment + { + get + { + // edge case: always on final segment when at end of inner stream + if (_innerStream.Position == _innerStream.Length) + { + return TotalSegments; + } + // when writing footer, inner stream is positioned at next segment, + // but this stream is still writing the previous one + if (_currentRegion == SMRegion.SegmentFooter) + { + return CurrentInnerSegment - 1; + } + return CurrentInnerSegment; + } + } + + /// + /// Segment length including header and footer. + /// + private int SegmentTotalLength => _segmentHeaderLength + _segmentContentLength + _segmentFooterLength; + + private int TotalSegments => GetTotalSegments(_innerStream, _segmentContentLength); + private static int GetTotalSegments(Stream innerStream, long segmentContentLength) + { + return (int)Math.Ceiling(innerStream.Length / (float)segmentContentLength); + } + #endregion + + public override bool CanRead => true; + + public override bool CanWrite => false; + + public override bool CanSeek => _innerStream.CanSeek; + + public override bool CanTimeout => _innerStream.CanTimeout; + + public override int ReadTimeout => _innerStream.ReadTimeout; + + public override int WriteTimeout => _innerStream.WriteTimeout; + + public override long Length => + _streamHeaderLength + _streamFooterLength + + (_segmentHeaderLength + _segmentFooterLength) * TotalSegments + + _innerStream.Length; + + #region Position + private enum SMRegion + { + StreamHeader, + StreamFooter, + SegmentHeader, + SegmentFooter, + SegmentContent, + } + + private SMRegion _currentRegion = SMRegion.StreamHeader; + private int _currentRegionPosition = 0; + + private long _maxSeekPosition = 0; + + public override long Position + { + get + { + return _currentRegion switch + { + SMRegion.StreamHeader => _currentRegionPosition, + SMRegion.StreamFooter => _streamHeaderLength + + TotalSegments * (_segmentHeaderLength + _segmentFooterLength) + + _innerStream.Length + + _currentRegionPosition, + SMRegion.SegmentHeader => _innerStream.Position + + _streamHeaderLength + + (CurrentEncodingSegment - 1) * (_segmentHeaderLength + _segmentFooterLength) + + _currentRegionPosition, + SMRegion.SegmentFooter => _innerStream.Position + + _streamHeaderLength + + // Inner stream has moved to next segment but we're still writing the previous segment footer + CurrentEncodingSegment * (_segmentHeaderLength + _segmentFooterLength) - + _segmentFooterLength + _currentRegionPosition, + SMRegion.SegmentContent => _innerStream.Position + + _streamHeaderLength + + CurrentEncodingSegment * (_segmentHeaderLength + _segmentFooterLength) - + _segmentFooterLength, + _ => throw new InvalidDataException($"{nameof(StructuredMessageEncodingStream)} invalid state."), + }; + } + set + { + Argument.AssertInRange(value, 0, _maxSeekPosition, nameof(value)); + if (value < _streamHeaderLength) + { + _currentRegion = SMRegion.StreamHeader; + _currentRegionPosition = (int)value; + _innerStream.Position = 0; + return; + } + if (value >= Length - _streamFooterLength) + { + _currentRegion = SMRegion.StreamFooter; + _currentRegionPosition = (int)(value - (Length - _streamFooterLength)); + _innerStream.Position = _innerStream.Length; + return; + } + int newSegmentNum = 1 + (int)Math.Floor((value - _streamHeaderLength) / (double)(_segmentHeaderLength + _segmentFooterLength + _segmentContentLength)); + int segmentPosition = (int)(value - _streamHeaderLength - + ((newSegmentNum - 1) * (_segmentHeaderLength + _segmentFooterLength + _segmentContentLength))); + + if (segmentPosition < _segmentHeaderLength) + { + _currentRegion = SMRegion.SegmentHeader; + _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength); + _innerStream.Position = (newSegmentNum - 1) * _segmentContentLength; + return; + } + if (segmentPosition < _segmentHeaderLength + _segmentContentLength) + { + _currentRegion = SMRegion.SegmentContent; + _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength) - + _segmentHeaderLength; + _innerStream.Position = (newSegmentNum - 1) * _segmentContentLength + _currentRegionPosition; + return; + } + + _currentRegion = SMRegion.SegmentFooter; + _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength) - + _segmentHeaderLength - _segmentContentLength; + _innerStream.Position = newSegmentNum * _segmentContentLength; + } + } + #endregion + + public StructuredMessageEncodingStream( + Stream innerStream, + int segmentContentLength, + StructuredMessage.Flags flags) + { + Argument.AssertNotNull(innerStream, nameof(innerStream)); + if (innerStream.GetLengthOrDefault() == default) + { + throw new ArgumentException("Stream must have known length.", nameof(innerStream)); + } + if (innerStream.Position != 0) + { + throw new ArgumentException("Stream must be at starting position.", nameof(innerStream)); + } + // stream logic likely breaks down with segment length of 1; enforce >=2 rather than just positive number + // real world scenarios will probably use a minimum of tens of KB + Argument.AssertInRange(segmentContentLength, 2, int.MaxValue, nameof(segmentContentLength)); + + _flags = flags; + _segmentContentLength = segmentContentLength; + + _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; + _streamFooterLength = UseCrcSegment ? StructuredMessage.Crc64Length : 0; + _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; + _segmentFooterLength = UseCrcSegment ? StructuredMessage.Crc64Length : 0; + + if (UseCrcSegment) + { + _totalCrc = StorageCrc64HashAlgorithm.Create(); + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + _segmentCrcs = ArrayPool.Shared.Rent( + GetTotalSegments(innerStream, segmentContentLength) * StructuredMessage.Crc64Length); + innerStream = ChecksumCalculatingStream.GetReadStream(innerStream, span => + { + _totalCrc.Append(span); + _segmentCrc.Append(span); + }); + } + + _innerStream = innerStream; + } + + #region Write + public override void Flush() => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + #endregion + + #region Read + public override int Read(byte[] buffer, int offset, int count) + => ReadInternal(buffer, offset, count, async: false, cancellationToken: default).EnsureCompleted(); + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => await ReadInternal(buffer, offset, count, async: true, cancellationToken).ConfigureAwait(false); + + private async ValueTask ReadInternal(byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < count && Position < Length) + { + int subreadOffset = offset + totalRead; + int subreadCount = count - totalRead; + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += await ReadFromInnerStreamInternal( + buffer, subreadOffset, subreadCount, async, cancellationToken).ConfigureAwait(false); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override int Read(Span buffer) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < buffer.Length && Position < Length) + { + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(buffer.Slice(totalRead)); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += ReadFromInnerStream(buffer.Slice(totalRead)); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < buffer.Length && Position < Length) + { + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(buffer.Slice(totalRead).Span); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += await ReadFromInnerStreamAsync(buffer.Slice(totalRead), cancellationToken).ConfigureAwait(false); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } +#endif + + #region Read Headers/Footers + private int ReadFromStreamHeader(Span buffer) + { + int read = Math.Min(buffer.Length, _streamHeaderLength - _currentRegionPosition); + using IDisposable _ = StructuredMessage.V1_0.GetStreamHeaderBytes( + ArrayPool.Shared, out Memory headerBytes, Length, _flags, TotalSegments); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _streamHeaderLength) + { + _currentRegion = SMRegion.SegmentHeader; + _currentRegionPosition = 0; + } + + return read; + } + + private int ReadFromStreamFooter(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); + if (read <= 0) + { + return 0; + } + + using IDisposable _ = StructuredMessage.V1_0.GetStreamFooterBytes( + ArrayPool.Shared, + out Memory footerBytes, + crc64: UseCrcSegment + ? _totalCrc.GetCurrentHash() // TODO array pooling + : default); + footerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + return read; + } + + private int ReadFromSegmentHeader(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentHeaderLength - _currentRegionPosition); + using IDisposable _ = StructuredMessage.V1_0.GetSegmentHeaderBytes( + ArrayPool.Shared, + out Memory headerBytes, + CurrentInnerSegment, + Math.Min(_segmentContentLength, _innerStream.Length - _innerStream.Position)); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _segmentHeaderLength) + { + _currentRegion = SMRegion.SegmentContent; + _currentRegionPosition = 0; + } + + return read; + } + + private int ReadFromSegmentFooter(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); + if (read < 0) + { + return 0; + } + + using IDisposable _ = StructuredMessage.V1_0.GetSegmentFooterBytes( + ArrayPool.Shared, + out Memory headerBytes, + crc64: UseCrcSegment + ? new Span( + _segmentCrcs, + (CurrentEncodingSegment-1) * _totalCrc.HashLengthInBytes, + _totalCrc.HashLengthInBytes) + : default); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _segmentFooterLength) + { + _currentRegion = _innerStream.Position == _innerStream.Length + ? SMRegion.StreamFooter : SMRegion.SegmentHeader; + _currentRegionPosition = 0; + } + + return read; + } + #endregion + + #region ReadUnderlyingStream + private int MaxInnerStreamRead => _segmentContentLength - _currentRegionPosition; + + private void CleanupContentSegment() + { + if (_currentRegionPosition == _segmentContentLength || _innerStream.Position >= _innerStream.Length) + { + _currentRegion = SMRegion.SegmentFooter; + _currentRegionPosition = 0; + if (UseCrcSegment && CurrentEncodingSegment - 1 == _latestSegmentCrcd) + { + _segmentCrc.GetCurrentHash(new Span( + _segmentCrcs, + _latestSegmentCrcd * _segmentCrc.HashLengthInBytes, + _segmentCrc.HashLengthInBytes)); + _latestSegmentCrcd++; + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + } + } + } + + private async ValueTask ReadFromInnerStreamInternal( + byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) + { + int read = async + ? await _innerStream.ReadAsync(buffer, offset, Math.Min(count, MaxInnerStreamRead)).ConfigureAwait(false) + : _innerStream.Read(buffer, offset, Math.Min(count, MaxInnerStreamRead)); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + private int ReadFromInnerStream(Span buffer) + { + if (MaxInnerStreamRead < buffer.Length) + { + buffer = buffer.Slice(0, MaxInnerStreamRead); + } + int read = _innerStream.Read(buffer); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } + + private async ValueTask ReadFromInnerStreamAsync(Memory buffer, CancellationToken cancellationToken) + { + if (MaxInnerStreamRead < buffer.Length) + { + buffer = buffer.Slice(0, MaxInnerStreamRead); + } + int read = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } +#endif + #endregion + + // don't allow stream to seek too far forward. track how far the stream has been naturally read. + private void UpdateLatestPosition() + { + if (_maxSeekPosition < Position) + { + _maxSeekPosition = Position; + } + } + #endregion + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + break; + case SeekOrigin.Current: + Position += offset; + break; + case SeekOrigin.End: + Position = Length + offset; + break; + } + return Position; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (_disposed) + { + return; + } + + if (disposing) + { + _innerStream.Dispose(); + _disposed = true; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs new file mode 100644 index 0000000000000..3569ef4339735 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs @@ -0,0 +1,451 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Pipeline; +using Azure.Storage.Common; + +namespace Azure.Storage.Shared; + +internal class StructuredMessagePrecalculatedCrcWrapperStream : Stream +{ + private readonly Stream _innerStream; + + private readonly int _streamHeaderLength; + private readonly int _streamFooterLength; + private readonly int _segmentHeaderLength; + private readonly int _segmentFooterLength; + + private bool _disposed; + + private readonly byte[] _crc; + + public override bool CanRead => true; + + public override bool CanWrite => false; + + public override bool CanSeek => _innerStream.CanSeek; + + public override bool CanTimeout => _innerStream.CanTimeout; + + public override int ReadTimeout => _innerStream.ReadTimeout; + + public override int WriteTimeout => _innerStream.WriteTimeout; + + public override long Length => + _streamHeaderLength + _streamFooterLength + + _segmentHeaderLength + _segmentFooterLength + + _innerStream.Length; + + #region Position + private enum SMRegion + { + StreamHeader, + StreamFooter, + SegmentHeader, + SegmentFooter, + SegmentContent, + } + + private SMRegion _currentRegion = SMRegion.StreamHeader; + private int _currentRegionPosition = 0; + + private long _maxSeekPosition = 0; + + public override long Position + { + get + { + return _currentRegion switch + { + SMRegion.StreamHeader => _currentRegionPosition, + SMRegion.SegmentHeader => _innerStream.Position + + _streamHeaderLength + + _currentRegionPosition, + SMRegion.SegmentContent => _streamHeaderLength + + _segmentHeaderLength + + _innerStream.Position, + SMRegion.SegmentFooter => _streamHeaderLength + + _segmentHeaderLength + + _innerStream.Length + + _currentRegionPosition, + SMRegion.StreamFooter => _streamHeaderLength + + _segmentHeaderLength + + _innerStream.Length + + _segmentFooterLength + + _currentRegionPosition, + _ => throw new InvalidDataException($"{nameof(StructuredMessageEncodingStream)} invalid state."), + }; + } + set + { + Argument.AssertInRange(value, 0, _maxSeekPosition, nameof(value)); + if (value < _streamHeaderLength) + { + _currentRegion = SMRegion.StreamHeader; + _currentRegionPosition = (int)value; + _innerStream.Position = 0; + return; + } + if (value < _streamHeaderLength + _segmentHeaderLength) + { + _currentRegion = SMRegion.SegmentHeader; + _currentRegionPosition = (int)(value - _streamHeaderLength); + _innerStream.Position = 0; + return; + } + if (value < _streamHeaderLength + _segmentHeaderLength + _innerStream.Length) + { + _currentRegion = SMRegion.SegmentContent; + _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength); + _innerStream.Position = value - _streamHeaderLength - _segmentHeaderLength; + return; + } + if (value < _streamHeaderLength + _segmentHeaderLength + _innerStream.Length + _segmentFooterLength) + { + _currentRegion = SMRegion.SegmentFooter; + _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength - _innerStream.Length); + _innerStream.Position = _innerStream.Length; + return; + } + + _currentRegion = SMRegion.StreamFooter; + _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength - _innerStream.Length - _segmentFooterLength); + _innerStream.Position = _innerStream.Length; + } + } + #endregion + + public StructuredMessagePrecalculatedCrcWrapperStream( + Stream innerStream, + ReadOnlySpan precalculatedCrc) + { + Argument.AssertNotNull(innerStream, nameof(innerStream)); + if (innerStream.GetLengthOrDefault() == default) + { + throw new ArgumentException("Stream must have known length.", nameof(innerStream)); + } + if (innerStream.Position != 0) + { + throw new ArgumentException("Stream must be at starting position.", nameof(innerStream)); + } + + _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; + _streamFooterLength = StructuredMessage.Crc64Length; + _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; + _segmentFooterLength = StructuredMessage.Crc64Length; + + _crc = ArrayPool.Shared.Rent(StructuredMessage.Crc64Length); + precalculatedCrc.CopyTo(_crc); + + _innerStream = innerStream; + } + + #region Write + public override void Flush() => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + #endregion + + #region Read + public override int Read(byte[] buffer, int offset, int count) + => ReadInternal(buffer, offset, count, async: false, cancellationToken: default).EnsureCompleted(); + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => await ReadInternal(buffer, offset, count, async: true, cancellationToken).ConfigureAwait(false); + + private async ValueTask ReadInternal(byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < count && Position < Length) + { + int subreadOffset = offset + totalRead; + int subreadCount = count - totalRead; + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += await ReadFromInnerStreamInternal( + buffer, subreadOffset, subreadCount, async, cancellationToken).ConfigureAwait(false); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override int Read(Span buffer) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < buffer.Length && Position < Length) + { + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(buffer.Slice(totalRead)); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += ReadFromInnerStream(buffer.Slice(totalRead)); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < buffer.Length && Position < Length) + { + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(buffer.Slice(totalRead).Span); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += await ReadFromInnerStreamAsync(buffer.Slice(totalRead), cancellationToken).ConfigureAwait(false); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } +#endif + + #region Read Headers/Footers + private int ReadFromStreamHeader(Span buffer) + { + int read = Math.Min(buffer.Length, _streamHeaderLength - _currentRegionPosition); + using IDisposable _ = StructuredMessage.V1_0.GetStreamHeaderBytes( + ArrayPool.Shared, + out Memory headerBytes, + Length, + StructuredMessage.Flags.StorageCrc64, + totalSegments: 1); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _streamHeaderLength) + { + _currentRegion = SMRegion.SegmentHeader; + _currentRegionPosition = 0; + } + + return read; + } + + private int ReadFromStreamFooter(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); + if (read <= 0) + { + return 0; + } + + using IDisposable _ = StructuredMessage.V1_0.GetStreamFooterBytes( + ArrayPool.Shared, + out Memory footerBytes, + new ReadOnlySpan(_crc, 0, StructuredMessage.Crc64Length)); + footerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + return read; + } + + private int ReadFromSegmentHeader(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentHeaderLength - _currentRegionPosition); + using IDisposable _ = StructuredMessage.V1_0.GetSegmentHeaderBytes( + ArrayPool.Shared, + out Memory headerBytes, + segmentNum: 1, + _innerStream.Length); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _segmentHeaderLength) + { + _currentRegion = SMRegion.SegmentContent; + _currentRegionPosition = 0; + } + + return read; + } + + private int ReadFromSegmentFooter(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); + if (read < 0) + { + return 0; + } + + using IDisposable _ = StructuredMessage.V1_0.GetSegmentFooterBytes( + ArrayPool.Shared, + out Memory headerBytes, + new ReadOnlySpan(_crc, 0, StructuredMessage.Crc64Length)); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _segmentFooterLength) + { + _currentRegion = _innerStream.Position == _innerStream.Length + ? SMRegion.StreamFooter : SMRegion.SegmentHeader; + _currentRegionPosition = 0; + } + + return read; + } + #endregion + + #region ReadUnderlyingStream + private void CleanupContentSegment() + { + if (_innerStream.Position >= _innerStream.Length) + { + _currentRegion = SMRegion.SegmentFooter; + _currentRegionPosition = 0; + } + } + + private async ValueTask ReadFromInnerStreamInternal( + byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) + { + int read = async + ? await _innerStream.ReadAsync(buffer, offset, count).ConfigureAwait(false) + : _innerStream.Read(buffer, offset, count); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + private int ReadFromInnerStream(Span buffer) + { + int read = _innerStream.Read(buffer); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } + + private async ValueTask ReadFromInnerStreamAsync(Memory buffer, CancellationToken cancellationToken) + { + int read = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } +#endif + #endregion + + // don't allow stream to seek too far forward. track how far the stream has been naturally read. + private void UpdateLatestPosition() + { + if (_maxSeekPosition < Position) + { + _maxSeekPosition = Position; + } + } + #endregion + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + break; + case SeekOrigin.Current: + Position += offset; + break; + case SeekOrigin.End: + Position = Length + offset; + break; + } + return Position; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (_disposed) + { + return; + } + + if (disposing) + { + ArrayPool.Shared.Return(_crc); + _innerStream.Dispose(); + _disposed = true; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs index af21588b4ae09..763d385240383 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs @@ -9,14 +9,7 @@ public static StorageChecksumAlgorithm ResolveAuto(this StorageChecksumAlgorithm { if (checksumAlgorithm == StorageChecksumAlgorithm.Auto) { -#if BlobSDK || DataLakeSDK || CommonSDK return StorageChecksumAlgorithm.StorageCrc64; -#elif FileSDK // file shares don't support crc64 - return StorageChecksumAlgorithm.MD5; -#else - throw new System.NotSupportedException( - $"{typeof(TransferValidationOptionsExtensions).FullName}.{nameof(ResolveAuto)} is not supported."); -#endif } return checksumAlgorithm; } diff --git a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj index 5db86ebee984b..2863b85f6feb2 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj +++ b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj @@ -13,9 +13,12 @@ + + + @@ -28,6 +31,7 @@ + @@ -46,6 +50,11 @@ + + + + + diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs index 7411eb1499312..f4e4b92ed73c4 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs @@ -15,6 +15,7 @@ internal class FaultyStream : Stream private readonly Exception _exceptionToRaise; private int _remainingExceptions; private Action _onFault; + private long _position = 0; public FaultyStream( Stream innerStream, @@ -40,7 +41,7 @@ public FaultyStream( public override long Position { - get => _innerStream.Position; + get => CanSeek ? _innerStream.Position : _position; set => _innerStream.Position = value; } @@ -53,7 +54,9 @@ public override int Read(byte[] buffer, int offset, int count) { if (_remainingExceptions == 0 || Position + count <= _raiseExceptionAt || _raiseExceptionAt >= _innerStream.Length) { - return _innerStream.Read(buffer, offset, count); + int read = _innerStream.Read(buffer, offset, count); + _position += read; + return read; } else { @@ -61,11 +64,13 @@ public override int Read(byte[] buffer, int offset, int count) } } - public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { if (_remainingExceptions == 0 || Position + count <= _raiseExceptionAt || _raiseExceptionAt >= _innerStream.Length) { - return _innerStream.ReadAsync(buffer, offset, count, cancellationToken); + int read = await _innerStream.ReadAsync(buffer, offset, count, cancellationToken); + _position += read; + return read; } else { diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs new file mode 100644 index 0000000000000..828c41179bba3 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using Azure.Core; +using Azure.Core.Pipeline; +using Azure.Storage.Shared; + +namespace Azure.Storage.Test.Shared +{ + internal class ObserveStructuredMessagePolicy : HttpPipelineSynchronousPolicy + { + private readonly HashSet _requestScopes = new(); + + private readonly HashSet _responseScopes = new(); + + public ObserveStructuredMessagePolicy() + { + } + + public override void OnSendingRequest(HttpMessage message) + { + if (_requestScopes.Count > 0) + { + byte[] encodedContent; + byte[] underlyingContent; + StructuredMessageDecodingStream.RawDecodedData decodedData; + using (MemoryStream ms = new()) + { + message.Request.Content.WriteTo(ms, default); + encodedContent = ms.ToArray(); + using (MemoryStream ms2 = new()) + { + (Stream s, decodedData) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedContent)); + s.CopyTo(ms2); + underlyingContent = ms2.ToArray(); + } + } + } + } + + public override void OnReceivedResponse(HttpMessage message) + { + } + + public IDisposable CheckRequestScope() => CheckMessageScope.CheckRequestScope(this); + + public IDisposable CheckResponseScope() => CheckMessageScope.CheckResponseScope(this); + + private class CheckMessageScope : IDisposable + { + private bool _isRequestScope; + private ObserveStructuredMessagePolicy _policy; + + public static CheckMessageScope CheckRequestScope(ObserveStructuredMessagePolicy policy) + { + CheckMessageScope result = new() + { + _isRequestScope = true, + _policy = policy + }; + result._policy._requestScopes.Add(result); + return result; + } + + public static CheckMessageScope CheckResponseScope(ObserveStructuredMessagePolicy policy) + { + CheckMessageScope result = new() + { + _isRequestScope = false, + _policy = policy + }; + result._policy._responseScopes.Add(result); + return result; + } + + public void Dispose() + { + (_isRequestScope ? _policy._requestScopes : _policy._responseScopes).Remove(this); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs new file mode 100644 index 0000000000000..ad395e862f827 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Linq; +using System.Text; +using Azure.Core; +using NUnit.Framework; + +namespace Azure.Storage; + +public static partial class RequestExtensions +{ + public static string AssertHeaderPresent(this Request request, string headerName) + { + if (request.Headers.TryGetValue(headerName, out string value)) + { + return headerName == Constants.StructuredMessage.StructuredMessageHeader ? null : value; + } + StringBuilder sb = new StringBuilder() + .AppendLine($"`{headerName}` expected on request but was not found.") + .AppendLine($"{request.Method} {request.Uri}") + .AppendLine(string.Join("\n", request.Headers.Select(h => $"{h.Name}: {h.Value}s"))) + ; + Assert.Fail(sb.ToString()); + return null; + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs index f4198e9dfd532..7e6c78117f53b 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs @@ -14,7 +14,7 @@ internal class TamperStreamContentsPolicy : HttpPipelineSynchronousPolicy /// /// Default tampering that changes the first byte of the stream. /// - private static readonly Func _defaultStreamTransform = stream => + private static Func GetTamperByteStreamTransform(long position) => stream => { if (stream is not MemoryStream) { @@ -23,10 +23,10 @@ internal class TamperStreamContentsPolicy : HttpPipelineSynchronousPolicy stream = buffer; } - stream.Position = 0; + stream.Position = position; var firstByte = stream.ReadByte(); - stream.Position = 0; + stream.Position = position; stream.WriteByte((byte)((firstByte + 1) % byte.MaxValue)); stream.Position = 0; @@ -37,9 +37,12 @@ internal class TamperStreamContentsPolicy : HttpPipelineSynchronousPolicy public TamperStreamContentsPolicy(Func streamTransform = default) { - _streamTransform = streamTransform ?? _defaultStreamTransform; + _streamTransform = streamTransform ?? GetTamperByteStreamTransform(0); } + public static TamperStreamContentsPolicy TamperByteAt(long position) + => new(GetTamperByteStreamTransform(position)); + public bool TransformRequestBody { get; set; } public bool TransformResponseBody { get; set; } diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index c18492d2fb4dd..248acf8811960 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -5,10 +5,13 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Threading.Tasks; using Azure.Core; +using Azure.Core.Diagnostics; +using Azure.Core.Pipeline; using Azure.Core.TestFramework; -using FastSerialization; +using Azure.Storage.Shared; using NUnit.Framework; namespace Azure.Storage.Test.Shared @@ -190,21 +193,15 @@ protected string GetNewResourceName() /// The actual checksum value expected to be on the request, if known. Defaults to no specific value expected or checked. /// /// An assertion to put into a pipeline policy. - internal static Action GetRequestChecksumAssertion(StorageChecksumAlgorithm algorithm, Func isChecksumExpected = default, byte[] expectedChecksum = default) + internal static Action GetRequestChecksumHeaderAssertion(StorageChecksumAlgorithm algorithm, Func isChecksumExpected = default, byte[] expectedChecksum = default) { // action to assert a request header is as expected - void AssertChecksum(RequestHeaders headers, string headerName) + void AssertChecksum(Request req, string headerName) { - if (headers.TryGetValue(headerName, out string checksum)) + string checksum = req.AssertHeaderPresent(headerName); + if (expectedChecksum != default) { - if (expectedChecksum != default) - { - Assert.AreEqual(Convert.ToBase64String(expectedChecksum), checksum); - } - } - else - { - Assert.Fail($"{headerName} expected on request but was not found."); + Assert.AreEqual(Convert.ToBase64String(expectedChecksum), checksum); } }; @@ -219,14 +216,39 @@ void AssertChecksum(RequestHeaders headers, string headerName) switch (algorithm.ResolveAuto()) { case StorageChecksumAlgorithm.MD5: - AssertChecksum(request.Headers, "Content-MD5"); + AssertChecksum(request, "Content-MD5"); break; case StorageChecksumAlgorithm.StorageCrc64: - AssertChecksum(request.Headers, "x-ms-content-crc64"); + AssertChecksum(request, Constants.StructuredMessage.StructuredMessageHeader); break; default: - throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumAssertion)}."); + throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); + } + }; + } + + internal static Action GetRequestStructuredMessageAssertion( + StructuredMessage.Flags flags, + Func isStructuredMessageExpected = default, + long? structuredContentSegmentLength = default) + { + return request => + { + // filter some requests out with predicate + if (isStructuredMessageExpected != default && !isStructuredMessageExpected(request)) + { + return; } + + Assert.That(request.Headers.TryGetValue("x-ms-structured-body", out string structuredBody)); + Assert.That(structuredBody, Does.Contain("XSM/1.0")); + if (flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + Assert.That(structuredBody, Does.Contain("crc64")); + } + + Assert.That(request.Headers.TryGetValue("Content-Length", out string contentLength)); + Assert.That(request.Headers.TryGetValue("x-ms-structured-content-length", out string structuredContentLength)); }; } @@ -278,32 +300,66 @@ void AssertChecksum(ResponseHeaders headers, string headerName) AssertChecksum(response.Headers, "Content-MD5"); break; case StorageChecksumAlgorithm.StorageCrc64: - AssertChecksum(response.Headers, "x-ms-content-crc64"); + AssertChecksum(response.Headers, Constants.StructuredMessage.StructuredMessageHeader); break; default: - throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumAssertion)}."); + throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); } }; } + internal static Action GetResponseStructuredMessageAssertion( + StructuredMessage.Flags flags, + Func isStructuredMessageExpected = default) + { + return response => + { + // filter some requests out with predicate + if (isStructuredMessageExpected != default && !isStructuredMessageExpected(response)) + { + return; + } + + Assert.That(response.Headers.TryGetValue("x-ms-structured-body", out string structuredBody)); + Assert.That(structuredBody, Does.Contain("XSM/1.0")); + if (flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + Assert.That(structuredBody, Does.Contain("crc64")); + } + + Assert.That(response.Headers.TryGetValue("Content-Length", out string contentLength)); + Assert.That(response.Headers.TryGetValue("x-ms-structured-content-length", out string structuredContentLength)); + }; + } + /// /// Asserts the service returned an error that expected checksum did not match checksum on upload. /// /// Async action to upload data to service. /// Checksum algorithm used. - internal static void AssertWriteChecksumMismatch(AsyncTestDelegate writeAction, StorageChecksumAlgorithm algorithm) + internal static void AssertWriteChecksumMismatch( + AsyncTestDelegate writeAction, + StorageChecksumAlgorithm algorithm, + bool expectStructuredMessage = false) { var exception = ThrowsOrInconclusiveAsync(writeAction); - switch (algorithm.ResolveAuto()) + if (expectStructuredMessage) { - case StorageChecksumAlgorithm.MD5: - Assert.AreEqual("Md5Mismatch", exception.ErrorCode); - break; - case StorageChecksumAlgorithm.StorageCrc64: - Assert.AreEqual("Crc64Mismatch", exception.ErrorCode); - break; - default: - throw new ArgumentException("Test arguments contain bad algorithm specifier."); + Assert.That(exception.ErrorCode, Is.EqualTo("Crc64Mismatch")); + } + else + { + switch (algorithm.ResolveAuto()) + { + case StorageChecksumAlgorithm.MD5: + Assert.That(exception.ErrorCode, Is.EqualTo("Md5Mismatch")); + break; + case StorageChecksumAlgorithm.StorageCrc64: + Assert.That(exception.ErrorCode, Is.EqualTo("Crc64Mismatch")); + break; + default: + throw new ArgumentException("Test arguments contain bad algorithm specifier."); + } } } #endregion @@ -348,6 +404,7 @@ public virtual async Task UploadPartitionSuccessfulHashComputation(StorageChecks await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); // Arrange + bool expectStructuredMessage = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64; const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var validationOptions = new UploadTransferValidationOptions @@ -356,7 +413,10 @@ public virtual async Task UploadPartitionSuccessfulHashComputation(StorageChecks }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm)); + var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) + : GetRequestChecksumHeaderAssertion(algorithm); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -406,7 +466,11 @@ public virtual async Task UploadPartitionUsePrecalculatedHash(StorageChecksumAlg }; // make pipeline assertion for checking precalculated checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm, expectedChecksum: precalculatedChecksum)); + // precalculated partition upload will never use structured message. always check header + var assertion = GetRequestChecksumHeaderAssertion( + algorithm, + expectedChecksum: algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? default : precalculatedChecksum); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -423,12 +487,12 @@ public virtual async Task UploadPartitionUsePrecalculatedHash(StorageChecksumAlg AsyncTestDelegate operation = async () => await UploadPartitionAsync(client, stream, validationOptions); // Assert - AssertWriteChecksumMismatch(operation, algorithm); + AssertWriteChecksumMismatch(operation, algorithm, algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64); } } [TestCaseSource(nameof(GetValidationAlgorithms))] - public virtual async Task UploadPartitionMismatchedHashThrows(StorageChecksumAlgorithm algorithm) + public virtual async Task UploadPartitionTamperedStreamThrows(StorageChecksumAlgorithm algorithm) { await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); @@ -441,7 +505,7 @@ public virtual async Task UploadPartitionMismatchedHashThrows(StorageChecksumAlg }; // Tamper with stream contents in the pipeline to simulate silent failure in the transit layer - var streamTamperPolicy = new TamperStreamContentsPolicy(); + var streamTamperPolicy = TamperStreamContentsPolicy.TamperByteAt(100); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(streamTamperPolicy, HttpPipelinePosition.PerCall); @@ -456,9 +520,10 @@ public virtual async Task UploadPartitionMismatchedHashThrows(StorageChecksumAlg // Act streamTamperPolicy.TransformRequestBody = true; AsyncTestDelegate operation = async () => await UploadPartitionAsync(client, stream, validationOptions); - + using var listener = AzureEventSourceListener.CreateConsoleLogger(); // Assert - AssertWriteChecksumMismatch(operation, algorithm); + AssertWriteChecksumMismatch(operation, algorithm, + expectStructuredMessage: algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64); } } @@ -473,7 +538,10 @@ public virtual async Task UploadPartitionUsesDefaultClientValidationOptions( var data = GetRandomBuffer(dataLength); // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(clientAlgorithm)); + var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) + : GetRequestChecksumHeaderAssertion(clientAlgorithm); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -512,7 +580,10 @@ public virtual async Task UploadPartitionOverwritesDefaultClientValidationOption }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(overrideAlgorithm)); + var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) + : GetRequestChecksumHeaderAssertion(overrideAlgorithm); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -555,10 +626,14 @@ public virtual async Task UploadPartitionDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains("x-ms-content-crc64")) + if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } + if (request.Headers.Contains("x-ms-structured-body")) + { + Assert.Fail($"Structured body used when none expected."); + } }); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -601,9 +676,11 @@ public virtual async Task OpenWriteSuccessfulHashComputation( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); + //ObserveStructuredMessagePolicy observe = new(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); + //clientOptions.AddPolicy(observe, HttpPipelinePosition.BeforeTransport); var client = await GetResourceClientAsync( disposingContainer.Container, @@ -616,6 +693,7 @@ public virtual async Task OpenWriteSuccessfulHashComputation( using var writeStream = await OpenWriteAsync(client, validationOptions, streamBufferSize); // Assert + //using var obsv = observe.CheckRequestScope(); using (checksumPipelineAssertion.CheckRequestScope()) { foreach (var _ in Enumerable.Range(0, streamWrites)) @@ -644,7 +722,7 @@ public virtual async Task OpenWriteMismatchedHashThrows(StorageChecksumAlgorithm // Tamper with stream contents in the pipeline to simulate silent failure in the transit layer var clientOptions = ClientBuilder.GetOptions(); - var tamperPolicy = new TamperStreamContentsPolicy(); + var tamperPolicy = TamperStreamContentsPolicy.TamperByteAt(100); clientOptions.AddPolicy(tamperPolicy, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync( @@ -682,7 +760,7 @@ public virtual async Task OpenWriteUsesDefaultClientValidationOptions( var data = GetRandomBuffer(dataLength); // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(clientAlgorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(clientAlgorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -726,7 +804,7 @@ public virtual async Task OpenWriteOverwritesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(overrideAlgorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(overrideAlgorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -774,7 +852,7 @@ public virtual async Task OpenWriteDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains("x-ms-content-crc64")) + if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -886,7 +964,7 @@ public virtual async Task ParallelUploadSplitSuccessfulHashComputation(StorageCh // make pipeline assertion for checking checksum was present on upload var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); + checkRequest: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -923,8 +1001,10 @@ public virtual async Task ParallelUploadOneShotSuccessfulHashComputation(Storage }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); + var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) + : GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -981,7 +1061,7 @@ public virtual async Task ParallelUploadPrecalculatedComposableHashAccepted(Stor PrecalculatedChecksum = hash }; - var client = await GetResourceClientAsync(disposingContainer.Container, dataLength); + var client = await GetResourceClientAsync(disposingContainer.Container, dataLength, createResource: true); // Act await DoesNotThrowOrInconclusiveAsync( @@ -1011,8 +1091,10 @@ public virtual async Task ParallelUploadUsesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion( - clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); + var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) + : GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1063,8 +1145,10 @@ public virtual async Task ParallelUploadOverwritesDefaultClientValidationOptions }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion( - overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); + var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) + : GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1119,7 +1203,7 @@ public virtual async Task ParallelUploadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains("x-ms-content-crc64")) + if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -1184,15 +1268,17 @@ public virtual async Task ParallelDownloadSuccessfulHashVerification( }; // Act - var dest = new MemoryStream(); + byte[] dest; + using (MemoryStream ms = new()) using (checksumPipelineAssertion.CheckRequestScope()) { - await ParallelDownloadAsync(client, dest, validationOptions, transferOptions); + await ParallelDownloadAsync(client, ms, validationOptions, transferOptions); + dest = ms.ToArray(); } // Assert // Assertion was in the pipeline and the SDK not throwing means the checksum was validated - Assert.IsTrue(dest.ToArray().SequenceEqual(data)); + Assert.IsTrue(dest.SequenceEqual(data)); } [Test] @@ -1357,7 +1443,7 @@ public virtual async Task ParallelDownloadDisablesDefaultClientValidationOptions { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains("x-ms-content-crc64")) + if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -1565,7 +1651,7 @@ public virtual async Task OpenReadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains("x-ms-content-crc64")) + if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -1615,7 +1701,7 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; // Act - var dest = new MemoryStream(); + using var dest = new MemoryStream(); var response = await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); // Assert @@ -1626,13 +1712,71 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains("x-ms-content-crc64")); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); break; } - Assert.IsTrue(dest.ToArray().SequenceEqual(data)); + var result = dest.ToArray(); + Assert.IsTrue(result.SequenceEqual(data)); + } + + [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader, false, false)] + [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader-1, false, false)] + [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader+1, true, false)] + [TestCase(StorageChecksumAlgorithm.MD5, Constants.StructuredMessage.MaxDownloadCrcWithHeader+1, false, true)] + public virtual async Task DownloadApporpriatelyUsesStructuredMessage( + StorageChecksumAlgorithm algorithm, + int? downloadLen, + bool expectStructuredMessage, + bool expectThrow) + { + await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); + + // Arrange + const int dataLength = Constants.KB; + var data = GetRandomBuffer(dataLength); + + var resourceName = GetNewResourceName(); + var client = await GetResourceClientAsync( + disposingContainer.Container, + resourceLength: dataLength, + createResource: true, + resourceName: resourceName); + await SetupDataAsync(client, new MemoryStream(data)); + + // make pipeline assertion for checking checksum was present on download + HttpPipelinePolicy checksumPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: expectStructuredMessage + ? GetResponseStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64) + : GetResponseChecksumAssertion(algorithm)); + TClientOptions clientOptions = ClientBuilder.GetOptions(); + clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); + + client = await GetResourceClientAsync( + disposingContainer.Container, + resourceLength: dataLength, + resourceName: resourceName, + createResource: false, + downloadAlgorithm: algorithm, + options: clientOptions); + + var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; + + // Act + var dest = new MemoryStream(); + AsyncTestDelegate operation = async () => await DownloadPartitionAsync( + client, dest, validationOptions, downloadLen.HasValue ? new HttpRange(length: downloadLen.Value) : default); + // Assert (policies checked use of content validation) + if (expectThrow) + { + Assert.That(operation, Throws.TypeOf()); + } + else + { + Assert.That(operation, Throws.Nothing); + Assert.IsTrue(dest.ToArray().SequenceEqual(data)); + } } [Test, Combinatorial] @@ -1658,7 +1802,9 @@ public virtual async Task DownloadHashMismatchThrows( // alter response contents in pipeline, forcing a checksum mismatch on verification step var clientOptions = ClientBuilder.GetOptions(); - clientOptions.AddPolicy(new TamperStreamContentsPolicy() { TransformResponseBody = true }, HttpPipelinePosition.PerCall); + var tamperPolicy = TamperStreamContentsPolicy.TamperByteAt(50); + tamperPolicy.TransformResponseBody = true; + clientOptions.AddPolicy(tamperPolicy, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource: false, @@ -1670,7 +1816,7 @@ public virtual async Task DownloadHashMismatchThrows( AsyncTestDelegate operation = async () => await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); // Assert - if (validate) + if (validate || algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) { // SDK responsible for finding bad checksum. Throw. ThrowsOrInconclusiveAsync(operation); @@ -1728,7 +1874,7 @@ public virtual async Task DownloadUsesDefaultClientValidationOptions( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains("x-ms-content-crc64")); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -1788,7 +1934,7 @@ public virtual async Task DownloadOverwritesDefaultClientValidationOptions( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains("x-ms-content-crc64")); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -1827,7 +1973,7 @@ public virtual async Task DownloadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains("x-ms-content-crc64")) + if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -1850,7 +1996,54 @@ public virtual async Task DownloadDisablesDefaultClientValidationOptions( // Assert // no policies this time; just check response headers Assert.False(response.Headers.Contains("Content-MD5")); - Assert.False(response.Headers.Contains("x-ms-content-crc64")); + Assert.False(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)); + Assert.IsTrue(dest.ToArray().SequenceEqual(data)); + } + + [Test] + public virtual async Task DownloadRecoversFromInterruptWithValidation( + [ValueSource(nameof(GetValidationAlgorithms))] StorageChecksumAlgorithm algorithm) + { + using var _ = AzureEventSourceListener.CreateConsoleLogger(); + int dataLen = algorithm.ResolveAuto() switch { + StorageChecksumAlgorithm.StorageCrc64 => 5 * Constants.MB, // >4MB for multisegment + _ => Constants.KB, + }; + + await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); + + // Arrange + var data = GetRandomBuffer(dataLen); + + TClientOptions options = ClientBuilder.GetOptions(); + options.AddPolicy(new FaultyDownloadPipelinePolicy(dataLen - 512, new IOException(), () => { }), HttpPipelinePosition.BeforeTransport); + var client = await GetResourceClientAsync( + disposingContainer.Container, + resourceLength: dataLen, + createResource: true, + options: options); + await SetupDataAsync(client, new MemoryStream(data)); + + var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; + + // Act + var dest = new MemoryStream(); + var response = await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); + + // Assert + // no policies this time; just check response headers + switch (algorithm.ResolveAuto()) + { + case StorageChecksumAlgorithm.MD5: + Assert.True(response.Headers.Contains("Content-MD5")); + break; + case StorageChecksumAlgorithm.StorageCrc64: + Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); + break; + default: + Assert.Fail("Test can't validate given algorithm type."); + break; + } Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } #endregion @@ -1891,7 +2084,7 @@ public async Task RoundtripWIthDefaults() // make pipeline assertion for checking checksum was present on upload AND download var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumAssertion(expectedAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected), + checkRequest: GetRequestChecksumHeaderAssertion(expectedAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected), checkResponse: GetResponseChecksumAssertion(expectedAlgorithm)); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs new file mode 100644 index 0000000000000..a0f9158040b11 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs @@ -0,0 +1,246 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Storage.Shared; +using Azure.Storage.Test.Shared; +using Microsoft.Diagnostics.Tracing.Parsers.AspNet; +using Moq; +using NUnit.Framework; + +namespace Azure.Storage.Tests; + +[TestFixture(true)] +[TestFixture(false)] +public class StructuredMessageDecodingRetriableStreamTests +{ + public bool Async { get; } + + public StructuredMessageDecodingRetriableStreamTests(bool async) + { + Async = async; + } + + private Mock AllExceptionsRetry() + { + Mock mock = new(MockBehavior.Strict); + mock.Setup(rc => rc.IsRetriableException(It.IsAny())).Returns(true); + return mock; + } + + [Test] + public async ValueTask UninterruptedStream() + { + byte[] data = new Random().NextBytesInline(4 * Constants.KB).ToArray(); + byte[] dest = new byte[data.Length]; + + // mock with a simple MemoryStream rather than an actual StructuredMessageDecodingStream + using (Stream src = new MemoryStream(data)) + using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream(src, new(), default, default, default, default, default, 1)) + using (Stream dst = new MemoryStream(dest)) + { + await retriableSrc.CopyToInternal(dst, Async, default); + } + + Assert.AreEqual(data, dest); + } + + [Test] + public async Task Interrupt_DataIntact([Values(true, false)] bool multipleInterrupts) + { + const int segments = 4; + const int segmentLen = Constants.KB; + const int readLen = 128; + const int interruptPos = segmentLen + (3 * readLen) + 10; + + Random r = new(); + byte[] data = r.NextBytesInline(segments * Constants.KB).ToArray(); + byte[] dest = new byte[data.Length]; + + // Mock a decoded data for the mocked StructuredMessageDecodingStream + StructuredMessageDecodingStream.RawDecodedData initialDecodedData = new() + { + TotalSegments = segments, + InnerStreamLength = data.Length, + Flags = StructuredMessage.Flags.StorageCrc64 + }; + // for test purposes, initialize a DecodedData, since we are not actively decoding in this test + initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); + + (Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData) Factory(long offset, bool faulty) + { + Stream stream = new MemoryStream(data, (int)offset, data.Length - (int)offset); + if (faulty) + { + stream = new FaultyStream(stream, interruptPos, 1, new Exception(), () => { }); + } + // Mock a decoded data for the mocked StructuredMessageDecodingStream + StructuredMessageDecodingStream.RawDecodedData decodedData = new() + { + TotalSegments = segments, + InnerStreamLength = data.Length, + Flags = StructuredMessage.Flags.StorageCrc64, + }; + // for test purposes, initialize a DecodedData, since we are not actively decoding in this test + initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); + return (stream, decodedData); + } + + // mock with a simple MemoryStream rather than an actual StructuredMessageDecodingStream + using (Stream src = new MemoryStream(data)) + using (Stream faultySrc = new FaultyStream(src, interruptPos, 1, new Exception(), () => { })) + using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream( + faultySrc, + initialDecodedData, + default, + offset => Factory(offset, multipleInterrupts), + offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)>(Factory(offset, multipleInterrupts)), + null, + AllExceptionsRetry().Object, + int.MaxValue)) + using (Stream dst = new MemoryStream(dest)) + { + await retriableSrc.CopyToInternal(dst, readLen, Async, default); + } + + Assert.AreEqual(data, dest); + } + + [Test] + public async Task Interrupt_AppropriateRewind() + { + const int segments = 2; + const int segmentLen = Constants.KB; + const int dataLen = segments * segmentLen; + const int readLen = segmentLen / 4; + const int interruptOffset = 10; + const int interruptPos = segmentLen + (2 * readLen) + interruptOffset; + Random r = new(); + + // Mock a decoded data for the mocked StructuredMessageDecodingStream + StructuredMessageDecodingStream.RawDecodedData initialDecodedData = new() + { + TotalSegments = segments, + InnerStreamLength = segments * segmentLen, + Flags = StructuredMessage.Flags.StorageCrc64, + }; + // By the time of interrupt, there will be one segment reported + initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); + + Mock mock = new(MockBehavior.Strict); + mock.SetupGet(s => s.CanRead).Returns(true); + mock.SetupGet(s => s.CanSeek).Returns(false); + if (Async) + { + mock.SetupSequence(s => s.ReadAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(Task.FromResult(readLen)) // start first segment + .Returns(Task.FromResult(readLen)) + .Returns(Task.FromResult(readLen)) + .Returns(Task.FromResult(readLen)) // finish first segment + .Returns(Task.FromResult(readLen)) // start second segment + .Returns(Task.FromResult(readLen)) + // faulty stream interrupt + .Returns(Task.FromResult(readLen * 2)) // restart second segment. fast-forward uses an internal 4KB buffer, so it will leap the 512 byte catchup all at once + .Returns(Task.FromResult(readLen)) + .Returns(Task.FromResult(readLen)) // end second segment + .Returns(Task.FromResult(0)) // signal end of stream + .Returns(Task.FromResult(0)) // second signal needed for stream wrapping reasons + ; + } + else + { + mock.SetupSequence(s => s.Read(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(readLen) // start first segment + .Returns(readLen) + .Returns(readLen) + .Returns(readLen) // finish first segment + .Returns(readLen) // start second segment + .Returns(readLen) + // faulty stream interrupt + .Returns(readLen * 2) // restart second segment. fast-forward uses an internal 4KB buffer, so it will leap the 512 byte catchup all at once + .Returns(readLen) + .Returns(readLen) // end second segment + .Returns(0) // signal end of stream + .Returns(0) // second signal needed for stream wrapping reasons + ; + } + Stream faultySrc = new FaultyStream(mock.Object, interruptPos, 1, new Exception(), default); + Stream retriableSrc = new StructuredMessageDecodingRetriableStream( + faultySrc, + initialDecodedData, + default, + offset => (mock.Object, new()), + offset => new(Task.FromResult((mock.Object, new StructuredMessageDecodingStream.RawDecodedData()))), + null, + AllExceptionsRetry().Object, + 1); + + int totalRead = 0; + int read = 0; + byte[] buf = new byte[readLen]; + if (Async) + { + while ((read = await retriableSrc.ReadAsync(buf, 0, buf.Length)) > 0) + { + totalRead += read; + } + } + else + { + while ((read = retriableSrc.Read(buf, 0, buf.Length)) > 0) + { + totalRead += read; + } + } + await retriableSrc.CopyToInternal(Stream.Null, readLen, Async, default); + + // Asserts we read exactly the data length, excluding the fastforward of the inner stream + Assert.That(totalRead, Is.EqualTo(dataLen)); + } + + [Test] + public async Task Interrupt_ProperDecode([Values(true, false)] bool multipleInterrupts) + { + // decoding stream inserts a buffered layer of 4 KB. use larger sizes to avoid interference from it. + const int segments = 4; + const int segmentLen = 128 * Constants.KB; + const int readLen = 8 * Constants.KB; + const int interruptPos = segmentLen + (3 * readLen) + 10; + + Random r = new(); + byte[] data = r.NextBytesInline(segments * Constants.KB).ToArray(); + byte[] dest = new byte[data.Length]; + + (Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData) Factory(long offset, bool faulty) + { + Stream stream = new MemoryStream(data, (int)offset, data.Length - (int)offset); + stream = new StructuredMessageEncodingStream(stream, segmentLen, StructuredMessage.Flags.StorageCrc64); + if (faulty) + { + stream = new FaultyStream(stream, interruptPos, 1, new Exception(), () => { }); + } + return StructuredMessageDecodingStream.WrapStream(stream); + } + + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = Factory(0, true); + using Stream retriableSrc = new StructuredMessageDecodingRetriableStream( + decodingStream, + decodedData, + default, + offset => Factory(offset, multipleInterrupts), + offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)>(Factory(offset, multipleInterrupts)), + null, + AllExceptionsRetry().Object, + int.MaxValue); + using Stream dst = new MemoryStream(dest); + + await retriableSrc.CopyToInternal(dst, readLen, Async, default); + + Assert.AreEqual(data, dest); + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs new file mode 100644 index 0000000000000..2789672df4976 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs @@ -0,0 +1,323 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.Dynamic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Blobs.Tests; +using Azure.Storage.Shared; +using NUnit.Framework; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Tests +{ + [TestFixture(ReadMethod.SyncArray)] + [TestFixture(ReadMethod.AsyncArray)] +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + [TestFixture(ReadMethod.SyncSpan)] + [TestFixture(ReadMethod.AsyncMemory)] +#endif + public class StructuredMessageDecodingStreamTests + { + // Cannot just implement as passthru in the stream + // Must test each one + public enum ReadMethod + { + SyncArray, + AsyncArray, +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + SyncSpan, + AsyncMemory +#endif + } + + public ReadMethod Method { get; } + + public StructuredMessageDecodingStreamTests(ReadMethod method) + { + Method = method; + } + + private class CopyStreamException : Exception + { + public long TotalCopied { get; } + + public CopyStreamException(Exception inner, long totalCopied) + : base($"Failed read after {totalCopied}-many bytes.", inner) + { + TotalCopied = totalCopied; + } + } + private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl + { + byte[] buf = new byte[bufferSize]; + int read; + long totalRead = 0; + try + { + switch (Method) + { + case ReadMethod.SyncArray: + while ((read = source.Read(buf, 0, bufferSize)) > 0) + { + totalRead += read; + destination.Write(buf, 0, read); + } + break; + case ReadMethod.AsyncArray: + while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) + { + totalRead += read; + await destination.WriteAsync(buf, 0, read); + } + break; +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + case ReadMethod.SyncSpan: + while ((read = source.Read(new Span(buf))) > 0) + { + totalRead += read; + destination.Write(new Span(buf, 0, read)); + } + break; + case ReadMethod.AsyncMemory: + while ((read = await source.ReadAsync(new Memory(buf))) > 0) + { + totalRead += read; + await destination.WriteAsync(new Memory(buf, 0, read)); + } + break; +#endif + } + destination.Flush(); + } + catch (Exception ex) + { + throw new CopyStreamException(ex, totalRead); + } + return totalRead; + } + + [Test] + [Pairwise] + public async Task DecodesData( + [Values(2048, 2005)] int dataLength, + [Values(default, 512)] int? seglen, + [Values(8*Constants.KB, 512, 530, 3)] int readLen, + [Values(true, false)] bool useCrc) + { + int segmentContentLength = seglen ?? int.MaxValue; + Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; + + byte[] originalData = new byte[dataLength]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); + + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); + byte[] decodedData; + using (MemoryStream dest = new()) + { + await CopyStream(decodingStream, dest, readLen); + decodedData = dest.ToArray(); + } + + Assert.That(new Span(decodedData).SequenceEqual(originalData)); + } + + [Test] + public void BadStreamBadVersion() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + encodedData[0] = byte.MaxValue; + + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public async Task BadSegmentCrcThrows() + { + const int segmentLength = 256; + Random r = new(); + + byte[] originalData = new byte[2048]; + r.NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentLength, Flags.StorageCrc64); + + const int badBytePos = 1024; + encodedData[badBytePos] = (byte)~encodedData[badBytePos]; + + MemoryStream encodedDataStream = new(encodedData); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(encodedDataStream); + + // manual try/catch to validate the proccess failed mid-stream rather than the end + const int copyBufferSize = 4; + bool caught = false; + try + { + await CopyStream(decodingStream, Stream.Null, copyBufferSize); + } + catch (CopyStreamException ex) + { + caught = true; + Assert.That(ex.TotalCopied, Is.LessThanOrEqualTo(badBytePos)); + } + Assert.That(caught); + } + + [Test] + public void BadStreamCrcThrows() + { + const int segmentLength = 256; + Random r = new(); + + byte[] originalData = new byte[2048]; + r.NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentLength, Flags.StorageCrc64); + + encodedData[originalData.Length - 1] = (byte)~encodedData[originalData.Length - 1]; + + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public void BadStreamWrongContentLength() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + BinaryPrimitives.WriteInt64LittleEndian(new Span(encodedData, V1_0.StreamHeaderMessageLengthOffset, 8), 123456789L); + + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [TestCase(-1)] + [TestCase(1)] + public void BadStreamWrongSegmentCount(int difference) + { + const int dataSize = 1024; + const int segmentSize = 256; + const int numSegments = 4; + + byte[] originalData = new byte[dataSize]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentSize, Flags.StorageCrc64); + + // rewrite the segment count to be different than the actual number of segments + BinaryPrimitives.WriteInt16LittleEndian( + new Span(encodedData, V1_0.StreamHeaderSegmentCountOffset, 2), (short)(numSegments + difference)); + + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public void BadStreamWrongSegmentNum() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + BinaryPrimitives.WriteInt16LittleEndian( + new Span(encodedData, V1_0.StreamHeaderLength + V1_0.SegmentHeaderNumOffset, 2), 123); + + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + [Combinatorial] + public async Task BadStreamWrongContentLength( + [Values(-1, 1)] int difference, + [Values(true, false)] bool lengthProvided) + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + BinaryPrimitives.WriteInt64LittleEndian( + new Span(encodedData, V1_0.StreamHeaderMessageLengthOffset, 8), + encodedData.Length + difference); + + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream( + new MemoryStream(encodedData), + lengthProvided ? (long?)encodedData.Length : default); + + // manual try/catch with tiny buffer to validate the proccess failed mid-stream rather than the end + const int copyBufferSize = 4; + bool caught = false; + try + { + await CopyStream(decodingStream, Stream.Null, copyBufferSize); + } + catch (CopyStreamException ex) + { + caught = true; + if (lengthProvided) + { + Assert.That(ex.TotalCopied, Is.EqualTo(0)); + } + else + { + Assert.That(ex.TotalCopied, Is.EqualTo(originalData.Length)); + } + } + Assert.That(caught); + } + + [Test] + public void BadStreamMissingExpectedStreamFooter() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + byte[] brokenData = new byte[encodedData.Length - Crc64Length]; + new Span(encodedData, 0, encodedData.Length - Crc64Length).CopyTo(brokenData); + + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(brokenData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public void NoSeek() + { + (Stream stream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream()); + + Assert.That(stream.CanSeek, Is.False); + Assert.That(() => stream.Length, Throws.TypeOf()); + Assert.That(() => stream.Position, Throws.TypeOf()); + Assert.That(() => stream.Position = 0, Throws.TypeOf()); + Assert.That(() => stream.Seek(0, SeekOrigin.Begin), Throws.TypeOf()); + } + + [Test] + public void NoWrite() + { + (Stream stream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream()); + byte[] data = new byte[1024]; + new Random().NextBytes(data); + + Assert.That(stream.CanWrite, Is.False); + Assert.That(() => stream.Write(data, 0, data.Length), + Throws.TypeOf()); + Assert.That(async () => await stream.WriteAsync(data, 0, data.Length, CancellationToken.None), + Throws.TypeOf()); +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + Assert.That(() => stream.Write(new Span(data)), + Throws.TypeOf()); + Assert.That(async () => await stream.WriteAsync(new Memory(data), CancellationToken.None), + Throws.TypeOf()); +#endif + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs new file mode 100644 index 0000000000000..e0f91dee7de3a --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs @@ -0,0 +1,271 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Azure.Storage.Blobs.Tests; +using Azure.Storage.Shared; +using NUnit.Framework; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Tests +{ + [TestFixture(ReadMethod.SyncArray)] + [TestFixture(ReadMethod.AsyncArray)] +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + [TestFixture(ReadMethod.SyncSpan)] + [TestFixture(ReadMethod.AsyncMemory)] +#endif + public class StructuredMessageEncodingStreamTests + { + // Cannot just implement as passthru in the stream + // Must test each one + public enum ReadMethod + { + SyncArray, + AsyncArray, +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + SyncSpan, + AsyncMemory +#endif + } + + public ReadMethod Method { get; } + + public StructuredMessageEncodingStreamTests(ReadMethod method) + { + Method = method; + } + + private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl + { + byte[] buf = new byte[bufferSize]; + int read; + switch (Method) + { + case ReadMethod.SyncArray: + while ((read = source.Read(buf, 0, bufferSize)) > 0) + { + destination.Write(buf, 0, read); + } + break; + case ReadMethod.AsyncArray: + while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) + { + await destination.WriteAsync(buf, 0, read); + } + break; +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + case ReadMethod.SyncSpan: + while ((read = source.Read(new Span(buf))) > 0) + { + destination.Write(new Span(buf, 0, read)); + } + break; + case ReadMethod.AsyncMemory: + while ((read = await source.ReadAsync(new Memory(buf))) > 0) + { + await destination.WriteAsync(new Memory(buf, 0, read)); + } + break; +#endif + } + destination.Flush(); + } + + [Test] + [Pairwise] + public async Task EncodesData( + [Values(2048, 2005)] int dataLength, + [Values(default, 512)] int? seglen, + [Values(8 * Constants.KB, 512, 530, 3)] int readLen, + [Values(true, false)] bool useCrc) + { + int segmentContentLength = seglen ?? int.MaxValue; + Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; + + byte[] originalData = new byte[dataLength]; + new Random().NextBytes(originalData); + byte[] expectedEncodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); + + Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(originalData), segmentContentLength, flags); + byte[] encodedData; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest, readLen); + encodedData = dest.ToArray(); + } + + Assert.That(new Span(encodedData).SequenceEqual(expectedEncodedData)); + } + + [TestCase(0, 0)] // start + [TestCase(5, 0)] // partway through stream header + [TestCase(V1_0.StreamHeaderLength, 0)] // start of segment + [TestCase(V1_0.StreamHeaderLength + 3, 0)] // partway through segment header + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength, 0)] // start of segment content + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 123, 123)] // partway through segment content + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 512, 512)] // start of segment footer + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 515, 512)] // partway through segment footer + [TestCase(V1_0.StreamHeaderLength + 3*V1_0.SegmentHeaderLength + 2*Crc64Length + 1500, 1500)] // partway through not first segment content + public async Task Seek(int targetRewindOffset, int expectedInnerStreamPosition) + { + const int segmentLength = 512; + const int dataLength = 2055; + byte[] data = new byte[dataLength]; + new Random().NextBytes(data); + + MemoryStream dataStream = new(data); + StructuredMessageEncodingStream encodingStream = new(dataStream, segmentLength, Flags.StorageCrc64); + + // no support for seeking past existing read, need to consume whole stream before seeking + await CopyStream(encodingStream, Stream.Null); + + encodingStream.Position = targetRewindOffset; + Assert.That(encodingStream.Position, Is.EqualTo(targetRewindOffset)); + Assert.That(dataStream.Position, Is.EqualTo(expectedInnerStreamPosition)); + } + + [TestCase(0)] // start + [TestCase(5)] // partway through stream header + [TestCase(V1_0.StreamHeaderLength)] // start of segment + [TestCase(V1_0.StreamHeaderLength + 3)] // partway through segment header + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength)] // start of segment content + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 123)] // partway through segment content + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 512)] // start of segment footer + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 515)] // partway through segment footer + [TestCase(V1_0.StreamHeaderLength + 2 * V1_0.SegmentHeaderLength + Crc64Length + 1500)] // partway through not first segment content + public async Task SupportsRewind(int targetRewindOffset) + { + const int segmentLength = 512; + const int dataLength = 2055; + byte[] data = new byte[dataLength]; + new Random().NextBytes(data); + + Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); + byte[] encodedData1; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest); + encodedData1 = dest.ToArray(); + } + encodingStream.Position = targetRewindOffset; + byte[] encodedData2; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest); + encodedData2 = dest.ToArray(); + } + + Assert.That(new Span(encodedData1).Slice(targetRewindOffset).SequenceEqual(encodedData2)); + } + + [Test] + public async Task SupportsFastForward() + { + const int segmentLength = 512; + const int dataLength = 2055; + byte[] data = new byte[dataLength]; + new Random().NextBytes(data); + + // must have read stream to fastforward. so read whole stream upfront & save result to check later + Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); + byte[] encodedData; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest); + encodedData = dest.ToArray(); + } + + encodingStream.Position = 0; + + bool skip = false; + const int increment = 499; + while (encodingStream.Position < encodingStream.Length) + { + if (skip) + { + encodingStream.Position = Math.Min(dataLength, encodingStream.Position + increment); + skip = !skip; + continue; + } + ReadOnlyMemory expected = new(encodedData, (int)encodingStream.Position, + (int)Math.Min(increment, encodedData.Length - encodingStream.Position)); + ReadOnlyMemory actual; + using (MemoryStream dest = new(increment)) + { + await CopyStream(WindowStream.GetWindow(encodingStream, increment), dest); + actual = dest.ToArray(); + } + Assert.That(expected.Span.SequenceEqual(actual.Span)); + skip = !skip; + } + } + + [Test] + public void NotSupportsFastForwardBeyondLatestRead() + { + const int segmentLength = 512; + const int dataLength = 2055; + byte[] data = new byte[dataLength]; + new Random().NextBytes(data); + + Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); + + Assert.That(() => encodingStream.Position = 123, Throws.TypeOf()); + } + + [Test] + [Pairwise] + public async Task WrapperStreamCorrectData( + [Values(2048, 2005)] int dataLength, + [Values(8 * Constants.KB, 512, 530, 3)] int readLen) + { + int segmentContentLength = dataLength; + Flags flags = Flags.StorageCrc64; + + byte[] originalData = new byte[dataLength]; + new Random().NextBytes(originalData); + byte[] crc = CrcInline(originalData); + byte[] expectedEncodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); + + Stream encodingStream = new StructuredMessagePrecalculatedCrcWrapperStream(new MemoryStream(originalData), crc); + byte[] encodedData; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest, readLen); + encodedData = dest.ToArray(); + } + + Assert.That(new Span(encodedData).SequenceEqual(expectedEncodedData)); + } + + private static void AssertExpectedStreamHeader(ReadOnlySpan actual, int originalDataLength, Flags flags, int expectedSegments) + { + int expectedFooterLen = flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; + + Assert.That(actual.Length, Is.EqualTo(V1_0.StreamHeaderLength)); + Assert.That(actual[0], Is.EqualTo(1)); + Assert.That(BinaryPrimitives.ReadInt64LittleEndian(actual.Slice(1, 8)), + Is.EqualTo(V1_0.StreamHeaderLength + expectedSegments * (V1_0.SegmentHeaderLength + expectedFooterLen) + originalDataLength)); + Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(9, 2)), Is.EqualTo((short)flags)); + Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(11, 2)), Is.EqualTo((short)expectedSegments)); + } + + private static void AssertExpectedSegmentHeader(ReadOnlySpan actual, int segmentNum, long contentLength) + { + Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(0, 2)), Is.EqualTo((short) segmentNum)); + Assert.That(BinaryPrimitives.ReadInt64LittleEndian(actual.Slice(2, 8)), Is.EqualTo(contentLength)); + } + + private static byte[] CrcInline(ReadOnlySpan data) + { + var crc = StorageCrc64HashAlgorithm.Create(); + crc.Append(data); + return crc.GetCurrentHash(); + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs new file mode 100644 index 0000000000000..59e80320d96a0 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Storage.Shared; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Blobs.Tests +{ + internal class StructuredMessageHelper + { + public static byte[] MakeEncodedData(ReadOnlySpan data, long segmentContentLength, Flags flags) + { + int segmentCount = (int) Math.Ceiling(data.Length / (double)segmentContentLength); + int segmentFooterLen = flags.HasFlag(Flags.StorageCrc64) ? 8 : 0; + int streamFooterLen = flags.HasFlag(Flags.StorageCrc64) ? 8 : 0; + + byte[] encodedData = new byte[ + V1_0.StreamHeaderLength + + segmentCount*(V1_0.SegmentHeaderLength + segmentFooterLen) + + streamFooterLen + + data.Length]; + V1_0.WriteStreamHeader( + new Span(encodedData, 0, V1_0.StreamHeaderLength), + encodedData.Length, + flags, + segmentCount); + + int i = V1_0.StreamHeaderLength; + int j = 0; + foreach (int seg in Enumerable.Range(1, segmentCount)) + { + int segContentLen = Math.Min((int)segmentContentLength, data.Length - j); + V1_0.WriteSegmentHeader( + new Span(encodedData, i, V1_0.SegmentHeaderLength), + seg, + segContentLen); + i += V1_0.SegmentHeaderLength; + + data.Slice(j, segContentLen) + .CopyTo(new Span(encodedData).Slice(i)); + i += segContentLen; + + if (flags.HasFlag(Flags.StorageCrc64)) + { + var crc = StorageCrc64HashAlgorithm.Create(); + crc.Append(data.Slice(j, segContentLen)); + crc.GetCurrentHash(new Span(encodedData, i, Crc64Length)); + i += Crc64Length; + } + j += segContentLen; + } + + if (flags.HasFlag(Flags.StorageCrc64)) + { + var crc = StorageCrc64HashAlgorithm.Create(); + crc.Append(data); + crc.GetCurrentHash(new Span(encodedData, i, Crc64Length)); + } + + return encodedData; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs new file mode 100644 index 0000000000000..61583aa1ebe4e --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Azure.Storage.Shared; +using NUnit.Framework; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Tests +{ + [TestFixture(ReadMethod.SyncArray)] + [TestFixture(ReadMethod.AsyncArray)] +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + [TestFixture(ReadMethod.SyncSpan)] + [TestFixture(ReadMethod.AsyncMemory)] +#endif + public class StructuredMessageStreamRoundtripTests + { + // Cannot just implement as passthru in the stream + // Must test each one + public enum ReadMethod + { + SyncArray, + AsyncArray, +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + SyncSpan, + AsyncMemory +#endif + } + + public ReadMethod Method { get; } + + public StructuredMessageStreamRoundtripTests(ReadMethod method) + { + Method = method; + } + + private class CopyStreamException : Exception + { + public long TotalCopied { get; } + + public CopyStreamException(Exception inner, long totalCopied) + : base($"Failed read after {totalCopied}-many bytes.", inner) + { + TotalCopied = totalCopied; + } + } + private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl + { + byte[] buf = new byte[bufferSize]; + int read; + long totalRead = 0; + try + { + switch (Method) + { + case ReadMethod.SyncArray: + while ((read = source.Read(buf, 0, bufferSize)) > 0) + { + totalRead += read; + destination.Write(buf, 0, read); + } + break; + case ReadMethod.AsyncArray: + while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) + { + totalRead += read; + await destination.WriteAsync(buf, 0, read); + } + break; +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + case ReadMethod.SyncSpan: + while ((read = source.Read(new Span(buf))) > 0) + { + totalRead += read; + destination.Write(new Span(buf, 0, read)); + } + break; + case ReadMethod.AsyncMemory: + while ((read = await source.ReadAsync(new Memory(buf))) > 0) + { + totalRead += read; + await destination.WriteAsync(new Memory(buf, 0, read)); + } + break; +#endif + } + destination.Flush(); + } + catch (Exception ex) + { + throw new CopyStreamException(ex, totalRead); + } + return totalRead; + } + + [Test] + [Pairwise] + public async Task RoundTrip( + [Values(2048, 2005)] int dataLength, + [Values(default, 512)] int? seglen, + [Values(8 * Constants.KB, 512, 530, 3)] int readLen, + [Values(true, false)] bool useCrc) + { + int segmentLength = seglen ?? int.MaxValue; + Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; + + byte[] originalData = new byte[dataLength]; + new Random().NextBytes(originalData); + + byte[] roundtripData; + using (MemoryStream source = new(originalData)) + using (Stream encode = new StructuredMessageEncodingStream(source, segmentLength, flags)) + using (Stream decode = StructuredMessageDecodingStream.WrapStream(encode).DecodedStream) + using (MemoryStream dest = new()) + { + await CopyStream(source, dest, readLen); + roundtripData = dest.ToArray(); + } + + Assert.That(originalData.SequenceEqual(roundtripData)); + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs new file mode 100644 index 0000000000000..b4f1dfe178246 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using NUnit.Framework; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Tests +{ + public class StructuredMessageTests + { + [TestCase(1024, Flags.None, 2)] + [TestCase(2000, Flags.StorageCrc64, 4)] + public void EncodeStreamHeader(int messageLength, int flags, int numSegments) + { + Span encoding = new(new byte[V1_0.StreamHeaderLength]); + V1_0.WriteStreamHeader(encoding, messageLength, (Flags)flags, numSegments); + + Assert.That(encoding[0], Is.EqualTo((byte)1)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(encoding.Slice(1, 8)), Is.EqualTo(messageLength)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(9, 2)), Is.EqualTo(flags)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(11, 2)), Is.EqualTo(numSegments)); + } + + [TestCase(V1_0.StreamHeaderLength)] + [TestCase(V1_0.StreamHeaderLength + 1)] + [TestCase(V1_0.StreamHeaderLength - 1)] + public void EncodeStreamHeaderRejectBadBufferSize(int bufferSize) + { + Random r = new(); + byte[] encoding = new byte[bufferSize]; + + void Action() => V1_0.WriteStreamHeader(encoding, r.Next(2, int.MaxValue), Flags.StorageCrc64, r.Next(2, int.MaxValue)); + if (bufferSize < V1_0.StreamHeaderLength) + { + Assert.That(Action, Throws.ArgumentException); + } + else + { + Assert.That(Action, Throws.Nothing); + } + } + + [TestCase(1, 1024)] + [TestCase(5, 39578)] + public void EncodeSegmentHeader(int segmentNum, int contentLength) + { + Span encoding = new(new byte[V1_0.SegmentHeaderLength]); + V1_0.WriteSegmentHeader(encoding, segmentNum, contentLength); + + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(0, 2)), Is.EqualTo(segmentNum)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(encoding.Slice(2, 8)), Is.EqualTo(contentLength)); + } + + [TestCase(V1_0.SegmentHeaderLength)] + [TestCase(V1_0.SegmentHeaderLength + 1)] + [TestCase(V1_0.SegmentHeaderLength - 1)] + public void EncodeSegmentHeaderRejectBadBufferSize(int bufferSize) + { + Random r = new(); + byte[] encoding = new byte[bufferSize]; + + void Action() => V1_0.WriteSegmentHeader(encoding, r.Next(1, int.MaxValue), r.Next(2, int.MaxValue)); + if (bufferSize < V1_0.SegmentHeaderLength) + { + Assert.That(Action, Throws.ArgumentException); + } + else + { + Assert.That(Action, Throws.Nothing); + } + } + + [TestCase(true)] + [TestCase(false)] + public void EncodeSegmentFooter(bool useCrc) + { + Span encoding = new(new byte[Crc64Length]); + Span crc = useCrc ? new Random().NextBytesInline(Crc64Length) : default; + V1_0.WriteSegmentFooter(encoding, crc); + + if (useCrc) + { + Assert.That(encoding.SequenceEqual(crc), Is.True); + } + else + { + Assert.That(encoding.SequenceEqual(new Span(new byte[Crc64Length])), Is.True); + } + } + + [TestCase(Crc64Length)] + [TestCase(Crc64Length + 1)] + [TestCase(Crc64Length - 1)] + public void EncodeSegmentFooterRejectBadBufferSize(int bufferSize) + { + byte[] encoding = new byte[bufferSize]; + byte[] crc = new byte[Crc64Length]; + new Random().NextBytes(crc); + + void Action() => V1_0.WriteSegmentFooter(encoding, crc); + if (bufferSize < Crc64Length) + { + Assert.That(Action, Throws.ArgumentException); + } + else + { + Assert.That(Action, Throws.Nothing); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Azure.Storage.DataMovement.Blobs.Samples.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Azure.Storage.DataMovement.Blobs.Samples.Tests.csproj index 7ab901e963e03..30d4b1f79daaf 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Azure.Storage.DataMovement.Blobs.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Azure.Storage.DataMovement.Blobs.Samples.Tests.csproj @@ -11,6 +11,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj index 66520948815e4..921dec1e469ee 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj @@ -37,6 +37,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs index 84d60b3bc37c4..2c6864f511571 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs @@ -99,7 +99,7 @@ internal static StorageResourceItemProperties ToStorageResourceItemProperties(th ContentRange contentRange = !string.IsNullOrWhiteSpace(result?.Details?.ContentRange) ? ContentRange.Parse(result.Details.ContentRange) : default; if (contentRange != default) { - size = contentRange.Size; + size = contentRange.TotalResourceLength; } return new StorageResourceItemProperties( @@ -151,7 +151,7 @@ internal static StorageResourceReadStreamResult ToReadStreamStorageResourceInfo( if (contentRange != default) { range = ContentRange.ToHttpRange(contentRange); - size = contentRange.Size; + size = contentRange.TotalResourceLength; } else if (result.Details.ContentLength > 0) { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj index 83e1264a0290a..753475c1adc47 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj @@ -22,11 +22,15 @@ + + + + @@ -40,6 +44,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj index a6abde432473f..66a9fea0861a2 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj @@ -35,6 +35,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj index 9cde066f64eb7..6a472b9f74158 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks) Microsoft Azure.Storage.DataMovement.Files.Shares client library samples @@ -11,6 +11,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs index 9cb7d338fcb60..16a164f61b060 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs @@ -335,14 +335,14 @@ internal static StorageResourceReadStreamResult ToStorageResourceReadStreamResul ContentRange contentRange = !string.IsNullOrWhiteSpace(info?.Details?.ContentRange) ? ContentRange.Parse(info.Details.ContentRange) : default; if (contentRange != default) { - size = contentRange.Size; + size = contentRange.TotalResourceLength; } return new StorageResourceReadStreamResult( content: info?.Content, range: ContentRange.ToHttpRange(contentRange), properties: new StorageResourceItemProperties( - resourceLength: contentRange.Size, + resourceLength: contentRange.TotalResourceLength, eTag: info.Details.ETag, lastModifiedTime: info.Details.LastModified, properties: properties)); diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj index 8e574bca36a48..d75775beceafd 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj @@ -27,6 +27,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj b/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj index fde61bae96877..3845bc4da8b4a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj +++ b/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks);net6.0 diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj index b5e3c42359976..7a40eb8026443 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj @@ -34,6 +34,7 @@ + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 4a64b8398f656..5127ea7e0c4db 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_d74597f1e3" + "Tag": "net/storage/Azure.Storage.Files.DataLake_48a38da58a" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Azure.Storage.Files.DataLake.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Files.DataLake/samples/Azure.Storage.Files.DataLake.Samples.Tests.csproj index c230f2ed8fa20..eecbe0543fe87 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Azure.Storage.Files.DataLake.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Azure.Storage.Files.DataLake.Samples.Tests.csproj @@ -15,6 +15,7 @@ + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj index 89a7d0d737796..559839b1adf8b 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj @@ -42,6 +42,7 @@ + @@ -81,6 +82,10 @@ + + + + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index e755faff2f5a7..46a555458acb6 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -16,6 +16,7 @@ using Azure.Storage.Common; using Azure.Storage.Files.DataLake.Models; using Azure.Storage.Sas; +using Azure.Storage.Shared; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Files.DataLake @@ -2332,13 +2333,39 @@ internal virtual async Task AppendInternal( using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(DataLakeFileClient))) { // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (content != null && + validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); + contentLength = content.Length - content.Position; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content?.WithNoDispose().WithProgress(progressHandler); + } - content = content?.WithNoDispose().WithProgress(progressHandler); ClientConfiguration.Pipeline.LogMethodEnter( nameof(DataLakeFileClient), message: @@ -2373,6 +2400,8 @@ internal virtual async Task AppendInternal( encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, leaseId: leaseId, leaseAction: leaseAction, leaseDuration: leaseDurationLong, @@ -2392,6 +2421,8 @@ internal virtual async Task AppendInternal( encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, leaseId: leaseId, leaseAction: leaseAction, leaseDuration: leaseDurationLong, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md index ec9675a014f70..a8340f1092bcb 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md @@ -23,7 +23,7 @@ directive: if (property.includes('/{filesystem}/{path}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/FileSystem") && false == param['$ref'].endsWith("#/parameters/Path"))}); - } + } else if (property.includes('/{filesystem}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/FileSystem"))}); @@ -127,7 +127,7 @@ directive: } $[newName] = $[oldName]; delete $[oldName]; - } + } else if (property.includes('/{filesystem}')) { var oldName = property; diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj b/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj index bef13bb21a1c6..1fa78690077be 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj @@ -6,6 +6,9 @@ Microsoft Azure.Storage.Files.DataLake client library tests false + + DataLakeSDK + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs index 4bdefdbf756cd..5067f98517bd2 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs @@ -34,7 +34,10 @@ protected override async Task> Get StorageChecksumAlgorithm uploadAlgorithm = StorageChecksumAlgorithm.None, StorageChecksumAlgorithm downloadAlgorithm = StorageChecksumAlgorithm.None) { - var disposingFileSystem = await ClientBuilder.GetNewFileSystem(service: service, fileSystemName: containerName); + var disposingFileSystem = await ClientBuilder.GetNewFileSystem( + service: service, + fileSystemName: containerName, + publicAccessType: PublicAccessType.None); disposingFileSystem.FileSystem.ClientConfiguration.TransferValidation.Upload.ChecksumAlgorithm = uploadAlgorithm; disposingFileSystem.FileSystem.ClientConfiguration.TransferValidation.Download.ChecksumAlgorithm = downloadAlgorithm; diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs index 430c0b16d2ebe..c49119c1f9f12 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs @@ -809,6 +809,7 @@ public partial class ShareFileDownloadInfo : System.IDisposable { internal ShareFileDownloadInfo() { } public System.IO.Stream Content { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public byte[] ContentHash { get { throw null; } } public long ContentLength { get { throw null; } } public string ContentType { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs index 430c0b16d2ebe..c49119c1f9f12 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs @@ -809,6 +809,7 @@ public partial class ShareFileDownloadInfo : System.IDisposable { internal ShareFileDownloadInfo() { } public System.IO.Stream Content { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public byte[] ContentHash { get { throw null; } } public long ContentLength { get { throw null; } } public string ContentType { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index c2b5c3d31e6a2..c33c8bb335398 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_df67d82d59" + "Tag": "net/storage/Azure.Storage.Files.Shares_4b545ae555" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/samples/Azure.Storage.Files.Shares.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Files.Shares/samples/Azure.Storage.Files.Shares.Samples.Tests.csproj index 0bcec423c144d..d1efeca0c2da2 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/samples/Azure.Storage.Files.Shares.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/samples/Azure.Storage.Files.Shares.Samples.Tests.csproj @@ -16,6 +16,7 @@ + PreserveNewest diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj index 740f463a07c5b..00b9fabda4394 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks);net6.0 @@ -42,6 +42,7 @@ + @@ -85,6 +86,11 @@ + + + + + diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs index 0165af94435a0..4037cbdfd875e 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs @@ -38,6 +38,12 @@ public partial class ShareFileDownloadInfo : IDisposable, IDownloadedContent public byte[] ContentHash { get; internal set; } #pragma warning restore CA1819 // Properties should not return arrays + /// + /// When requested using , this value contains the CRC for the download blob range. + /// This value may only become populated once the network stream is fully consumed. + /// + public byte[] ContentCrc { get; internal set; } + /// /// Details returned when downloading a file /// diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs index f776384d06add..0b27510aaa6c4 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs @@ -17,20 +17,5 @@ public static InvalidOperationException FileOrShareMissing( string fileClient, string shareClient) => new InvalidOperationException($"{leaseClient} requires either a {fileClient} or {shareClient}"); - - public static void AssertAlgorithmSupport(StorageChecksumAlgorithm? algorithm) - { - StorageChecksumAlgorithm resolved = (algorithm ?? StorageChecksumAlgorithm.None).ResolveAuto(); - switch (resolved) - { - case StorageChecksumAlgorithm.None: - case StorageChecksumAlgorithm.MD5: - return; - case StorageChecksumAlgorithm.StorageCrc64: - throw new ArgumentException("Azure File Shares do not support CRC-64."); - default: - throw new ArgumentException($"{nameof(StorageChecksumAlgorithm)} does not support value {Enum.GetName(typeof(StorageChecksumAlgorithm), resolved) ?? ((int)resolved).ToString(CultureInfo.InvariantCulture)}."); - } - } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs index f713200a524de..ea3f8554b944d 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs @@ -2397,51 +2397,70 @@ private async Task> DownloadInternal( // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - initialResponse.Value.Content = RetriableStream.Create( - stream, - startOffset => - { - (Response Response, Stream ContentStream) = StartDownloadAsync( - range, - validationOptions, - conditions, - startOffset, - async, - cancellationToken) - .EnsureCompleted(); - if (etag != Response.GetRawResponse().Headers.ETag) - { - throw new ShareFileModifiedException( - "File has been modified concurrently", - Uri, etag, Response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); - } - return ContentStream; - }, - async startOffset => + async ValueTask> Factory(long offset, bool async, CancellationToken cancellationToken) + { + (Response response, Stream contentStream) = await StartDownloadAsync( + range, + validationOptions, + conditions, + offset, + async, + cancellationToken).ConfigureAwait(false); + if (etag != response.GetRawResponse().Headers.ETag) { - (Response Response, Stream ContentStream) = await StartDownloadAsync( - range, - validationOptions, - conditions, - startOffset, - async, - cancellationToken) - .ConfigureAwait(false); - if (etag != Response.GetRawResponse().Headers.ETag) + throw new ShareFileModifiedException( + "File has been modified concurrently", + Uri, etag, response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); + } + return response; + } + async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)> StructuredMessageFactory( + long offset, bool async, CancellationToken cancellationToken) + { + Response result = await Factory(offset, async, cancellationToken).ConfigureAwait(false); + return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.ContentLength); + } + + if (initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) + { + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( + initialResponse.Value.Content, initialResponse.Value.ContentLength); + initialResponse.Value.Content = new StructuredMessageDecodingRetriableStream( + decodingStream, + decodedData, + StructuredMessage.Flags.StorageCrc64, + startOffset => StructuredMessageFactory(startOffset, async: false, cancellationToken) + .EnsureCompleted(), + async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) + .ConfigureAwait(false), + decodedData => { - throw new ShareFileModifiedException( - "File has been modified concurrently", - Uri, etag, Response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); - } - return ContentStream; - }, - ClientConfiguration.Pipeline.ResponseClassifier, - Constants.MaxReliabilityRetries); + initialResponse.Value.ContentCrc = new byte[StructuredMessage.Crc64Length]; + decodedData.Crc.WriteCrc64(initialResponse.Value.ContentCrc); + }, + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); + } + else + { + initialResponse.Value.Content = RetriableStream.Create( + initialResponse.Value.Content, + startOffset => Factory(startOffset, async: false, cancellationToken) + .EnsureCompleted().Value.Content, + async startOffset => (await Factory(startOffset, async: true, cancellationToken) + .ConfigureAwait(false)).Value.Content, + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); + } // buffer response stream and ensure it matches the transactional hash if any // Storage will not return a hash for payload >4MB, so this buffer is capped similarly // hashing is opt-in, so this buffer is part of that opt-in - if (validationOptions != default && validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && validationOptions.AutoValidateChecksum) + if (validationOptions != default && + validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && + validationOptions.AutoValidateChecksum && + // structured message decoding does the validation for us + !initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) { // safe-buffer; transactional hash download limit well below maxInt var readDestStream = new MemoryStream((int)initialResponse.Value.ContentLength); @@ -2524,8 +2543,6 @@ await ContentHasher.AssertResponseHashMatchInternal( bool async = true, CancellationToken cancellationToken = default) { - ShareErrors.AssertAlgorithmSupport(transferValidationOverride?.ChecksumAlgorithm); - // calculation gets illegible with null coalesce; just pre-initialize var pageRange = range; pageRange = new HttpRange( @@ -2535,13 +2552,27 @@ await ContentHasher.AssertResponseHashMatchInternal( (long?)null); ClientConfiguration.Pipeline.LogTrace($"Download {Uri} with range: {pageRange}"); - ResponseWithHeaders response; + bool? rangeGetContentMD5 = null; + string structuredBodyType = null; + switch (transferValidationOverride?.ChecksumAlgorithm.ResolveAuto()) + { + case StorageChecksumAlgorithm.MD5: + rangeGetContentMD5 = true; + break; + case StorageChecksumAlgorithm.StorageCrc64: + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + break; + default: + break; + } + ResponseWithHeaders response; if (async) { response = await FileRestClient.DownloadAsync( range: pageRange == default ? null : pageRange.ToString(), - rangeGetContentMD5: transferValidationOverride?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, + rangeGetContentMD5: rangeGetContentMD5, + structuredBodyType: structuredBodyType, shareFileRequestConditions: conditions, cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -2550,7 +2581,8 @@ await ContentHasher.AssertResponseHashMatchInternal( { response = FileRestClient.Download( range: pageRange == default ? null : pageRange.ToString(), - rangeGetContentMD5: transferValidationOverride?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, + rangeGetContentMD5: rangeGetContentMD5, + structuredBodyType: structuredBodyType, shareFileRequestConditions: conditions, cancellationToken: cancellationToken); } @@ -4630,7 +4662,6 @@ internal async Task> UploadRangeInternal( CancellationToken cancellationToken) { UploadTransferValidationOptions validationOptions = transferValidationOverride ?? ClientConfiguration.TransferValidation.Upload; - ShareErrors.AssertAlgorithmSupport(validationOptions?.ChecksumAlgorithm); using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(ShareFileClient))) { @@ -4646,14 +4677,38 @@ internal async Task> UploadRangeInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content.WithNoDispose().WithProgress(progressHandler); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + contentLength = (content?.Length - content?.Position) ?? 0; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); + contentLength = (content?.Length - content?.Position) ?? 0; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content.WithNoDispose().WithProgress(progressHandler); + } ResponseWithHeaders response; @@ -4666,6 +4721,8 @@ internal async Task> UploadRangeInternal( fileLastWrittenMode: fileLastWrittenMode, optionalbody: content, contentMD5: hashResult?.MD5AsArray, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, shareFileRequestConditions: conditions, cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -4679,6 +4736,8 @@ internal async Task> UploadRangeInternal( fileLastWrittenMode: fileLastWrittenMode, optionalbody: content, contentMD5: hashResult?.MD5AsArray, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, shareFileRequestConditions: conditions, cancellationToken: cancellationToken); } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md index ed634ae302734..c688a6c0d1093 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md @@ -25,7 +25,7 @@ directive: if (property.includes('/{shareName}/{directory}/{fileName}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ShareName") && false == param['$ref'].endsWith("#/parameters/DirectoryPath") && false == param['$ref'].endsWith("#/parameters/FilePath"))}); - } + } else if (property.includes('/{shareName}/{directory}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ShareName") && false == param['$ref'].endsWith("#/parameters/DirectoryPath"))}); @@ -46,7 +46,7 @@ directive: $.Metrics.type = "object"; ``` -### Times aren't required +### Times aren't required ``` yaml directive: - from: swagger-document diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj index 398a4b6367489..d09dd8fe8949f 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj @@ -17,6 +17,7 @@ + PreserveNewest diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs index 3dcdb21f27b36..9fd8905e388b1 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs @@ -64,10 +64,6 @@ protected override async Task GetResourceClientAsync( private void AssertSupportsHashAlgorithm(StorageChecksumAlgorithm algorithm) { - if (algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) - { - TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "Azure File Share does not support CRC64."); - } } protected override async Task UploadPartitionAsync(ShareFileClient client, Stream source, UploadTransferValidationOptions transferValidation) @@ -147,8 +143,44 @@ protected override async Task SetupDataAsync(ShareFileClient client, Stream data public override void TestAutoResolve() { Assert.AreEqual( - StorageChecksumAlgorithm.MD5, + StorageChecksumAlgorithm.StorageCrc64, TransferValidationOptionsExtensions.ResolveAuto(StorageChecksumAlgorithm.Auto)); } + + [Test] + public async Task StructuredMessagePopulatesCrcDownloadStreaming() + { + await using DisposingShare disposingContainer = await ClientBuilder.GetTestShareAsync(); + + const int dataLength = Constants.KB; + byte[] data = GetRandomBuffer(dataLength); + byte[] dataCrc = new byte[8]; + StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); + + ShareFileClient file = disposingContainer.Container.GetRootDirectoryClient().GetFileClient(GetNewResourceName()); + await file.CreateAsync(data.Length); + await file.UploadAsync(new MemoryStream(data)); + + Response response = await file.DownloadAsync(new ShareFileDownloadOptions() + { + TransferValidation = new DownloadTransferValidationOptions + { + ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 + } + }); + + // crc is not present until response stream is consumed + Assert.That(response.Value.ContentCrc, Is.Null); + + byte[] downloadedData; + using (MemoryStream ms = new()) + { + await response.Value.Content.CopyToAsync(ms); + downloadedData = ms.ToArray(); + } + + Assert.That(response.Value.ContentCrc, Is.EqualTo(dataCrc)); + Assert.That(downloadedData, Is.EqualTo(data)); + } } } diff --git a/sdk/storage/Azure.Storage.Queues/samples/Azure.Storage.Queues.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Queues/samples/Azure.Storage.Queues.Samples.Tests.csproj index f9ed70da2e75d..12794e190f4e1 100644 --- a/sdk/storage/Azure.Storage.Queues/samples/Azure.Storage.Queues.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/samples/Azure.Storage.Queues.Samples.Tests.csproj @@ -16,6 +16,7 @@ + PreserveNewest diff --git a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj index e0a6fab3c753b..4d0334255f041 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj @@ -21,6 +21,7 @@ + From 7cc7459f3a64af9116e34ec9893bc653b73ba7bf Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Thu, 14 Nov 2024 11:20:21 -0800 Subject: [PATCH 03/10] Revert "Content Validation STG 97 (#46489)" (#47164) This reverts commit f93ed21438ef4ffdcf32f9e3b46535a39d779cc2. --- ...e.Storage.Blobs.Batch.Samples.Tests.csproj | 1 - .../Azure.Storage.Blobs.Batch.Tests.csproj | 3 +- ...rage.Blobs.ChangeFeed.Samples.Tests.csproj | 3 +- ...zure.Storage.Blobs.ChangeFeed.Tests.csproj | 3 +- .../api/Azure.Storage.Blobs.net6.0.cs | 3 - .../api/Azure.Storage.Blobs.netstandard2.0.cs | 3 - .../api/Azure.Storage.Blobs.netstandard2.1.cs | 3 - sdk/storage/Azure.Storage.Blobs/assets.json | 2 +- .../Azure.Storage.Blobs.Samples.Tests.csproj | 1 - .../src/AppendBlobClient.cs | 45 +- .../src/Azure.Storage.Blobs.csproj | 7 - .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 110 +--- .../src/BlobClientOptions.cs | 2 - .../src/BlobClientSideDecryptor.cs | 2 +- .../src/BlockBlobClient.cs | 92 +-- .../src/Models/BlobDownloadDetails.cs | 8 - .../src/Models/BlobDownloadInfo.cs | 10 - .../src/Models/BlobDownloadStreamingResult.cs | 8 - .../Azure.Storage.Blobs/src/PageBlobClient.cs | 49 +- .../src/PartitionedDownloader.cs | 95 ++- .../Azure.Storage.Blobs/src/autorest.md | 4 +- .../tests/Azure.Storage.Blobs.Tests.csproj | 3 - .../BlobBaseClientTransferValidationTests.cs | 114 ++-- .../tests/ClientSideEncryptionTests.cs | 2 +- .../tests/PartitionedDownloaderTests.cs | 2 +- .../Azure.Storage.Common.Samples.Tests.csproj | 1 - .../src/Shared/ChecksumExtensions.cs | 22 - .../src/Shared/Constants.cs | 9 - .../src/Shared/ContentRange.cs | 18 +- .../src/Shared/ContentRangeExtensions.cs | 14 - .../src/Shared/Errors.Clients.cs | 10 - .../Azure.Storage.Common/src/Shared/Errors.cs | 19 - .../src/Shared/LazyLoadingReadOnlyStream.cs | 40 +- .../src/Shared/PooledMemoryStream.cs | 2 +- .../src/Shared/StorageCrc64Composer.cs | 48 +- .../StorageRequestValidationPipelinePolicy.cs | 29 - .../src/Shared/StreamExtensions.cs | 22 +- .../src/Shared/StructuredMessage.cs | 244 -------- ...tructuredMessageDecodingRetriableStream.cs | 264 --------- .../Shared/StructuredMessageDecodingStream.cs | 542 ----------------- .../Shared/StructuredMessageEncodingStream.cs | 545 ------------------ ...redMessagePrecalculatedCrcWrapperStream.cs | 451 --------------- .../TransferValidationOptionsExtensions.cs | 7 + .../tests/Azure.Storage.Common.Tests.csproj | 9 - .../tests/Shared/FaultyStream.cs | 13 +- .../Shared/ObserveStructuredMessagePolicy.cs | 85 --- .../tests/Shared/RequestExtensions.cs | 27 - .../Shared/TamperStreamContentsPolicy.cs | 11 +- .../Shared/TransferValidationTestBase.cs | 325 +++-------- ...uredMessageDecodingRetriableStreamTests.cs | 246 -------- .../StructuredMessageDecodingStreamTests.cs | 323 ----------- .../StructuredMessageEncodingStreamTests.cs | 271 --------- .../tests/StructuredMessageHelper.cs | 68 --- .../StructuredMessageStreamRoundtripTests.cs | 127 ---- .../tests/StructuredMessageTests.cs | 114 ---- ...ge.DataMovement.Blobs.Samples.Tests.csproj | 1 - .../Azure.Storage.DataMovement.Blobs.csproj | 1 - .../src/DataMovementBlobsExtensions.cs | 4 +- ...re.Storage.DataMovement.Blobs.Tests.csproj | 5 - ...taMovement.Blobs.Files.Shares.Tests.csproj | 1 - ...Movement.Files.Shares.Samples.Tests.csproj | 3 +- .../src/DataMovementSharesExtensions.cs | 4 +- ...age.DataMovement.Files.Shares.Tests.csproj | 1 - .../src/Azure.Storage.DataMovement.csproj | 2 +- .../Azure.Storage.DataMovement.Tests.csproj | 1 - .../Azure.Storage.Files.DataLake/assets.json | 2 +- ...torage.Files.DataLake.Samples.Tests.csproj | 1 - .../src/Azure.Storage.Files.DataLake.csproj | 5 - .../src/DataLakeFileClient.cs | 43 +- .../src/autorest.md | 4 +- .../Azure.Storage.Files.DataLake.Tests.csproj | 3 - ...taLakeFileClientTransferValidationTests.cs | 5 +- .../api/Azure.Storage.Files.Shares.net6.0.cs | 1 - ...ure.Storage.Files.Shares.netstandard2.0.cs | 1 - .../Azure.Storage.Files.Shares/assets.json | 2 +- ....Storage.Files.Shares.Samples.Tests.csproj | 1 - .../src/Azure.Storage.Files.Shares.csproj | 8 +- .../src/Models/ShareFileDownloadInfo.cs | 6 - .../src/ShareErrors.cs | 15 + .../src/ShareFileClient.cs | 165 ++---- .../src/autorest.md | 4 +- .../Azure.Storage.Files.Shares.Tests.csproj | 1 - .../ShareFileClientTransferValidationTests.cs | 42 +- .../Azure.Storage.Queues.Samples.Tests.csproj | 1 - .../tests/Azure.Storage.Queues.Tests.csproj | 1 - 85 files changed, 387 insertions(+), 4428 deletions(-) delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ContentRangeExtensions.cs delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs delete mode 100644 sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs delete mode 100644 sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs delete mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs delete mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs delete mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs delete mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs delete mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs delete mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj index 6009a5336b8b9..3dea34a02b7ea 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj @@ -17,7 +17,6 @@ - PreserveNewest diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj index 286ab317256bf..2b77907e9aaac 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj @@ -23,7 +23,6 @@ - PreserveNewest @@ -43,4 +42,4 @@ - + \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/samples/Azure.Storage.Blobs.ChangeFeed.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/samples/Azure.Storage.Blobs.ChangeFeed.Samples.Tests.csproj index 6f8fcaf6528b3..7711cae537db6 100644 --- a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/samples/Azure.Storage.Blobs.ChangeFeed.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/samples/Azure.Storage.Blobs.ChangeFeed.Samples.Tests.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks) Microsoft Azure.Storage.Blobs.ChangeFeed client library samples @@ -14,7 +14,6 @@ - diff --git a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj index 8cf13cd60744f..9682ab15ecd60 100644 --- a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj @@ -17,7 +17,6 @@ - @@ -29,4 +28,4 @@ PreserveNewest - + \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs index d0e0ceb1f36b7..18170fd30ce7a 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs @@ -523,7 +523,6 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } - public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } @@ -569,7 +568,6 @@ internal BlobDownloadInfo() { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public string ContentType { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } - public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadOptions @@ -591,7 +589,6 @@ public partial class BlobDownloadStreamingResult : System.IDisposable internal BlobDownloadStreamingResult() { } public System.IO.Stream Content { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } - public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadToOptions diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 785d09b1b8b8b..8707ec2108684 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -523,7 +523,6 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } - public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } @@ -569,7 +568,6 @@ internal BlobDownloadInfo() { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public string ContentType { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } - public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadOptions @@ -591,7 +589,6 @@ public partial class BlobDownloadStreamingResult : System.IDisposable internal BlobDownloadStreamingResult() { } public System.IO.Stream Content { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } - public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadToOptions diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs index 785d09b1b8b8b..8707ec2108684 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs @@ -523,7 +523,6 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } - public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } @@ -569,7 +568,6 @@ internal BlobDownloadInfo() { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public string ContentType { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } - public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadOptions @@ -591,7 +589,6 @@ public partial class BlobDownloadStreamingResult : System.IDisposable internal BlobDownloadStreamingResult() { } public System.IO.Stream Content { get { throw null; } } public Azure.Storage.Blobs.Models.BlobDownloadDetails Details { get { throw null; } } - public bool ExpectTrailingDetails { get { throw null; } } public void Dispose() { } } public partial class BlobDownloadToOptions diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index 1994292f7b658..0facb33e2a026 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_c5174c4663" + "Tag": "net/storage/Azure.Storage.Blobs_5c382dfb14" } diff --git a/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj index 568dd6cba9516..77fd767c3486c 100644 --- a/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj @@ -16,7 +16,6 @@ - diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index 9a110cf8eb13a..e70d5e02c82d7 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -1242,39 +1242,14 @@ internal async Task> AppendBlockInternal( BlobErrors.VerifyHttpsCustomerProvidedKey(Uri, ClientConfiguration.CustomerProvidedKey); Errors.VerifyStreamPosition(content, nameof(content)); - ContentHasher.GetHashResult hashResult = null; - long contentLength = (content?.Length - content?.Position) ?? 0; - long? structuredContentLength = default; - string structuredBodyType = null; - if (validationOptions != null && - validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - ClientSideEncryption == null) // don't allow feature combination - { - // report progress in terms of caller bytes, not encoded bytes - structuredContentLength = contentLength; - contentLength = (content?.Length - content?.Position) ?? 0; - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - content = content.WithNoDispose().WithProgress(progressHandler); - content = validationOptions.PrecalculatedChecksum.IsEmpty - ? new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64) - : new StructuredMessagePrecalculatedCrcWrapperStream( - content, - validationOptions.PrecalculatedChecksum.Span); - contentLength = (content?.Length - content?.Position) ?? 0; - } - else - { - // compute hash BEFORE attaching progress handler - hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - content = content.WithNoDispose().WithProgress(progressHandler); - } + // compute hash BEFORE attaching progress handler + ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + + content = content.WithNoDispose().WithProgress(progressHandler); ResponseWithHeaders response; @@ -1292,8 +1267,6 @@ internal async Task> AppendBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, ifModifiedSince: conditions?.IfModifiedSince, ifUnmodifiedSince: conditions?.IfUnmodifiedSince, ifMatch: conditions?.IfMatch?.ToString(), @@ -1316,8 +1289,6 @@ internal async Task> AppendBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, ifModifiedSince: conditions?.IfModifiedSince, ifUnmodifiedSince: conditions?.IfUnmodifiedSince, ifMatch: conditions?.IfMatch?.ToString(), diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index e2e54aa39cc84..79c36fd3da840 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -54,8 +54,6 @@ - - @@ -95,11 +93,6 @@ - - - - - diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 7b88735731f1b..5186b550b74e4 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1031,7 +1031,6 @@ private async Task> DownloadInternal( ContentHash = blobDownloadDetails.ContentHash, ContentLength = blobDownloadDetails.ContentLength, ContentType = blobDownloadDetails.ContentType, - ExpectTrailingDetails = blobDownloadStreamingResult.ExpectTrailingDetails, }, response.GetRawResponse()); } #endregion @@ -1548,52 +1547,30 @@ internal virtual async ValueTask> Download // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - ValueTask> Factory(long offset, bool async, CancellationToken cancellationToken) - => StartDownloadAsync( - range, - conditionsWithEtag, - validationOptions, - offset, - async, - cancellationToken); - async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)> StructuredMessageFactory( - long offset, bool async, CancellationToken cancellationToken) - { - Response result = await Factory(offset, async, cancellationToken).ConfigureAwait(false); - return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.Details.ContentLength); - } - Stream stream; - if (response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) - { - (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( - response.Value.Content, response.Value.Details.ContentLength); - stream = new StructuredMessageDecodingRetriableStream( - decodingStream, - decodedData, - StructuredMessage.Flags.StorageCrc64, - startOffset => StructuredMessageFactory(startOffset, async: false, cancellationToken) - .EnsureCompleted(), - async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) - .ConfigureAwait(false), - decodedData => - { - response.Value.Details.ContentCrc = new byte[StructuredMessage.Crc64Length]; - decodedData.Crc.WriteCrc64(response.Value.Details.ContentCrc); - }, - ClientConfiguration.Pipeline.ResponseClassifier, - Constants.MaxReliabilityRetries); - } - else - { - stream = RetriableStream.Create( - response.Value.Content, - startOffset => Factory(startOffset, async: false, cancellationToken) - .EnsureCompleted().Value.Content, - async startOffset => (await Factory(startOffset, async: true, cancellationToken) - .ConfigureAwait(false)).Value.Content, - ClientConfiguration.Pipeline.ResponseClassifier, - Constants.MaxReliabilityRetries); - } + Stream stream = RetriableStream.Create( + response.Value.Content, + startOffset => + StartDownloadAsync( + range, + conditionsWithEtag, + validationOptions, + startOffset, + async, + cancellationToken) + .EnsureCompleted() + .Value.Content, + async startOffset => + (await StartDownloadAsync( + range, + conditionsWithEtag, + validationOptions, + startOffset, + async, + cancellationToken) + .ConfigureAwait(false)) + .Value.Content, + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); stream = stream.WithProgress(progressHandler); @@ -1601,11 +1578,7 @@ ValueTask> Factory(long offset, bool async * Buffer response stream and ensure it matches the transactional checksum if any. * Storage will not return a checksum for payload >4MB, so this buffer is capped similarly. * Checksum validation is opt-in, so this buffer is part of that opt-in. */ - if (validationOptions != default && - validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && - validationOptions.AutoValidateChecksum && - // structured message decoding does the validation for us - !response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) + if (validationOptions != default && validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && validationOptions.AutoValidateChecksum) { // safe-buffer; transactional hash download limit well below maxInt var readDestStream = new MemoryStream((int)response.Value.Details.ContentLength); @@ -1676,8 +1649,8 @@ await ContentHasher.AssertResponseHashMatchInternal( /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded blob. contains + /// A describing the + /// downloaded blob. contains /// the blob's data. /// /// @@ -1716,29 +1689,13 @@ private async ValueTask> StartDownloadAsyn operationName: nameof(BlobBaseClient.Download), parameterName: nameof(conditions)); - bool? rangeGetContentMD5 = null; - bool? rangeGetContentCRC64 = null; - string structuredBodyType = null; - switch (validationOptions?.ChecksumAlgorithm.ResolveAuto()) - { - case StorageChecksumAlgorithm.MD5: - rangeGetContentMD5 = true; - break; - case StorageChecksumAlgorithm.StorageCrc64: - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - break; - default: - break; - } - if (async) { response = await BlobRestClient.DownloadAsync( range: pageRange?.ToString(), leaseId: conditions?.LeaseId, - rangeGetContentMD5: rangeGetContentMD5, - rangeGetContentCRC64: rangeGetContentCRC64, - structuredBodyType: structuredBodyType, + rangeGetContentMD5: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, + rangeGetContentCRC64: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? true : null, encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, @@ -1755,9 +1712,8 @@ private async ValueTask> StartDownloadAsyn response = BlobRestClient.Download( range: pageRange?.ToString(), leaseId: conditions?.LeaseId, - rangeGetContentMD5: rangeGetContentMD5, - rangeGetContentCRC64: rangeGetContentCRC64, - structuredBodyType: structuredBodyType, + rangeGetContentMD5: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, + rangeGetContentCRC64: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? true : null, encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, @@ -1773,11 +1729,9 @@ private async ValueTask> StartDownloadAsyn long length = response.IsUnavailable() ? 0 : response.Headers.ContentLength ?? 0; ClientConfiguration.Pipeline.LogTrace($"Response: {response.GetRawResponse().Status}, ContentLength: {length}"); - Response result = Response.FromValue( + return Response.FromValue( response.ToBlobDownloadStreamingResult(), response.GetRawResponse()); - result.Value.ExpectTrailingDetails = structuredBodyType != null; - return result; } #endregion diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs index e8def313432dc..0f90cb42066d2 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs @@ -323,8 +323,6 @@ private void AddHeadersAndQueryParameters() Diagnostics.LoggedHeaderNames.Add("x-ms-encryption-key-sha256"); Diagnostics.LoggedHeaderNames.Add("x-ms-copy-source-error-code"); Diagnostics.LoggedHeaderNames.Add("x-ms-copy-source-status-code"); - Diagnostics.LoggedHeaderNames.Add("x-ms-structured-body"); - Diagnostics.LoggedHeaderNames.Add("x-ms-structured-content-length"); Diagnostics.LoggedQueryParameters.Add("comp"); Diagnostics.LoggedQueryParameters.Add("maxresults"); diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs index 2ab39be2784cc..0d03c057fee52 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs @@ -187,7 +187,7 @@ private static bool CanIgnorePadding(ContentRange? contentRange) // did we request the last block? // end is inclusive/0-index, so end = n and size = n+1 means we requested the last block - if (contentRange.Value.TotalResourceLength - contentRange.Value.End == 1) + if (contentRange.Value.Size - contentRange.Value.End == 1) { return false; } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index 0d65fae7b5011..f9269ce6cb893 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -875,35 +875,14 @@ internal virtual async Task> UploadInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - ContentHasher.GetHashResult hashResult = null; - long contentLength = (content?.Length - content?.Position) ?? 0; - long? structuredContentLength = default; - string structuredBodyType = null; - if (content != null && - validationOptions != null && - validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - ClientSideEncryption == null) // don't allow feature combination - { - // report progress in terms of caller bytes, not encoded bytes - structuredContentLength = contentLength; - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - content = content.WithNoDispose().WithProgress(progressHandler); - content = new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64); - contentLength = content.Length - content.Position; - } - else - { - // compute hash BEFORE attaching progress handler - hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - content = content.WithNoDispose().WithProgress(progressHandler); - } + // compute hash BEFORE attaching progress handler + ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + + content = content?.WithNoDispose().WithProgress(progressHandler); ResponseWithHeaders response; @@ -942,8 +921,6 @@ internal virtual async Task> UploadInternal( legalHold: legalHold, transactionalContentMD5: hashResult?.MD5AsArray, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -976,8 +953,6 @@ internal virtual async Task> UploadInternal( legalHold: legalHold, transactionalContentMD5: hashResult?.MD5AsArray, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, cancellationToken: cancellationToken); } @@ -1330,39 +1305,14 @@ internal virtual async Task> StageBlockInternal( Errors.VerifyStreamPosition(content, nameof(content)); - ContentHasher.GetHashResult hashResult = null; - long contentLength = (content?.Length - content?.Position) ?? 0; - long? structuredContentLength = default; - string structuredBodyType = null; - if (validationOptions != null && - validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - ClientSideEncryption == null) // don't allow feature combination - { - // report progress in terms of caller bytes, not encoded bytes - structuredContentLength = contentLength; - contentLength = (content?.Length - content?.Position) ?? 0; - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - content = content.WithNoDispose().WithProgress(progressHandler); - content = validationOptions.PrecalculatedChecksum.IsEmpty - ? new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64) - : new StructuredMessagePrecalculatedCrcWrapperStream( - content, - validationOptions.PrecalculatedChecksum.Span); - contentLength = (content?.Length - content?.Position) ?? 0; - } - else - { - // compute hash BEFORE attaching progress handler - hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - content = content.WithNoDispose().WithProgress(progressHandler); - } + // compute hash BEFORE attaching progress handler + ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + + content = content.WithNoDispose().WithProgress(progressHandler); ResponseWithHeaders response; @@ -1370,7 +1320,7 @@ internal virtual async Task> StageBlockInternal( { response = await BlockBlobRestClient.StageBlockAsync( blockId: base64BlockId, - contentLength: contentLength, + contentLength: (content?.Length - content?.Position) ?? 0, body: content, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, transactionalContentMD5: hashResult?.MD5AsArray, @@ -1379,8 +1329,6 @@ internal virtual async Task> StageBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -1388,7 +1336,7 @@ internal virtual async Task> StageBlockInternal( { response = BlockBlobRestClient.StageBlock( blockId: base64BlockId, - contentLength: contentLength, + contentLength: (content?.Length - content?.Position) ?? 0, body: content, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, transactionalContentMD5: hashResult?.MD5AsArray, @@ -1397,8 +1345,6 @@ internal virtual async Task> StageBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, cancellationToken: cancellationToken); } @@ -2845,7 +2791,7 @@ internal async Task OpenWriteInternal( immutabilityPolicy: default, legalHold: default, progressHandler: default, - transferValidationOverride: new() { ChecksumAlgorithm = StorageChecksumAlgorithm.None }, + transferValidationOverride: default, operationName: default, async: async, cancellationToken: cancellationToken) diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs index 0490ec239798e..bc119822cdc12 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs @@ -34,14 +34,6 @@ public class BlobDownloadDetails public byte[] ContentHash { get; internal set; } #pragma warning restore CA1819 // Properties should not return arrays - /// - /// When requested using , this value contains the CRC for the download blob range. - /// This value may only become populated once the network stream is fully consumed. If this instance is accessed through - /// , the network stream has already been consumed. Otherwise, consume the content stream before - /// checking this value. - /// - public byte[] ContentCrc { get; internal set; } - /// /// Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob. /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs index b42801e36ab55..e034573b54b3a 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs @@ -4,8 +4,6 @@ using System; using System.ComponentModel; using System.IO; -using System.Threading.Tasks; -using Azure.Core; using Azure.Storage.Shared; namespace Azure.Storage.Blobs.Models @@ -51,14 +49,6 @@ public class BlobDownloadInfo : IDisposable, IDownloadedContent /// public BlobDownloadDetails Details { get; internal set; } - /// - /// Indicates some contents of are mixed into the response stream. - /// They will not be set until has been fully consumed. These details - /// will be extracted from the content stream by the library before the calling code can - /// encounter them. - /// - public bool ExpectTrailingDetails { get; internal set; } - /// /// Constructor. /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs index 9b7d4d4e00dad..4fbada6e67aad 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs @@ -24,14 +24,6 @@ internal BlobDownloadStreamingResult() { } /// public Stream Content { get; internal set; } - /// - /// Indicates some contents of are mixed into the response stream. - /// They will not be set until has been fully consumed. These details - /// will be extracted from the content stream by the library before the calling code can - /// encounter them. - /// - public bool ExpectTrailingDetails { get; internal set; } - /// /// Disposes the by calling Dispose on the underlying stream. /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs index 7038897531fbb..fa575e41b8ebe 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs @@ -1363,42 +1363,15 @@ internal async Task> UploadPagesInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - ContentHasher.GetHashResult hashResult = null; - long contentLength = (content?.Length - content?.Position) ?? 0; - long? structuredContentLength = default; - string structuredBodyType = null; - HttpRange range; - if (validationOptions != null && - validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - ClientSideEncryption == null) // don't allow feature combination - { - // report progress in terms of caller bytes, not encoded bytes - structuredContentLength = contentLength; - contentLength = (content?.Length - content?.Position) ?? 0; - range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - content = content?.WithNoDispose().WithProgress(progressHandler); - content = validationOptions.PrecalculatedChecksum.IsEmpty - ? new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64) - : new StructuredMessagePrecalculatedCrcWrapperStream( - content, - validationOptions.PrecalculatedChecksum.Span); - contentLength = (content?.Length - content?.Position) ?? 0; - } - else - { - // compute hash BEFORE attaching progress handler - hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - content = content?.WithNoDispose().WithProgress(progressHandler); - range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); - } + // compute hash BEFORE attaching progress handler + ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + + content = content?.WithNoDispose().WithProgress(progressHandler); + HttpRange range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); ResponseWithHeaders response; @@ -1415,8 +1388,6 @@ internal async Task> UploadPagesInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, ifSequenceNumberLessThanOrEqualTo: conditions?.IfSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan: conditions?.IfSequenceNumberLessThan, ifSequenceNumberEqualTo: conditions?.IfSequenceNumberEqual, @@ -1441,8 +1412,6 @@ internal async Task> UploadPagesInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, ifSequenceNumberLessThanOrEqualTo: conditions?.IfSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan: conditions?.IfSequenceNumberLessThan, ifSequenceNumberEqualTo: conditions?.IfSequenceNumberEqual, diff --git a/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs b/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs index 601ee2ac9d398..361594561ed7e 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs @@ -23,8 +23,6 @@ internal class PartitionedDownloader private const string _operationName = nameof(BlobBaseClient) + "." + nameof(BlobBaseClient.DownloadTo); private const string _innerOperationName = nameof(BlobBaseClient) + "." + nameof(BlobBaseClient.DownloadStreaming); - private const int Crc64Len = Constants.StorageCrc64SizeInBytes; - /// /// The client used to download the blob. /// @@ -51,7 +49,6 @@ internal class PartitionedDownloader /// private readonly StorageChecksumAlgorithm _validationAlgorithm; private readonly int _checksumSize; - // TODO disabling master crc temporarily. segment CRCs still handled. private bool UseMasterCrc => _validationAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64; private StorageCrc64HashAlgorithm _masterCrcCalculator = null; @@ -205,31 +202,20 @@ public async Task DownloadToInternal( } // Destination wrapped in master crc step if needed (must wait until after encryption wrap check) - byte[] composedCrcBuf = default; + Memory composedCrc = default; if (UseMasterCrc) { _masterCrcCalculator = StorageCrc64HashAlgorithm.Create(); destination = ChecksumCalculatingStream.GetWriteStream(destination, _masterCrcCalculator.Append); - disposables.Add(_arrayPool.RentDisposable(Crc64Len, out composedCrcBuf)); - composedCrcBuf.Clear(); + disposables.Add(_arrayPool.RentAsMemoryDisposable( + Constants.StorageCrc64SizeInBytes, out composedCrc)); + composedCrc.Span.Clear(); } // If the first segment was the entire blob, we'll copy that to // the output stream and finish now - long initialLength; - long totalLength; - // Get blob content length downloaded from content range when available to handle transit encoding - if (string.IsNullOrWhiteSpace(initialResponse.Value.Details.ContentRange)) - { - initialLength = initialResponse.Value.Details.ContentLength; - totalLength = 0; - } - else - { - ContentRange recievedRange = ContentRange.Parse(initialResponse.Value.Details.ContentRange); - initialLength = recievedRange.GetRangeLength(); - totalLength = recievedRange.TotalResourceLength.Value; - } + long initialLength = initialResponse.Value.Details.ContentLength; + long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange); if (initialLength == totalLength) { await HandleOneShotDownload(initialResponse, destination, async, cancellationToken) @@ -254,16 +240,15 @@ await HandleOneShotDownload(initialResponse, destination, async, cancellationTok } else { - using (_arrayPool.RentDisposable(_checksumSize, out byte[] partitionChecksum)) + using (_arrayPool.RentAsMemoryDisposable(_checksumSize, out Memory partitionChecksum)) { - await CopyToInternal(initialResponse, destination, new(partitionChecksum, 0, _checksumSize), async, cancellationToken).ConfigureAwait(false); + await CopyToInternal(initialResponse, destination, partitionChecksum, async, cancellationToken).ConfigureAwait(false); if (UseMasterCrc) { StorageCrc64Composer.Compose( - (composedCrcBuf, 0L), - (partitionChecksum, initialResponse.Value.Details.ContentRange.GetContentRangeLengthOrDefault() - ?? initialResponse.Value.Details.ContentLength) - ).AsSpan(0, Crc64Len).CopyTo(composedCrcBuf); + (composedCrc.ToArray(), 0L), + (partitionChecksum.ToArray(), initialResponse.Value.Details.ContentLength) + ).CopyTo(composedCrc); } } } @@ -302,16 +287,15 @@ await HandleOneShotDownload(initialResponse, destination, async, cancellationTok else { Response result = await responseValueTask.ConfigureAwait(false); - using (_arrayPool.RentDisposable(_checksumSize, out byte[] partitionChecksum)) + using (_arrayPool.RentAsMemoryDisposable(_checksumSize, out Memory partitionChecksum)) { - await CopyToInternal(result, destination, new(partitionChecksum, 0, _checksumSize), async, cancellationToken).ConfigureAwait(false); + await CopyToInternal(result, destination, partitionChecksum, async, cancellationToken).ConfigureAwait(false); if (UseMasterCrc) { StorageCrc64Composer.Compose( - (composedCrcBuf, 0L), - (partitionChecksum, result.Value.Details.ContentRange.GetContentRangeLengthOrDefault() - ?? result.Value.Details.ContentLength) - ).AsSpan(0, Crc64Len).CopyTo(composedCrcBuf); + (composedCrc.ToArray(), 0L), + (partitionChecksum.ToArray(), result.Value.Details.ContentLength) + ).CopyTo(composedCrc); } } } @@ -327,7 +311,7 @@ await HandleOneShotDownload(initialResponse, destination, async, cancellationTok } #pragma warning restore AZC0110 // DO NOT use await keyword in possibly synchronous scope. - await FinalizeDownloadInternal(destination, composedCrcBuf?.AsMemory(0, Crc64Len) ?? default, async, cancellationToken) + await FinalizeDownloadInternal(destination, composedCrc, async, cancellationToken) .ConfigureAwait(false); return initialResponse.GetRawResponse(); @@ -345,7 +329,7 @@ async Task ConsumeQueuedTask() // CopyToAsync causes ConsumeQueuedTask to wait until the // download is complete - using (_arrayPool.RentDisposable(_checksumSize, out byte[] partitionChecksum)) + using (_arrayPool.RentAsMemoryDisposable(_checksumSize, out Memory partitionChecksum)) { await CopyToInternal( response, @@ -354,14 +338,13 @@ await CopyToInternal( async, cancellationToken) .ConfigureAwait(false); - if (UseMasterCrc) - { - StorageCrc64Composer.Compose( - (composedCrcBuf, 0L), - (partitionChecksum, response.Value.Details.ContentRange.GetContentRangeLengthOrDefault() - ?? response.Value.Details.ContentLength) - ).AsSpan(0, Crc64Len).CopyTo(composedCrcBuf); - } + if (UseMasterCrc) + { + StorageCrc64Composer.Compose( + (composedCrc.ToArray(), 0L), + (partitionChecksum.ToArray(), response.Value.Details.ContentLength) + ).CopyTo(composedCrc); + } } } } @@ -408,7 +391,7 @@ await FinalizeDownloadInternal(destination, partitionChecksum, async, cancellati private async Task FinalizeDownloadInternal( Stream destination, - ReadOnlyMemory composedCrc, + Memory composedCrc, bool async, CancellationToken cancellationToken) { @@ -424,6 +407,20 @@ private async Task FinalizeDownloadInternal( } } + private static long ParseRangeTotalLength(string range) + { + if (range == null) + { + return 0; + } + int lengthSeparator = range.IndexOf("/", StringComparison.InvariantCultureIgnoreCase); + if (lengthSeparator == -1) + { + throw BlobErrors.ParsingFullHttpRangeFailed(range); + } + return long.Parse(range.Substring(lengthSeparator + 1), CultureInfo.InvariantCulture); + } + private async Task CopyToInternal( Response response, Stream destination, @@ -432,9 +429,7 @@ private async Task CopyToInternal( CancellationToken cancellationToken) { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); - // if structured message, this crc is validated in the decoding process. don't decode it here. - bool structuredMessage = response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader); - using IHasher hasher = structuredMessage ? null : ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm); + using IHasher hasher = ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm); using Stream rawSource = response.Value.Content; using Stream source = hasher != null ? ChecksumCalculatingStream.GetReadStream(rawSource, hasher.AppendHash) @@ -446,13 +441,7 @@ await source.CopyToInternal( cancellationToken) .ConfigureAwait(false); - // with structured message, the message integrity will already be validated, - // but we can still get the checksum out of the response object - if (structuredMessage) - { - response.Value.Details.ContentCrc?.CopyTo(checksumBuffer.Span); - } - else if (hasher != null) + if (hasher != null) { hasher.GetFinalHash(checksumBuffer.Span); (ReadOnlyMemory checksum, StorageChecksumAlgorithm _) diff --git a/sdk/storage/Azure.Storage.Blobs/src/autorest.md b/sdk/storage/Azure.Storage.Blobs/src/autorest.md index 72e1adb821b41..6c18c66066ebd 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/autorest.md +++ b/sdk/storage/Azure.Storage.Blobs/src/autorest.md @@ -34,7 +34,7 @@ directive: if (property.includes('/{containerName}/{blob}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ContainerName") && false == param['$ref'].endsWith("#/parameters/Blob"))}); - } + } else if (property.includes('/{containerName}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ContainerName"))}); @@ -158,7 +158,7 @@ directive: var newName = property.replace('/{containerName}/{blob}', ''); $[newName] = $[oldName]; delete $[oldName]; - } + } else if (property.includes('/{containerName}')) { var oldName = property; diff --git a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj index 1c3856c83b64e..62c7b6d17e63e 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj @@ -6,9 +6,6 @@ Microsoft Azure.Storage.Blobs client library tests false - - BlobSDK - diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs index 3ec448e6d1ed0..73d11612f1d8c 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using System; -using System.Buffers; using System.IO; using System.Threading.Tasks; using Azure.Core.TestFramework; @@ -39,10 +37,7 @@ protected override async Task> GetDispo StorageChecksumAlgorithm uploadAlgorithm = StorageChecksumAlgorithm.None, StorageChecksumAlgorithm downloadAlgorithm = StorageChecksumAlgorithm.None) { - var disposingContainer = await ClientBuilder.GetTestContainerAsync( - service: service, - containerName: containerName, - publicAccessType: PublicAccessType.None); + var disposingContainer = await ClientBuilder.GetTestContainerAsync(service: service, containerName: containerName); disposingContainer.Container.ClientConfiguration.TransferValidation.Upload.ChecksumAlgorithm = uploadAlgorithm; disposingContainer.Container.ClientConfiguration.TransferValidation.Download.ChecksumAlgorithm = downloadAlgorithm; @@ -96,96 +91,57 @@ public override void TestAutoResolve() } #region Added Tests - [Test] - public virtual async Task OlderServiceVersionThrowsOnStructuredMessage() + [TestCaseSource("GetValidationAlgorithms")] + public async Task ExpectedDownloadStreamingStreamTypeReturned(StorageChecksumAlgorithm algorithm) { - // use service version before structured message was introduced - await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( - service: ClientBuilder.GetServiceClient_SharedKey( - InstrumentClientOptions(new BlobClientOptions(BlobClientOptions.ServiceVersion.V2024_11_04))), - publicAccessType: PublicAccessType.None); + await using var test = await GetDisposingContainerAsync(); // Arrange - const int dataLength = Constants.KB; - var data = GetRandomBuffer(dataLength); - - var resourceName = GetNewResourceName(); - var blob = InstrumentClient(disposingContainer.Container.GetBlobClient(GetNewResourceName())); - await blob.UploadAsync(BinaryData.FromBytes(data)); - - var validationOptions = new DownloadTransferValidationOptions + var data = GetRandomBuffer(Constants.KB); + BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName())); + using (var stream = new MemoryStream(data)) { - ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 - }; - AsyncTestDelegate operation = async () => await (await blob.DownloadStreamingAsync( - new BlobDownloadOptions - { - Range = new HttpRange(length: Constants.StructuredMessage.MaxDownloadCrcWithHeader + 1), - TransferValidation = validationOptions, - })).Value.Content.CopyToAsync(Stream.Null); - Assert.That(operation, Throws.TypeOf()); - } - - [Test] - public async Task StructuredMessagePopulatesCrcDownloadStreaming() - { - await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( - publicAccessType: PublicAccessType.None); - - const int dataLength = Constants.KB; - byte[] data = GetRandomBuffer(dataLength); - byte[] dataCrc = new byte[8]; - StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); - - var blob = disposingContainer.Container.GetBlobClient(GetNewResourceName()); - await blob.UploadAsync(BinaryData.FromBytes(data)); + await blob.UploadAsync(stream); + } + // don't make options instance at all for no hash request + DownloadTransferValidationOptions transferValidation = algorithm == StorageChecksumAlgorithm.None + ? default + : new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; - Response response = await blob.DownloadStreamingAsync(new() + // Act + Response response = await blob.DownloadStreamingAsync(new BlobDownloadOptions { - TransferValidation = new DownloadTransferValidationOptions - { - ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 - } + TransferValidation = transferValidation, + Range = new HttpRange(length: data.Length) }); - // crc is not present until response stream is consumed - Assert.That(response.Value.Details.ContentCrc, Is.Null); - - byte[] downloadedData; - using (MemoryStream ms = new()) - { - await response.Value.Content.CopyToAsync(ms); - downloadedData = ms.ToArray(); - } - - Assert.That(response.Value.Details.ContentCrc, Is.EqualTo(dataCrc)); - Assert.That(downloadedData, Is.EqualTo(data)); + // Assert + // validated stream is buffered + Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType()); } [Test] - public async Task StructuredMessagePopulatesCrcDownloadContent() + public async Task ExpectedDownloadStreamingStreamTypeReturned_None() { - await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( - publicAccessType: PublicAccessType.None); + await using var test = await GetDisposingContainerAsync(); - const int dataLength = Constants.KB; - byte[] data = GetRandomBuffer(dataLength); - byte[] dataCrc = new byte[8]; - StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); - - var blob = disposingContainer.Container.GetBlobClient(GetNewResourceName()); - await blob.UploadAsync(BinaryData.FromBytes(data)); + // Arrange + var data = GetRandomBuffer(Constants.KB); + BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName())); + using (var stream = new MemoryStream(data)) + { + await blob.UploadAsync(stream); + } - Response response = await blob.DownloadContentAsync(new BlobDownloadOptions() + // Act + Response response = await blob.DownloadStreamingAsync(new BlobDownloadOptions { - TransferValidation = new DownloadTransferValidationOptions - { - ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 - } + Range = new HttpRange(length: data.Length) }); - Assert.That(response.Value.Details.ContentCrc, Is.EqualTo(dataCrc)); - Assert.That(response.Value.Content.ToArray(), Is.EqualTo(data)); + // Assert + // unvalidated stream type is private; just check we didn't get back a buffered stream + Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType()); } #endregion } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs index 9b0fe99f6cb98..20ad6f6afb4b1 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -1358,7 +1358,7 @@ public void CanParseLargeContentRange() { long compareValue = (long)Int32.MaxValue + 1; //Increase max int32 by one ContentRange contentRange = ContentRange.Parse($"bytes 0 {compareValue} {compareValue}"); - Assert.AreEqual((long)Int32.MaxValue + 1, contentRange.TotalResourceLength); + Assert.AreEqual((long)Int32.MaxValue + 1, contentRange.Size); Assert.AreEqual(0, contentRange.Start); Assert.AreEqual((long)Int32.MaxValue + 1, contentRange.End); } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs index af408264c5bfa..d8d4756a510c1 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs @@ -305,7 +305,7 @@ public Response GetStream(HttpRange range, BlobRequ ContentHash = new byte[] { 1, 2, 3 }, LastModified = DateTimeOffset.Now, Metadata = new Dictionary() { { "meta", "data" } }, - ContentRange = $"bytes {range.Offset}-{Math.Max(1, range.Offset + contentLength - 1)}/{_length}", + ContentRange = $"bytes {range.Offset}-{range.Offset + contentLength}/{_length}", ETag = s_etag, ContentEncoding = "test", CacheControl = "test", diff --git a/sdk/storage/Azure.Storage.Common/samples/Azure.Storage.Common.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Common/samples/Azure.Storage.Common.Samples.Tests.csproj index aeca4497a8770..7d454aeaa0af2 100644 --- a/sdk/storage/Azure.Storage.Common/samples/Azure.Storage.Common.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Common/samples/Azure.Storage.Common.Samples.Tests.csproj @@ -19,7 +19,6 @@ - PreserveNewest diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs deleted file mode 100644 index 48304640eee43..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers.Binary; - -namespace Azure.Storage; - -internal static class ChecksumExtensions -{ - public static void WriteCrc64(this ulong crc, Span dest) - => BinaryPrimitives.WriteUInt64LittleEndian(dest, crc); - - public static bool TryWriteCrc64(this ulong crc, Span dest) - => BinaryPrimitives.TryWriteUInt64LittleEndian(dest, crc); - - public static ulong ReadCrc64(this ReadOnlySpan crc) - => BinaryPrimitives.ReadUInt64LittleEndian(crc); - - public static bool TryReadCrc64(this ReadOnlySpan crc, out ulong value) - => BinaryPrimitives.TryReadUInt64LittleEndian(crc, out value); -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index 48f93c8422fb4..c16ed90ead55b 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -657,15 +657,6 @@ internal static class AccountResources internal static readonly int[] PathStylePorts = { 10000, 10001, 10002, 10003, 10004, 10100, 10101, 10102, 10103, 10104, 11000, 11001, 11002, 11003, 11004, 11100, 11101, 11102, 11103, 11104 }; } - internal static class StructuredMessage - { - public const string StructuredMessageHeader = "x-ms-structured-body"; - public const string StructuredContentLength = "x-ms-structured-content-length"; - public const string CrcStructuredMessage = "XSM/1.0; properties=crc64"; - public const int DefaultSegmentContentLength = 4 * MB; - public const int MaxDownloadCrcWithHeader = 4 * MB; - } - internal static class ClientSideEncryption { public const string HttpMessagePropertyKeyV1 = "Azure.Storage.StorageTelemetryPolicy.ClientSideEncryption.V1"; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ContentRange.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ContentRange.cs index cb3b0a7bee189..f656382efad2b 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ContentRange.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ContentRange.cs @@ -82,20 +82,20 @@ public RangeUnit(string value) public long? End { get; } /// - /// Size of the entire resource this range is from, measured in this instance's . + /// Size of this range, measured in this instance's . /// - public long? TotalResourceLength { get; } + public long? Size { get; } /// /// Unit this range is measured in. Generally "bytes". /// public RangeUnit Unit { get; } - public ContentRange(RangeUnit unit, long? start, long? end, long? totalResourceLength) + public ContentRange(RangeUnit unit, long? start, long? end, long? size) { Start = start; End = end; - TotalResourceLength = totalResourceLength; + Size = size; Unit = unit; } @@ -113,7 +113,7 @@ public static ContentRange Parse(string headerValue) string unit = default; long? start = default; long? end = default; - long? resourceSize = default; + long? size = default; try { @@ -136,10 +136,10 @@ public static ContentRange Parse(string headerValue) var rawSize = tokens[blobSizeIndex]; if (rawSize != WildcardMarker) { - resourceSize = long.Parse(rawSize, CultureInfo.InvariantCulture); + size = long.Parse(rawSize, CultureInfo.InvariantCulture); } - return new ContentRange(unit, start, end, resourceSize); + return new ContentRange(unit, start, end, size); } catch (IndexOutOfRangeException) { @@ -165,7 +165,7 @@ public static HttpRange ToHttpRange(ContentRange contentRange) /// /// Indicates whether this instance and a specified are equal /// - public bool Equals(ContentRange other) => (other.Start == Start) && (other.End == End) && (other.Unit == Unit) && (other.TotalResourceLength == TotalResourceLength); + public bool Equals(ContentRange other) => (other.Start == Start) && (other.End == End) && (other.Unit == Unit) && (other.Size == Size); /// /// Determines if two values are the same. @@ -185,6 +185,6 @@ public static HttpRange ToHttpRange(ContentRange contentRange) /// [EditorBrowsable(EditorBrowsableState.Never)] - public override int GetHashCode() => HashCodeBuilder.Combine(Start, End, TotalResourceLength, Unit.GetHashCode()); + public override int GetHashCode() => HashCodeBuilder.Combine(Start, End, Size, Unit.GetHashCode()); } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ContentRangeExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ContentRangeExtensions.cs deleted file mode 100644 index 160a69b19a9c8..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ContentRangeExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -namespace Azure.Storage.Cryptography; - -internal static class ContentRangeExtensions -{ - public static long? GetContentRangeLengthOrDefault(this string contentRange) - => string.IsNullOrWhiteSpace(contentRange) - ? default : ContentRange.Parse(contentRange).GetRangeLength(); - - public static long GetRangeLength(this ContentRange contentRange) - => contentRange.End.Value - contentRange.Start.Value + 1; -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs index 867607e551e6a..2a5fe38668104 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs @@ -3,7 +3,6 @@ using System; using System.Globalization; -using System.IO; using System.Linq; using System.Security.Authentication; using System.Xml.Serialization; @@ -106,18 +105,9 @@ public static ArgumentException VersionNotSupported(string paramName) public static RequestFailedException ClientRequestIdMismatch(Response response, string echo, string original) => new RequestFailedException(response.Status, $"Response x-ms-client-request-id '{echo}' does not match the original expected request id, '{original}'.", null); - public static InvalidDataException StructuredMessageNotAcknowledgedGET(Response response) - => new InvalidDataException($"Response does not acknowledge structured message was requested. Unknown data structure in response body."); - - public static InvalidDataException StructuredMessageNotAcknowledgedPUT(Response response) - => new InvalidDataException($"Response does not acknowledge structured message was sent. Unexpected data may have been persisted to storage."); - public static ArgumentException TransactionalHashingNotSupportedWithClientSideEncryption() => new ArgumentException("Client-side encryption and transactional hashing are not supported at the same time."); - public static InvalidDataException ExpectedStructuredMessage() - => new InvalidDataException($"Expected {Constants.StructuredMessage.StructuredMessageHeader} in response, but found none."); - public static void VerifyHttpsTokenAuth(Uri uri) { if (uri.Scheme != Constants.Https) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs index e3372665928c1..6b89a59011d51 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs @@ -72,9 +72,6 @@ public static ArgumentException CannotDeferTransactionalHashVerification() public static ArgumentException CannotInitializeWriteStreamWithData() => new ArgumentException("Initialized buffer for StorageWriteStream must be empty."); - public static InvalidDataException InvalidStructuredMessage(string optionalMessage = default) - => new InvalidDataException(("Invalid structured message data. " + optionalMessage ?? "").Trim()); - internal static void VerifyStreamPosition(Stream stream, string streamName) { if (stream != null && stream.CanSeek && stream.Length > 0 && stream.Position >= stream.Length) @@ -83,22 +80,6 @@ internal static void VerifyStreamPosition(Stream stream, string streamName) } } - internal static void AssertBufferMinimumSize(ReadOnlySpan buffer, int minSize, string paramName) - { - if (buffer.Length < minSize) - { - throw new ArgumentException($"Expected buffer Length of at least {minSize} bytes. Got {buffer.Length}.", paramName); - } - } - - internal static void AssertBufferExactSize(ReadOnlySpan buffer, int size, string paramName) - { - if (buffer.Length != size) - { - throw new ArgumentException($"Expected buffer Length of exactly {size} bytes. Got {buffer.Length}.", paramName); - } - } - public static void ThrowIfParamNull(object obj, string paramName) { if (obj == null) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs index fe2db427bef02..c3e9c641c3fea 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs @@ -249,9 +249,41 @@ private async Task DownloadInternal(bool async, CancellationToken cancellat response = await _downloadInternalFunc(range, _validationOptions, async, cancellationToken).ConfigureAwait(false); using Stream networkStream = response.Value.Content; - // use stream copy to ensure consumption of any trailing metadata (e.g. structured message) - // allow buffer limits to catch the error of data size mismatch - int totalCopiedBytes = (int) await networkStream.CopyToInternal(new MemoryStream(_buffer), async, cancellationToken).ConfigureAwait((false)); + + // The number of bytes we just downloaded. + long downloadSize = GetResponseRange(response.GetRawResponse()).Length.Value; + + // The number of bytes we copied in the last loop. + int copiedBytes; + + // Bytes we have copied so far. + int totalCopiedBytes = 0; + + // Bytes remaining to copy. It is save to truncate the long because we asked for a max of int _buffer size bytes. + int remainingBytes = (int)downloadSize; + + do + { + if (async) + { + copiedBytes = await networkStream.ReadAsync( + buffer: _buffer, + offset: totalCopiedBytes, + count: remainingBytes, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + copiedBytes = networkStream.Read( + buffer: _buffer, + offset: totalCopiedBytes, + count: remainingBytes); + } + + totalCopiedBytes += copiedBytes; + remainingBytes -= copiedBytes; + } + while (copiedBytes != 0); _bufferPosition = 0; _bufferLength = totalCopiedBytes; @@ -259,7 +291,7 @@ private async Task DownloadInternal(bool async, CancellationToken cancellat // if we deferred transactional hash validation on download, validate now // currently we always defer but that may change - if (_validationOptions != default && _validationOptions.ChecksumAlgorithm == StorageChecksumAlgorithm.MD5 && !_validationOptions.AutoValidateChecksum) // TODO better condition + if (_validationOptions != default && _validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && !_validationOptions.AutoValidateChecksum) { ContentHasher.AssertResponseHashMatch(_buffer, _bufferPosition, _bufferLength, _validationOptions.ChecksumAlgorithm, response.GetRawResponse()); } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs index 6070329d10d3d..3e218d18a90af 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs @@ -251,7 +251,7 @@ public override int Read(byte[] buffer, int offset, int count) Length - Position, bufferCount - (Position - offsetOfBuffer), count - read); - Array.Copy(currentBuffer, Position - offsetOfBuffer, buffer, offset + read, toCopy); + Array.Copy(currentBuffer, Position - offsetOfBuffer, buffer, read, toCopy); read += toCopy; Position += toCopy; } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs index 307ff23b21144..ab6b76d78a87e 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs @@ -12,52 +12,22 @@ namespace Azure.Storage /// internal static class StorageCrc64Composer { - public static byte[] Compose(params (byte[] Crc64, long OriginalDataLength)[] partitions) - => Compose(partitions.AsEnumerable()); - - public static byte[] Compose(IEnumerable<(byte[] Crc64, long OriginalDataLength)> partitions) - { - ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64, 0), tup.OriginalDataLength))); - return BitConverter.GetBytes(result); - } - - public static byte[] Compose(params (ReadOnlyMemory Crc64, long OriginalDataLength)[] partitions) - => Compose(partitions.AsEnumerable()); - - public static byte[] Compose(IEnumerable<(ReadOnlyMemory Crc64, long OriginalDataLength)> partitions) + public static Memory Compose(params (byte[] Crc64, long OriginalDataLength)[] partitions) { -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64.Span), tup.OriginalDataLength))); -#else - ulong result = Compose(partitions.Select(tup => (System.BitConverter.ToUInt64(tup.Crc64.ToArray(), 0), tup.OriginalDataLength))); -#endif - return BitConverter.GetBytes(result); + return Compose(partitions.AsEnumerable()); } - public static byte[] Compose( - ReadOnlySpan leftCrc64, long leftOriginalDataLength, - ReadOnlySpan rightCrc64, long rightOriginalDataLength) + public static Memory Compose(IEnumerable<(byte[] Crc64, long OriginalDataLength)> partitions) { -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - ulong result = Compose( - (BitConverter.ToUInt64(leftCrc64), leftOriginalDataLength), - (BitConverter.ToUInt64(rightCrc64), rightOriginalDataLength)); -#else - ulong result = Compose( - (BitConverter.ToUInt64(leftCrc64.ToArray(), 0), leftOriginalDataLength), - (BitConverter.ToUInt64(rightCrc64.ToArray(), 0), rightOriginalDataLength)); -#endif - return BitConverter.GetBytes(result); + ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64, 0), tup.OriginalDataLength))); + return new Memory(BitConverter.GetBytes(result)); } - public static ulong Compose(params (ulong Crc64, long OriginalDataLength)[] partitions) - => Compose(partitions.AsEnumerable()); - public static ulong Compose(IEnumerable<(ulong Crc64, long OriginalDataLength)> partitions) { ulong composedCrc = 0; long composedDataLength = 0; - foreach ((ulong crc64, long originalDataLength) in partitions) + foreach (var tup in partitions) { composedCrc = StorageCrc64Calculator.Concatenate( uInitialCrcAB: 0, @@ -65,9 +35,9 @@ public static ulong Compose(IEnumerable<(ulong Crc64, long OriginalDataLength)> uFinalCrcA: composedCrc, uSizeA: (ulong) composedDataLength, uInitialCrcB: 0, - uFinalCrcB: crc64, - uSizeB: (ulong)originalDataLength); - composedDataLength += originalDataLength; + uFinalCrcB: tup.Crc64, + uSizeB: (ulong)tup.OriginalDataLength); + composedDataLength += tup.OriginalDataLength; } return composedCrc; } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs index 9f4ddb5249e82..0cef4f4d8d4ed 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs @@ -33,35 +33,6 @@ public override void OnReceivedResponse(HttpMessage message) { throw Errors.ClientRequestIdMismatch(message.Response, echo.First(), original); } - - if (message.Request.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader) && - message.Request.Headers.Contains(Constants.StructuredMessage.StructuredContentLength)) - { - AssertStructuredMessageAcknowledgedPUT(message); - } - else if (message.Request.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) - { - AssertStructuredMessageAcknowledgedGET(message); - } - } - - private static void AssertStructuredMessageAcknowledgedPUT(HttpMessage message) - { - if (!message.Response.IsError && - !message.Response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) - { - throw Errors.StructuredMessageNotAcknowledgedPUT(message.Response); - } - } - - private static void AssertStructuredMessageAcknowledgedGET(HttpMessage message) - { - if (!message.Response.IsError && - !(message.Response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader) && - message.Response.Headers.Contains(Constants.StructuredMessage.StructuredContentLength))) - { - throw Errors.StructuredMessageNotAcknowledgedGET(message.Response); - } } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs index c8803ecf421e7..31f121d414ea4 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using System; -using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -50,7 +48,7 @@ public static async Task WriteInternal( } } - public static Task CopyToInternal( + public static Task CopyToInternal( this Stream src, Stream dest, bool async, @@ -81,33 +79,21 @@ public static Task CopyToInternal( /// Cancellation token for the operation. /// /// - public static async Task CopyToInternal( + public static async Task CopyToInternal( this Stream src, Stream dest, int bufferSize, bool async, CancellationToken cancellationToken) { - using IDisposable _ = ArrayPool.Shared.RentDisposable(bufferSize, out byte[] buffer); - long totalRead = 0; - int read; if (async) { - while (0 < (read = await src.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false))) - { - totalRead += read; - await dest.WriteAsync(buffer, 0, read, cancellationToken).ConfigureAwait(false); - } + await src.CopyToAsync(dest, bufferSize, cancellationToken).ConfigureAwait(false); } else { - while (0 < (read = src.Read(buffer, 0, buffer.Length))) - { - totalRead += read; - dest.Write(buffer, 0, read); - } + src.CopyTo(dest, bufferSize); } - return totalRead; } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs deleted file mode 100644 index a0a46837797b9..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers; -using System.Buffers.Binary; -using System.IO; -using Azure.Storage.Common; - -namespace Azure.Storage.Shared; - -internal static class StructuredMessage -{ - public const int Crc64Length = 8; - - [Flags] - public enum Flags - { - None = 0, - StorageCrc64 = 1, - } - - public static class V1_0 - { - public const byte MessageVersionByte = 1; - - public const int StreamHeaderLength = 13; - public const int StreamHeaderVersionOffset = 0; - public const int StreamHeaderMessageLengthOffset = 1; - public const int StreamHeaderFlagsOffset = 9; - public const int StreamHeaderSegmentCountOffset = 11; - - public const int SegmentHeaderLength = 10; - public const int SegmentHeaderNumOffset = 0; - public const int SegmentHeaderContentLengthOffset = 2; - - #region Stream Header - public static void ReadStreamHeader( - ReadOnlySpan buffer, - out long messageLength, - out Flags flags, - out int totalSegments) - { - Errors.AssertBufferExactSize(buffer, 13, nameof(buffer)); - if (buffer[StreamHeaderVersionOffset] != 1) - { - throw new InvalidDataException("Unrecognized version of structured message."); - } - messageLength = (long)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(StreamHeaderMessageLengthOffset, 8)); - flags = (Flags)BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(StreamHeaderFlagsOffset, 2)); - totalSegments = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(StreamHeaderSegmentCountOffset, 2)); - } - - public static int WriteStreamHeader( - Span buffer, - long messageLength, - Flags flags, - int totalSegments) - { - const int versionOffset = 0; - const int messageLengthOffset = 1; - const int flagsOffset = 9; - const int numSegmentsOffset = 11; - - Errors.AssertBufferMinimumSize(buffer, StreamHeaderLength, nameof(buffer)); - - buffer[versionOffset] = MessageVersionByte; - BinaryPrimitives.WriteUInt64LittleEndian(buffer.Slice(messageLengthOffset, 8), (ulong)messageLength); - BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(flagsOffset, 2), (ushort)flags); - BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(numSegmentsOffset, 2), (ushort)totalSegments); - - return StreamHeaderLength; - } - - /// - /// Gets stream header in a buffer rented from the provided ArrayPool. - /// - /// - /// Disposable to return the buffer to the pool. - /// - public static IDisposable GetStreamHeaderBytes( - ArrayPool pool, - out Memory bytes, - long messageLength, - Flags flags, - int totalSegments) - { - Argument.AssertNotNull(pool, nameof(pool)); - IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); - WriteStreamHeader(bytes.Span, messageLength, flags, totalSegments); - return disposable; - } - #endregion - - #region StreamFooter - public static int GetStreamFooterSize(Flags flags) - => flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; - - public static void ReadStreamFooter( - ReadOnlySpan buffer, - Flags flags, - out ulong crc64) - { - int expectedBufferSize = GetSegmentFooterSize(flags); - Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); - - crc64 = flags.HasFlag(Flags.StorageCrc64) ? buffer.ReadCrc64() : default; - } - - public static int WriteStreamFooter(Span buffer, ReadOnlySpan crc64 = default) - { - int requiredSpace = 0; - if (!crc64.IsEmpty) - { - Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); - requiredSpace += Crc64Length; - } - - Errors.AssertBufferMinimumSize(buffer, requiredSpace, nameof(buffer)); - int offset = 0; - if (!crc64.IsEmpty) - { - crc64.CopyTo(buffer.Slice(offset, Crc64Length)); - offset += Crc64Length; - } - - return offset; - } - - /// - /// Gets stream header in a buffer rented from the provided ArrayPool. - /// - /// - /// Disposable to return the buffer to the pool. - /// - public static IDisposable GetStreamFooterBytes( - ArrayPool pool, - out Memory bytes, - ReadOnlySpan crc64 = default) - { - Argument.AssertNotNull(pool, nameof(pool)); - IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); - WriteStreamFooter(bytes.Span, crc64); - return disposable; - } - #endregion - - #region SegmentHeader - public static void ReadSegmentHeader( - ReadOnlySpan buffer, - out int segmentNum, - out long contentLength) - { - Errors.AssertBufferExactSize(buffer, 10, nameof(buffer)); - segmentNum = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(0, 2)); - contentLength = (long)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(2, 8)); - } - - public static int WriteSegmentHeader(Span buffer, int segmentNum, long segmentLength) - { - const int segmentNumOffset = 0; - const int segmentLengthOffset = 2; - - Errors.AssertBufferMinimumSize(buffer, SegmentHeaderLength, nameof(buffer)); - - BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(segmentNumOffset, 2), (ushort)segmentNum); - BinaryPrimitives.WriteUInt64LittleEndian(buffer.Slice(segmentLengthOffset, 8), (ulong)segmentLength); - - return SegmentHeaderLength; - } - - /// - /// Gets segment header in a buffer rented from the provided ArrayPool. - /// - /// - /// Disposable to return the buffer to the pool. - /// - public static IDisposable GetSegmentHeaderBytes( - ArrayPool pool, - out Memory bytes, - int segmentNum, - long segmentLength) - { - Argument.AssertNotNull(pool, nameof(pool)); - IDisposable disposable = pool.RentAsMemoryDisposable(SegmentHeaderLength, out bytes); - WriteSegmentHeader(bytes.Span, segmentNum, segmentLength); - return disposable; - } - #endregion - - #region SegmentFooter - public static int GetSegmentFooterSize(Flags flags) - => flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; - - public static void ReadSegmentFooter( - ReadOnlySpan buffer, - Flags flags, - out ulong crc64) - { - int expectedBufferSize = GetSegmentFooterSize(flags); - Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); - - crc64 = flags.HasFlag(Flags.StorageCrc64) ? buffer.ReadCrc64() : default; - } - - public static int WriteSegmentFooter(Span buffer, ReadOnlySpan crc64 = default) - { - int requiredSpace = 0; - if (!crc64.IsEmpty) - { - Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); - requiredSpace += Crc64Length; - } - - Errors.AssertBufferMinimumSize(buffer, requiredSpace, nameof(buffer)); - int offset = 0; - if (!crc64.IsEmpty) - { - crc64.CopyTo(buffer.Slice(offset, Crc64Length)); - offset += Crc64Length; - } - - return offset; - } - - /// - /// Gets stream header in a buffer rented from the provided ArrayPool. - /// - /// - /// Disposable to return the buffer to the pool. - /// - public static IDisposable GetSegmentFooterBytes( - ArrayPool pool, - out Memory bytes, - ReadOnlySpan crc64 = default) - { - Argument.AssertNotNull(pool, nameof(pool)); - IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); - WriteSegmentFooter(bytes.Span, crc64); - return disposable; - } - #endregion - } -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs deleted file mode 100644 index 22dfaef259972..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers; -using System.Buffers.Binary; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Azure.Core; -using Azure.Core.Pipeline; - -namespace Azure.Storage.Shared; - -internal class StructuredMessageDecodingRetriableStream : Stream -{ - public class DecodedData - { - public ulong Crc { get; set; } - } - - private readonly Stream _innerRetriable; - private long _decodedBytesRead; - - private readonly StructuredMessage.Flags _expectedFlags; - private readonly List _decodedDatas; - private readonly Action _onComplete; - - private StorageCrc64HashAlgorithm _totalContentCrc; - - private readonly Func _decodingStreamFactory; - private readonly Func> _decodingAsyncStreamFactory; - - public StructuredMessageDecodingRetriableStream( - Stream initialDecodingStream, - StructuredMessageDecodingStream.RawDecodedData initialDecodedData, - StructuredMessage.Flags expectedFlags, - Func decodingStreamFactory, - Func> decodingAsyncStreamFactory, - Action onComplete, - ResponseClassifier responseClassifier, - int maxRetries) - { - _decodingStreamFactory = decodingStreamFactory; - _decodingAsyncStreamFactory = decodingAsyncStreamFactory; - _innerRetriable = RetriableStream.Create(initialDecodingStream, StreamFactory, StreamFactoryAsync, responseClassifier, maxRetries); - _decodedDatas = new() { initialDecodedData }; - _expectedFlags = expectedFlags; - _onComplete = onComplete; - - if (expectedFlags.HasFlag(StructuredMessage.Flags.StorageCrc64)) - { - _totalContentCrc = StorageCrc64HashAlgorithm.Create(); - } - } - - private Stream StreamFactory(long _) - { - long offset = _decodedDatas.SelectMany(d => d.SegmentCrcs).Select(s => s.SegmentLen).Sum(); - (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = _decodingStreamFactory(offset); - _decodedDatas.Add(decodedData); - FastForwardInternal(decodingStream, _decodedBytesRead - offset, false).EnsureCompleted(); - return decodingStream; - } - - private async ValueTask StreamFactoryAsync(long _) - { - long offset = _decodedDatas.SelectMany(d => d.SegmentCrcs).Select(s => s.SegmentLen).Sum(); - (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = await _decodingAsyncStreamFactory(offset).ConfigureAwait(false); - _decodedDatas.Add(decodedData); - await FastForwardInternal(decodingStream, _decodedBytesRead - offset, true).ConfigureAwait(false); - return decodingStream; - } - - private static async ValueTask FastForwardInternal(Stream stream, long bytes, bool async) - { - using (ArrayPool.Shared.RentDisposable(4 * Constants.KB, out byte[] buffer)) - { - if (async) - { - while (bytes > 0) - { - bytes -= await stream.ReadAsync(buffer, 0, (int)Math.Min(bytes, buffer.Length)).ConfigureAwait(false); - } - } - else - { - while (bytes > 0) - { - bytes -= stream.Read(buffer, 0, (int)Math.Min(bytes, buffer.Length)); - } - } - } - } - - protected override void Dispose(bool disposing) - { - _decodedDatas.Clear(); - _innerRetriable.Dispose(); - } - - private void OnCompleted() - { - DecodedData final = new(); - if (_totalContentCrc != null) - { - final.Crc = ValidateCrc(); - } - _onComplete?.Invoke(final); - } - - private ulong ValidateCrc() - { - using IDisposable _ = ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf); - Span calculatedBytes = new(buf, 0, StructuredMessage.Crc64Length); - _totalContentCrc.GetCurrentHash(calculatedBytes); - ulong calculated = BinaryPrimitives.ReadUInt64LittleEndian(calculatedBytes); - - ulong reported = _decodedDatas.Count == 1 - ? _decodedDatas.First().TotalCrc.Value - : StorageCrc64Composer.Compose(_decodedDatas.SelectMany(d => d.SegmentCrcs)); - - if (calculated != reported) - { - Span reportedBytes = new(buf, calculatedBytes.Length, StructuredMessage.Crc64Length); - BinaryPrimitives.WriteUInt64LittleEndian(reportedBytes, reported); - throw Errors.ChecksumMismatch(calculatedBytes, reportedBytes); - } - - return calculated; - } - - #region Read - public override int Read(byte[] buffer, int offset, int count) - { - int read = _innerRetriable.Read(buffer, offset, count); - _decodedBytesRead += read; - if (read == 0) - { - OnCompleted(); - } - else - { - _totalContentCrc?.Append(new ReadOnlySpan(buffer, offset, read)); - } - return read; - } - - public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - int read = await _innerRetriable.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); - _decodedBytesRead += read; - if (read == 0) - { - OnCompleted(); - } - else - { - _totalContentCrc?.Append(new ReadOnlySpan(buffer, offset, read)); - } - return read; - } - -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - public override int Read(Span buffer) - { - int read = _innerRetriable.Read(buffer); - _decodedBytesRead += read; - if (read == 0) - { - OnCompleted(); - } - else - { - _totalContentCrc?.Append(buffer.Slice(0, read)); - } - return read; - } - - public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) - { - int read = await _innerRetriable.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); - _decodedBytesRead += read; - if (read == 0) - { - OnCompleted(); - } - else - { - _totalContentCrc?.Append(buffer.Span.Slice(0, read)); - } - return read; - } -#endif - - public override int ReadByte() - { - int val = _innerRetriable.ReadByte(); - _decodedBytesRead += 1; - if (val == -1) - { - OnCompleted(); - } - return val; - } - - public override int EndRead(IAsyncResult asyncResult) - { - int read = _innerRetriable.EndRead(asyncResult); - _decodedBytesRead += read; - if (read == 0) - { - OnCompleted(); - } - return read; - } - #endregion - - #region Passthru - public override bool CanRead => _innerRetriable.CanRead; - - public override bool CanSeek => _innerRetriable.CanSeek; - - public override bool CanWrite => _innerRetriable.CanWrite; - - public override bool CanTimeout => _innerRetriable.CanTimeout; - - public override long Length => _innerRetriable.Length; - - public override long Position { get => _innerRetriable.Position; set => _innerRetriable.Position = value; } - - public override void Flush() => _innerRetriable.Flush(); - - public override Task FlushAsync(CancellationToken cancellationToken) => _innerRetriable.FlushAsync(cancellationToken); - - public override long Seek(long offset, SeekOrigin origin) => _innerRetriable.Seek(offset, origin); - - public override void SetLength(long value) => _innerRetriable.SetLength(value); - - public override void Write(byte[] buffer, int offset, int count) => _innerRetriable.Write(buffer, offset, count); - - public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => _innerRetriable.WriteAsync(buffer, offset, count, cancellationToken); - - public override void WriteByte(byte value) => _innerRetriable.WriteByte(value); - - public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback callback, object state) => _innerRetriable.BeginWrite(buffer, offset, count, callback, state); - - public override void EndWrite(IAsyncResult asyncResult) => _innerRetriable.EndWrite(asyncResult); - - public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback callback, object state) => _innerRetriable.BeginRead(buffer, offset, count, callback, state); - - public override int ReadTimeout { get => _innerRetriable.ReadTimeout; set => _innerRetriable.ReadTimeout = value; } - - public override int WriteTimeout { get => _innerRetriable.WriteTimeout; set => _innerRetriable.WriteTimeout = value; } - -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - public override void Write(ReadOnlySpan buffer) => _innerRetriable.Write(buffer); - - public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => _innerRetriable.WriteAsync(buffer, cancellationToken); -#endif - #endregion -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs deleted file mode 100644 index e6b193ae18260..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ /dev/null @@ -1,542 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers; -using System.Buffers.Binary; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Azure.Storage.Common; - -namespace Azure.Storage.Shared; - -/// -/// Decodes a structured message stream as the data is read. -/// -/// -/// Wraps the inner stream in a , which avoids using its internal -/// buffer if individual Read() calls are larger than it. This ensures one of the three scenarios -/// -/// -/// Read buffer >= stream buffer: -/// There is enough space in the read buffer for inline metadata to be safely -/// extracted in only one read to the true inner stream. -/// -/// -/// Read buffer < next inline metadata: -/// The stream buffer has been activated, and we can read multiple small times from the inner stream -/// without multi-reading the real stream, even when partway through an existing stream buffer. -/// -/// -/// Else: -/// Same as #1, but also the already-allocated stream buffer has been used to slightly improve -/// resource churn when reading inner stream. -/// -/// -/// -internal class StructuredMessageDecodingStream : Stream -{ - internal class RawDecodedData - { - public long? InnerStreamLength { get; set; } - public int? TotalSegments { get; set; } - public StructuredMessage.Flags? Flags { get; set; } - public List<(ulong SegmentCrc, long SegmentLen)> SegmentCrcs { get; } = new(); - public ulong? TotalCrc { get; set; } - public bool DecodeCompleted { get; set; } - } - - private enum SMRegion - { - StreamHeader, - StreamFooter, - SegmentHeader, - SegmentFooter, - SegmentContent, - } - - private readonly Stream _innerBufferedStream; - - private byte[] _metadataBuffer = ArrayPool.Shared.Rent(Constants.KB); - private int _metadataBufferOffset = 0; - private int _metadataBufferLength = 0; - - private int _streamHeaderLength; - private int _streamFooterLength; - private int _segmentHeaderLength; - private int _segmentFooterLength; - - private long? _expectedInnerStreamLength; - - private bool _disposed; - - private readonly RawDecodedData _decodedData; - private StorageCrc64HashAlgorithm _totalContentCrc; - private StorageCrc64HashAlgorithm _segmentCrc; - - private readonly bool _validateChecksums; - - public override bool CanRead => true; - - public override bool CanWrite => false; - - public override bool CanSeek => false; - - public override bool CanTimeout => _innerBufferedStream.CanTimeout; - - public override int ReadTimeout => _innerBufferedStream.ReadTimeout; - - public override int WriteTimeout => _innerBufferedStream.WriteTimeout; - - public override long Length => throw new NotSupportedException(); - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public static (Stream DecodedStream, RawDecodedData DecodedData) WrapStream( - Stream innerStream, - long? expextedStreamLength = default) - { - RawDecodedData data = new(); - return (new StructuredMessageDecodingStream(innerStream, data, expextedStreamLength), data); - } - - private StructuredMessageDecodingStream( - Stream innerStream, - RawDecodedData decodedData, - long? expectedStreamLength) - { - Argument.AssertNotNull(innerStream, nameof(innerStream)); - Argument.AssertNotNull(decodedData, nameof(decodedData)); - - _expectedInnerStreamLength = expectedStreamLength; - _innerBufferedStream = new BufferedStream(innerStream); - _decodedData = decodedData; - - // Assumes stream will be structured message 1.0. Will validate this when consuming stream. - _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; - _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; - - _validateChecksums = true; - } - - #region Write - public override void Flush() => throw new NotSupportedException(); - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - #endregion - - #region Read - public override int Read(byte[] buf, int offset, int count) - { - int decodedRead; - int read; - do - { - read = _innerBufferedStream.Read(buf, offset, count); - _innerStreamConsumed += read; - decodedRead = Decode(new Span(buf, offset, read)); - } while (decodedRead <= 0 && read > 0); - - if (read <= 0) - { - AssertDecodeFinished(); - } - - return decodedRead; - } - - public override async Task ReadAsync(byte[] buf, int offset, int count, CancellationToken cancellationToken) - { - int decodedRead; - int read; - do - { - read = await _innerBufferedStream.ReadAsync(buf, offset, count, cancellationToken).ConfigureAwait(false); - _innerStreamConsumed += read; - decodedRead = Decode(new Span(buf, offset, read)); - } while (decodedRead <= 0 && read > 0); - - if (read <= 0) - { - AssertDecodeFinished(); - } - - return decodedRead; - } - -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - public override int Read(Span buf) - { - int decodedRead; - int read; - do - { - read = _innerBufferedStream.Read(buf); - _innerStreamConsumed += read; - decodedRead = Decode(buf.Slice(0, read)); - } while (decodedRead <= 0 && read > 0); - - if (read <= 0) - { - AssertDecodeFinished(); - } - - return decodedRead; - } - - public override async ValueTask ReadAsync(Memory buf, CancellationToken cancellationToken = default) - { - int decodedRead; - int read; - do - { - read = await _innerBufferedStream.ReadAsync(buf).ConfigureAwait(false); - _innerStreamConsumed += read; - decodedRead = Decode(buf.Slice(0, read).Span); - } while (decodedRead <= 0 && read > 0); - - if (read <= 0) - { - AssertDecodeFinished(); - } - - return decodedRead; - } -#endif - - private void AssertDecodeFinished() - { - if (_streamFooterLength > 0 && !_decodedData.DecodeCompleted) - { - throw Errors.InvalidStructuredMessage("Premature end of stream."); - } - _decodedData.DecodeCompleted = true; - } - - private long _innerStreamConsumed = 0; - private long _decodedContentConsumed = 0; - private SMRegion _currentRegion = SMRegion.StreamHeader; - private int _currentSegmentNum = 0; - private long _currentSegmentContentLength; - private long _currentSegmentContentRemaining; - private long CurrentRegionLength => _currentRegion switch - { - SMRegion.StreamHeader => _streamHeaderLength, - SMRegion.StreamFooter => _streamFooterLength, - SMRegion.SegmentHeader => _segmentHeaderLength, - SMRegion.SegmentFooter => _segmentFooterLength, - SMRegion.SegmentContent => _currentSegmentContentLength, - _ => 0, - }; - - /// - /// Decodes given bytes in place. Decoding based on internal stream position info. - /// Decoded data size will be less than or equal to encoded data length. - /// - /// - /// Length of the decoded data in . - /// - private int Decode(Span buffer) - { - if (buffer.IsEmpty) - { - return 0; - } - List<(int Offset, int Count)> gaps = new(); - - int bufferConsumed = ProcessMetadataBuffer(buffer); - - if (bufferConsumed > 0) - { - gaps.Add((0, bufferConsumed)); - } - - while (bufferConsumed < buffer.Length) - { - if (_currentRegion == SMRegion.SegmentContent) - { - int read = (int)Math.Min(buffer.Length - bufferConsumed, _currentSegmentContentRemaining); - _totalContentCrc?.Append(buffer.Slice(bufferConsumed, read)); - _segmentCrc?.Append(buffer.Slice(bufferConsumed, read)); - bufferConsumed += read; - _decodedContentConsumed += read; - _currentSegmentContentRemaining -= read; - if (_currentSegmentContentRemaining == 0) - { - _currentRegion = SMRegion.SegmentFooter; - } - } - else if (buffer.Length - bufferConsumed < CurrentRegionLength) - { - SavePartialMetadata(buffer.Slice(bufferConsumed)); - gaps.Add((bufferConsumed, buffer.Length - bufferConsumed)); - bufferConsumed = buffer.Length; - } - else - { - int processed = _currentRegion switch - { - SMRegion.StreamHeader => ProcessStreamHeader(buffer.Slice(bufferConsumed)), - SMRegion.StreamFooter => ProcessStreamFooter(buffer.Slice(bufferConsumed)), - SMRegion.SegmentHeader => ProcessSegmentHeader(buffer.Slice(bufferConsumed)), - SMRegion.SegmentFooter => ProcessSegmentFooter(buffer.Slice(bufferConsumed)), - _ => 0, - }; - // TODO surface error if processed is 0 - gaps.Add((bufferConsumed, processed)); - bufferConsumed += processed; - } - } - - if (gaps.Count == 0) - { - return buffer.Length; - } - - // gaps is already sorted by offset due to how it was assembled - int gap = 0; - for (int i = gaps.First().Offset; i < buffer.Length; i++) - { - if (gaps.Count > 0 && gaps.First().Offset == i) - { - int count = gaps.First().Count; - gap += count; - i += count - 1; - gaps.RemoveAt(0); - } - else - { - buffer[i - gap] = buffer[i]; - } - } - return buffer.Length - gap; - } - - /// - /// Processes metadata in the internal buffer, if any. Appends any necessary data - /// from the append buffer to complete metadata. - /// - /// - /// Bytes consumed from . - /// - private int ProcessMetadataBuffer(ReadOnlySpan append) - { - if (_metadataBufferLength == 0) - { - return 0; - } - if (_currentRegion == SMRegion.SegmentContent) - { - return 0; - } - int appended = 0; - if (_metadataBufferLength < CurrentRegionLength && append.Length > 0) - { - appended = Math.Min((int)CurrentRegionLength - _metadataBufferLength, append.Length); - SavePartialMetadata(append.Slice(0, appended)); - } - if (_metadataBufferLength == CurrentRegionLength) - { - Span metadata = new(_metadataBuffer, _metadataBufferOffset, (int)CurrentRegionLength); - switch (_currentRegion) - { - case SMRegion.StreamHeader: - ProcessStreamHeader(metadata); - break; - case SMRegion.StreamFooter: - ProcessStreamFooter(metadata); - break; - case SMRegion.SegmentHeader: - ProcessSegmentHeader(metadata); - break; - case SMRegion.SegmentFooter: - ProcessSegmentFooter(metadata); - break; - } - _metadataBufferOffset = 0; - _metadataBufferLength = 0; - } - return appended; - } - - private void SavePartialMetadata(ReadOnlySpan span) - { - // safety array resize w/ArrayPool - if (_metadataBufferLength + span.Length > _metadataBuffer.Length) - { - ResizeMetadataBuffer(2 * (_metadataBufferLength + span.Length)); - } - - // realign any existing content if necessary - if (_metadataBufferLength != 0 && _metadataBufferOffset != 0) - { - // don't use Array.Copy() to move elements in the same array - for (int i = 0; i < _metadataBufferLength; i++) - { - _metadataBuffer[i] = _metadataBuffer[i + _metadataBufferOffset]; - } - _metadataBufferOffset = 0; - } - - span.CopyTo(new Span(_metadataBuffer, _metadataBufferOffset + _metadataBufferLength, span.Length)); - _metadataBufferLength += span.Length; - } - - private int ProcessStreamHeader(ReadOnlySpan span) - { - StructuredMessage.V1_0.ReadStreamHeader( - span.Slice(0, _streamHeaderLength), - out long streamLength, - out StructuredMessage.Flags flags, - out int totalSegments); - - _decodedData.InnerStreamLength = streamLength; - _decodedData.Flags = flags; - _decodedData.TotalSegments = totalSegments; - - if (_expectedInnerStreamLength.HasValue && _expectedInnerStreamLength.Value != streamLength) - { - throw Errors.InvalidStructuredMessage("Unexpected message size."); - } - - if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) - { - _segmentFooterLength = StructuredMessage.Crc64Length; - _streamFooterLength = StructuredMessage.Crc64Length; - if (_validateChecksums) - { - _segmentCrc = StorageCrc64HashAlgorithm.Create(); - _totalContentCrc = StorageCrc64HashAlgorithm.Create(); - } - } - _currentRegion = SMRegion.SegmentHeader; - return _streamHeaderLength; - } - - private int ProcessStreamFooter(ReadOnlySpan span) - { - int footerLen = StructuredMessage.V1_0.GetStreamFooterSize(_decodedData.Flags.Value); - StructuredMessage.V1_0.ReadStreamFooter( - span.Slice(0, footerLen), - _decodedData.Flags.Value, - out ulong reportedCrc); - if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) - { - if (_validateChecksums) - { - ValidateCrc64(_totalContentCrc, reportedCrc); - } - _decodedData.TotalCrc = reportedCrc; - } - - if (_innerStreamConsumed != _decodedData.InnerStreamLength) - { - throw Errors.InvalidStructuredMessage("Unexpected message size."); - } - if (_currentSegmentNum != _decodedData.TotalSegments) - { - throw Errors.InvalidStructuredMessage("Missing expected message segments."); - } - - _decodedData.DecodeCompleted = true; - return footerLen; - } - - private int ProcessSegmentHeader(ReadOnlySpan span) - { - StructuredMessage.V1_0.ReadSegmentHeader( - span.Slice(0, _segmentHeaderLength), - out int newSegNum, - out _currentSegmentContentLength); - _currentSegmentContentRemaining = _currentSegmentContentLength; - if (newSegNum != _currentSegmentNum + 1) - { - throw Errors.InvalidStructuredMessage("Unexpected segment number in structured message."); - } - _currentSegmentNum = newSegNum; - _currentRegion = SMRegion.SegmentContent; - return _segmentHeaderLength; - } - - private int ProcessSegmentFooter(ReadOnlySpan span) - { - int footerLen = StructuredMessage.V1_0.GetSegmentFooterSize(_decodedData.Flags.Value); - StructuredMessage.V1_0.ReadSegmentFooter( - span.Slice(0, footerLen), - _decodedData.Flags.Value, - out ulong reportedCrc); - if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) - { - if (_validateChecksums) - { - ValidateCrc64(_segmentCrc, reportedCrc); - _segmentCrc = StorageCrc64HashAlgorithm.Create(); - } - _decodedData.SegmentCrcs.Add((reportedCrc, _currentSegmentContentLength)); - } - _currentRegion = _currentSegmentNum == _decodedData.TotalSegments ? SMRegion.StreamFooter : SMRegion.SegmentHeader; - return footerLen; - } - - private static void ValidateCrc64(StorageCrc64HashAlgorithm calculation, ulong reported) - { - using IDisposable _ = ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf); - Span calculatedBytes = new(buf, 0, StructuredMessage.Crc64Length); - Span reportedBytes = new(buf, calculatedBytes.Length, StructuredMessage.Crc64Length); - calculation.GetCurrentHash(calculatedBytes); - reported.WriteCrc64(reportedBytes); - if (!calculatedBytes.SequenceEqual(reportedBytes)) - { - throw Errors.ChecksumMismatch(calculatedBytes, reportedBytes); - } - } - #endregion - - public override long Seek(long offset, SeekOrigin origin) - => throw new NotSupportedException(); - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - - if (_disposed) - { - return; - } - - if (disposing) - { - _innerBufferedStream.Dispose(); - _disposed = true; - } - } - - private void ResizeMetadataBuffer(int newSize) - { - byte[] newBuf = ArrayPool.Shared.Rent(newSize); - Array.Copy(_metadataBuffer, _metadataBufferOffset, newBuf, 0, _metadataBufferLength); - ArrayPool.Shared.Return(_metadataBuffer); - _metadataBuffer = newBuf; - } - - private void AlignMetadataBuffer() - { - if (_metadataBufferOffset != 0 && _metadataBufferLength != 0) - { - for (int i = 0; i < _metadataBufferLength; i++) - { - _metadataBuffer[i] = _metadataBuffer[_metadataBufferOffset + i]; - } - _metadataBufferOffset = 0; - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs deleted file mode 100644 index cb0ef340155ec..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs +++ /dev/null @@ -1,545 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Azure.Core.Pipeline; -using Azure.Storage.Common; - -namespace Azure.Storage.Shared; - -internal class StructuredMessageEncodingStream : Stream -{ - private readonly Stream _innerStream; - - private readonly int _streamHeaderLength; - private readonly int _streamFooterLength; - private readonly int _segmentHeaderLength; - private readonly int _segmentFooterLength; - private readonly int _segmentContentLength; - - private readonly StructuredMessage.Flags _flags; - private bool _disposed; - - private bool UseCrcSegment => _flags.HasFlag(StructuredMessage.Flags.StorageCrc64); - private readonly StorageCrc64HashAlgorithm _totalCrc; - private StorageCrc64HashAlgorithm _segmentCrc; - private readonly byte[] _segmentCrcs; - private int _latestSegmentCrcd = 0; - - #region Segments - /// - /// Gets the 1-indexed segment number the underlying stream is currently positioned in. - /// 1-indexed to match segment labelling as specified by SM spec. - /// - private int CurrentInnerSegment => (int)Math.Floor(_innerStream.Position / (float)_segmentContentLength) + 1; - - /// - /// Gets the 1-indexed segment number the encoded data stream is currently positioned in. - /// 1-indexed to match segment labelling as specified by SM spec. - /// - private int CurrentEncodingSegment - { - get - { - // edge case: always on final segment when at end of inner stream - if (_innerStream.Position == _innerStream.Length) - { - return TotalSegments; - } - // when writing footer, inner stream is positioned at next segment, - // but this stream is still writing the previous one - if (_currentRegion == SMRegion.SegmentFooter) - { - return CurrentInnerSegment - 1; - } - return CurrentInnerSegment; - } - } - - /// - /// Segment length including header and footer. - /// - private int SegmentTotalLength => _segmentHeaderLength + _segmentContentLength + _segmentFooterLength; - - private int TotalSegments => GetTotalSegments(_innerStream, _segmentContentLength); - private static int GetTotalSegments(Stream innerStream, long segmentContentLength) - { - return (int)Math.Ceiling(innerStream.Length / (float)segmentContentLength); - } - #endregion - - public override bool CanRead => true; - - public override bool CanWrite => false; - - public override bool CanSeek => _innerStream.CanSeek; - - public override bool CanTimeout => _innerStream.CanTimeout; - - public override int ReadTimeout => _innerStream.ReadTimeout; - - public override int WriteTimeout => _innerStream.WriteTimeout; - - public override long Length => - _streamHeaderLength + _streamFooterLength + - (_segmentHeaderLength + _segmentFooterLength) * TotalSegments + - _innerStream.Length; - - #region Position - private enum SMRegion - { - StreamHeader, - StreamFooter, - SegmentHeader, - SegmentFooter, - SegmentContent, - } - - private SMRegion _currentRegion = SMRegion.StreamHeader; - private int _currentRegionPosition = 0; - - private long _maxSeekPosition = 0; - - public override long Position - { - get - { - return _currentRegion switch - { - SMRegion.StreamHeader => _currentRegionPosition, - SMRegion.StreamFooter => _streamHeaderLength + - TotalSegments * (_segmentHeaderLength + _segmentFooterLength) + - _innerStream.Length + - _currentRegionPosition, - SMRegion.SegmentHeader => _innerStream.Position + - _streamHeaderLength + - (CurrentEncodingSegment - 1) * (_segmentHeaderLength + _segmentFooterLength) + - _currentRegionPosition, - SMRegion.SegmentFooter => _innerStream.Position + - _streamHeaderLength + - // Inner stream has moved to next segment but we're still writing the previous segment footer - CurrentEncodingSegment * (_segmentHeaderLength + _segmentFooterLength) - - _segmentFooterLength + _currentRegionPosition, - SMRegion.SegmentContent => _innerStream.Position + - _streamHeaderLength + - CurrentEncodingSegment * (_segmentHeaderLength + _segmentFooterLength) - - _segmentFooterLength, - _ => throw new InvalidDataException($"{nameof(StructuredMessageEncodingStream)} invalid state."), - }; - } - set - { - Argument.AssertInRange(value, 0, _maxSeekPosition, nameof(value)); - if (value < _streamHeaderLength) - { - _currentRegion = SMRegion.StreamHeader; - _currentRegionPosition = (int)value; - _innerStream.Position = 0; - return; - } - if (value >= Length - _streamFooterLength) - { - _currentRegion = SMRegion.StreamFooter; - _currentRegionPosition = (int)(value - (Length - _streamFooterLength)); - _innerStream.Position = _innerStream.Length; - return; - } - int newSegmentNum = 1 + (int)Math.Floor((value - _streamHeaderLength) / (double)(_segmentHeaderLength + _segmentFooterLength + _segmentContentLength)); - int segmentPosition = (int)(value - _streamHeaderLength - - ((newSegmentNum - 1) * (_segmentHeaderLength + _segmentFooterLength + _segmentContentLength))); - - if (segmentPosition < _segmentHeaderLength) - { - _currentRegion = SMRegion.SegmentHeader; - _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength); - _innerStream.Position = (newSegmentNum - 1) * _segmentContentLength; - return; - } - if (segmentPosition < _segmentHeaderLength + _segmentContentLength) - { - _currentRegion = SMRegion.SegmentContent; - _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength) - - _segmentHeaderLength; - _innerStream.Position = (newSegmentNum - 1) * _segmentContentLength + _currentRegionPosition; - return; - } - - _currentRegion = SMRegion.SegmentFooter; - _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength) - - _segmentHeaderLength - _segmentContentLength; - _innerStream.Position = newSegmentNum * _segmentContentLength; - } - } - #endregion - - public StructuredMessageEncodingStream( - Stream innerStream, - int segmentContentLength, - StructuredMessage.Flags flags) - { - Argument.AssertNotNull(innerStream, nameof(innerStream)); - if (innerStream.GetLengthOrDefault() == default) - { - throw new ArgumentException("Stream must have known length.", nameof(innerStream)); - } - if (innerStream.Position != 0) - { - throw new ArgumentException("Stream must be at starting position.", nameof(innerStream)); - } - // stream logic likely breaks down with segment length of 1; enforce >=2 rather than just positive number - // real world scenarios will probably use a minimum of tens of KB - Argument.AssertInRange(segmentContentLength, 2, int.MaxValue, nameof(segmentContentLength)); - - _flags = flags; - _segmentContentLength = segmentContentLength; - - _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; - _streamFooterLength = UseCrcSegment ? StructuredMessage.Crc64Length : 0; - _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; - _segmentFooterLength = UseCrcSegment ? StructuredMessage.Crc64Length : 0; - - if (UseCrcSegment) - { - _totalCrc = StorageCrc64HashAlgorithm.Create(); - _segmentCrc = StorageCrc64HashAlgorithm.Create(); - _segmentCrcs = ArrayPool.Shared.Rent( - GetTotalSegments(innerStream, segmentContentLength) * StructuredMessage.Crc64Length); - innerStream = ChecksumCalculatingStream.GetReadStream(innerStream, span => - { - _totalCrc.Append(span); - _segmentCrc.Append(span); - }); - } - - _innerStream = innerStream; - } - - #region Write - public override void Flush() => throw new NotSupportedException(); - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - #endregion - - #region Read - public override int Read(byte[] buffer, int offset, int count) - => ReadInternal(buffer, offset, count, async: false, cancellationToken: default).EnsureCompleted(); - - public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - => await ReadInternal(buffer, offset, count, async: true, cancellationToken).ConfigureAwait(false); - - private async ValueTask ReadInternal(byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) - { - int totalRead = 0; - bool readInner = false; - while (totalRead < count && Position < Length) - { - int subreadOffset = offset + totalRead; - int subreadCount = count - totalRead; - switch (_currentRegion) - { - case SMRegion.StreamHeader: - totalRead += ReadFromStreamHeader(new Span(buffer, subreadOffset, subreadCount)); - break; - case SMRegion.StreamFooter: - totalRead += ReadFromStreamFooter(new Span(buffer, subreadOffset, subreadCount)); - break; - case SMRegion.SegmentHeader: - totalRead += ReadFromSegmentHeader(new Span(buffer, subreadOffset, subreadCount)); - break; - case SMRegion.SegmentFooter: - totalRead += ReadFromSegmentFooter(new Span(buffer, subreadOffset, subreadCount)); - break; - case SMRegion.SegmentContent: - // don't double read from stream. Allow caller to multi-read when desired. - if (readInner) - { - UpdateLatestPosition(); - return totalRead; - } - totalRead += await ReadFromInnerStreamInternal( - buffer, subreadOffset, subreadCount, async, cancellationToken).ConfigureAwait(false); - readInner = true; - break; - default: - break; - } - } - UpdateLatestPosition(); - return totalRead; - } - -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - public override int Read(Span buffer) - { - int totalRead = 0; - bool readInner = false; - while (totalRead < buffer.Length && Position < Length) - { - switch (_currentRegion) - { - case SMRegion.StreamHeader: - totalRead += ReadFromStreamHeader(buffer.Slice(totalRead)); - break; - case SMRegion.StreamFooter: - totalRead += ReadFromStreamFooter(buffer.Slice(totalRead)); - break; - case SMRegion.SegmentHeader: - totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead)); - break; - case SMRegion.SegmentFooter: - totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead)); - break; - case SMRegion.SegmentContent: - // don't double read from stream. Allow caller to multi-read when desired. - if (readInner) - { - UpdateLatestPosition(); - return totalRead; - } - totalRead += ReadFromInnerStream(buffer.Slice(totalRead)); - readInner = true; - break; - default: - break; - } - } - UpdateLatestPosition(); - return totalRead; - } - - public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) - { - int totalRead = 0; - bool readInner = false; - while (totalRead < buffer.Length && Position < Length) - { - switch (_currentRegion) - { - case SMRegion.StreamHeader: - totalRead += ReadFromStreamHeader(buffer.Slice(totalRead).Span); - break; - case SMRegion.StreamFooter: - totalRead += ReadFromStreamFooter(buffer.Slice(totalRead).Span); - break; - case SMRegion.SegmentHeader: - totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead).Span); - break; - case SMRegion.SegmentFooter: - totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead).Span); - break; - case SMRegion.SegmentContent: - // don't double read from stream. Allow caller to multi-read when desired. - if (readInner) - { - UpdateLatestPosition(); - return totalRead; - } - totalRead += await ReadFromInnerStreamAsync(buffer.Slice(totalRead), cancellationToken).ConfigureAwait(false); - readInner = true; - break; - default: - break; - } - } - UpdateLatestPosition(); - return totalRead; - } -#endif - - #region Read Headers/Footers - private int ReadFromStreamHeader(Span buffer) - { - int read = Math.Min(buffer.Length, _streamHeaderLength - _currentRegionPosition); - using IDisposable _ = StructuredMessage.V1_0.GetStreamHeaderBytes( - ArrayPool.Shared, out Memory headerBytes, Length, _flags, TotalSegments); - headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); - _currentRegionPosition += read; - - if (_currentRegionPosition == _streamHeaderLength) - { - _currentRegion = SMRegion.SegmentHeader; - _currentRegionPosition = 0; - } - - return read; - } - - private int ReadFromStreamFooter(Span buffer) - { - int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); - if (read <= 0) - { - return 0; - } - - using IDisposable _ = StructuredMessage.V1_0.GetStreamFooterBytes( - ArrayPool.Shared, - out Memory footerBytes, - crc64: UseCrcSegment - ? _totalCrc.GetCurrentHash() // TODO array pooling - : default); - footerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); - _currentRegionPosition += read; - - return read; - } - - private int ReadFromSegmentHeader(Span buffer) - { - int read = Math.Min(buffer.Length, _segmentHeaderLength - _currentRegionPosition); - using IDisposable _ = StructuredMessage.V1_0.GetSegmentHeaderBytes( - ArrayPool.Shared, - out Memory headerBytes, - CurrentInnerSegment, - Math.Min(_segmentContentLength, _innerStream.Length - _innerStream.Position)); - headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); - _currentRegionPosition += read; - - if (_currentRegionPosition == _segmentHeaderLength) - { - _currentRegion = SMRegion.SegmentContent; - _currentRegionPosition = 0; - } - - return read; - } - - private int ReadFromSegmentFooter(Span buffer) - { - int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); - if (read < 0) - { - return 0; - } - - using IDisposable _ = StructuredMessage.V1_0.GetSegmentFooterBytes( - ArrayPool.Shared, - out Memory headerBytes, - crc64: UseCrcSegment - ? new Span( - _segmentCrcs, - (CurrentEncodingSegment-1) * _totalCrc.HashLengthInBytes, - _totalCrc.HashLengthInBytes) - : default); - headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); - _currentRegionPosition += read; - - if (_currentRegionPosition == _segmentFooterLength) - { - _currentRegion = _innerStream.Position == _innerStream.Length - ? SMRegion.StreamFooter : SMRegion.SegmentHeader; - _currentRegionPosition = 0; - } - - return read; - } - #endregion - - #region ReadUnderlyingStream - private int MaxInnerStreamRead => _segmentContentLength - _currentRegionPosition; - - private void CleanupContentSegment() - { - if (_currentRegionPosition == _segmentContentLength || _innerStream.Position >= _innerStream.Length) - { - _currentRegion = SMRegion.SegmentFooter; - _currentRegionPosition = 0; - if (UseCrcSegment && CurrentEncodingSegment - 1 == _latestSegmentCrcd) - { - _segmentCrc.GetCurrentHash(new Span( - _segmentCrcs, - _latestSegmentCrcd * _segmentCrc.HashLengthInBytes, - _segmentCrc.HashLengthInBytes)); - _latestSegmentCrcd++; - _segmentCrc = StorageCrc64HashAlgorithm.Create(); - } - } - } - - private async ValueTask ReadFromInnerStreamInternal( - byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) - { - int read = async - ? await _innerStream.ReadAsync(buffer, offset, Math.Min(count, MaxInnerStreamRead)).ConfigureAwait(false) - : _innerStream.Read(buffer, offset, Math.Min(count, MaxInnerStreamRead)); - _currentRegionPosition += read; - CleanupContentSegment(); - return read; - } - -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - private int ReadFromInnerStream(Span buffer) - { - if (MaxInnerStreamRead < buffer.Length) - { - buffer = buffer.Slice(0, MaxInnerStreamRead); - } - int read = _innerStream.Read(buffer); - _currentRegionPosition += read; - CleanupContentSegment(); - return read; - } - - private async ValueTask ReadFromInnerStreamAsync(Memory buffer, CancellationToken cancellationToken) - { - if (MaxInnerStreamRead < buffer.Length) - { - buffer = buffer.Slice(0, MaxInnerStreamRead); - } - int read = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); - _currentRegionPosition += read; - CleanupContentSegment(); - return read; - } -#endif - #endregion - - // don't allow stream to seek too far forward. track how far the stream has been naturally read. - private void UpdateLatestPosition() - { - if (_maxSeekPosition < Position) - { - _maxSeekPosition = Position; - } - } - #endregion - - public override long Seek(long offset, SeekOrigin origin) - { - switch (origin) - { - case SeekOrigin.Begin: - Position = offset; - break; - case SeekOrigin.Current: - Position += offset; - break; - case SeekOrigin.End: - Position = Length + offset; - break; - } - return Position; - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - - if (_disposed) - { - return; - } - - if (disposing) - { - _innerStream.Dispose(); - _disposed = true; - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs deleted file mode 100644 index 3569ef4339735..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs +++ /dev/null @@ -1,451 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Azure.Core.Pipeline; -using Azure.Storage.Common; - -namespace Azure.Storage.Shared; - -internal class StructuredMessagePrecalculatedCrcWrapperStream : Stream -{ - private readonly Stream _innerStream; - - private readonly int _streamHeaderLength; - private readonly int _streamFooterLength; - private readonly int _segmentHeaderLength; - private readonly int _segmentFooterLength; - - private bool _disposed; - - private readonly byte[] _crc; - - public override bool CanRead => true; - - public override bool CanWrite => false; - - public override bool CanSeek => _innerStream.CanSeek; - - public override bool CanTimeout => _innerStream.CanTimeout; - - public override int ReadTimeout => _innerStream.ReadTimeout; - - public override int WriteTimeout => _innerStream.WriteTimeout; - - public override long Length => - _streamHeaderLength + _streamFooterLength + - _segmentHeaderLength + _segmentFooterLength + - _innerStream.Length; - - #region Position - private enum SMRegion - { - StreamHeader, - StreamFooter, - SegmentHeader, - SegmentFooter, - SegmentContent, - } - - private SMRegion _currentRegion = SMRegion.StreamHeader; - private int _currentRegionPosition = 0; - - private long _maxSeekPosition = 0; - - public override long Position - { - get - { - return _currentRegion switch - { - SMRegion.StreamHeader => _currentRegionPosition, - SMRegion.SegmentHeader => _innerStream.Position + - _streamHeaderLength + - _currentRegionPosition, - SMRegion.SegmentContent => _streamHeaderLength + - _segmentHeaderLength + - _innerStream.Position, - SMRegion.SegmentFooter => _streamHeaderLength + - _segmentHeaderLength + - _innerStream.Length + - _currentRegionPosition, - SMRegion.StreamFooter => _streamHeaderLength + - _segmentHeaderLength + - _innerStream.Length + - _segmentFooterLength + - _currentRegionPosition, - _ => throw new InvalidDataException($"{nameof(StructuredMessageEncodingStream)} invalid state."), - }; - } - set - { - Argument.AssertInRange(value, 0, _maxSeekPosition, nameof(value)); - if (value < _streamHeaderLength) - { - _currentRegion = SMRegion.StreamHeader; - _currentRegionPosition = (int)value; - _innerStream.Position = 0; - return; - } - if (value < _streamHeaderLength + _segmentHeaderLength) - { - _currentRegion = SMRegion.SegmentHeader; - _currentRegionPosition = (int)(value - _streamHeaderLength); - _innerStream.Position = 0; - return; - } - if (value < _streamHeaderLength + _segmentHeaderLength + _innerStream.Length) - { - _currentRegion = SMRegion.SegmentContent; - _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength); - _innerStream.Position = value - _streamHeaderLength - _segmentHeaderLength; - return; - } - if (value < _streamHeaderLength + _segmentHeaderLength + _innerStream.Length + _segmentFooterLength) - { - _currentRegion = SMRegion.SegmentFooter; - _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength - _innerStream.Length); - _innerStream.Position = _innerStream.Length; - return; - } - - _currentRegion = SMRegion.StreamFooter; - _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength - _innerStream.Length - _segmentFooterLength); - _innerStream.Position = _innerStream.Length; - } - } - #endregion - - public StructuredMessagePrecalculatedCrcWrapperStream( - Stream innerStream, - ReadOnlySpan precalculatedCrc) - { - Argument.AssertNotNull(innerStream, nameof(innerStream)); - if (innerStream.GetLengthOrDefault() == default) - { - throw new ArgumentException("Stream must have known length.", nameof(innerStream)); - } - if (innerStream.Position != 0) - { - throw new ArgumentException("Stream must be at starting position.", nameof(innerStream)); - } - - _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; - _streamFooterLength = StructuredMessage.Crc64Length; - _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; - _segmentFooterLength = StructuredMessage.Crc64Length; - - _crc = ArrayPool.Shared.Rent(StructuredMessage.Crc64Length); - precalculatedCrc.CopyTo(_crc); - - _innerStream = innerStream; - } - - #region Write - public override void Flush() => throw new NotSupportedException(); - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - #endregion - - #region Read - public override int Read(byte[] buffer, int offset, int count) - => ReadInternal(buffer, offset, count, async: false, cancellationToken: default).EnsureCompleted(); - - public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - => await ReadInternal(buffer, offset, count, async: true, cancellationToken).ConfigureAwait(false); - - private async ValueTask ReadInternal(byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) - { - int totalRead = 0; - bool readInner = false; - while (totalRead < count && Position < Length) - { - int subreadOffset = offset + totalRead; - int subreadCount = count - totalRead; - switch (_currentRegion) - { - case SMRegion.StreamHeader: - totalRead += ReadFromStreamHeader(new Span(buffer, subreadOffset, subreadCount)); - break; - case SMRegion.StreamFooter: - totalRead += ReadFromStreamFooter(new Span(buffer, subreadOffset, subreadCount)); - break; - case SMRegion.SegmentHeader: - totalRead += ReadFromSegmentHeader(new Span(buffer, subreadOffset, subreadCount)); - break; - case SMRegion.SegmentFooter: - totalRead += ReadFromSegmentFooter(new Span(buffer, subreadOffset, subreadCount)); - break; - case SMRegion.SegmentContent: - // don't double read from stream. Allow caller to multi-read when desired. - if (readInner) - { - UpdateLatestPosition(); - return totalRead; - } - totalRead += await ReadFromInnerStreamInternal( - buffer, subreadOffset, subreadCount, async, cancellationToken).ConfigureAwait(false); - readInner = true; - break; - default: - break; - } - } - UpdateLatestPosition(); - return totalRead; - } - -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - public override int Read(Span buffer) - { - int totalRead = 0; - bool readInner = false; - while (totalRead < buffer.Length && Position < Length) - { - switch (_currentRegion) - { - case SMRegion.StreamHeader: - totalRead += ReadFromStreamHeader(buffer.Slice(totalRead)); - break; - case SMRegion.StreamFooter: - totalRead += ReadFromStreamFooter(buffer.Slice(totalRead)); - break; - case SMRegion.SegmentHeader: - totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead)); - break; - case SMRegion.SegmentFooter: - totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead)); - break; - case SMRegion.SegmentContent: - // don't double read from stream. Allow caller to multi-read when desired. - if (readInner) - { - UpdateLatestPosition(); - return totalRead; - } - totalRead += ReadFromInnerStream(buffer.Slice(totalRead)); - readInner = true; - break; - default: - break; - } - } - UpdateLatestPosition(); - return totalRead; - } - - public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) - { - int totalRead = 0; - bool readInner = false; - while (totalRead < buffer.Length && Position < Length) - { - switch (_currentRegion) - { - case SMRegion.StreamHeader: - totalRead += ReadFromStreamHeader(buffer.Slice(totalRead).Span); - break; - case SMRegion.StreamFooter: - totalRead += ReadFromStreamFooter(buffer.Slice(totalRead).Span); - break; - case SMRegion.SegmentHeader: - totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead).Span); - break; - case SMRegion.SegmentFooter: - totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead).Span); - break; - case SMRegion.SegmentContent: - // don't double read from stream. Allow caller to multi-read when desired. - if (readInner) - { - UpdateLatestPosition(); - return totalRead; - } - totalRead += await ReadFromInnerStreamAsync(buffer.Slice(totalRead), cancellationToken).ConfigureAwait(false); - readInner = true; - break; - default: - break; - } - } - UpdateLatestPosition(); - return totalRead; - } -#endif - - #region Read Headers/Footers - private int ReadFromStreamHeader(Span buffer) - { - int read = Math.Min(buffer.Length, _streamHeaderLength - _currentRegionPosition); - using IDisposable _ = StructuredMessage.V1_0.GetStreamHeaderBytes( - ArrayPool.Shared, - out Memory headerBytes, - Length, - StructuredMessage.Flags.StorageCrc64, - totalSegments: 1); - headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); - _currentRegionPosition += read; - - if (_currentRegionPosition == _streamHeaderLength) - { - _currentRegion = SMRegion.SegmentHeader; - _currentRegionPosition = 0; - } - - return read; - } - - private int ReadFromStreamFooter(Span buffer) - { - int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); - if (read <= 0) - { - return 0; - } - - using IDisposable _ = StructuredMessage.V1_0.GetStreamFooterBytes( - ArrayPool.Shared, - out Memory footerBytes, - new ReadOnlySpan(_crc, 0, StructuredMessage.Crc64Length)); - footerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); - _currentRegionPosition += read; - - return read; - } - - private int ReadFromSegmentHeader(Span buffer) - { - int read = Math.Min(buffer.Length, _segmentHeaderLength - _currentRegionPosition); - using IDisposable _ = StructuredMessage.V1_0.GetSegmentHeaderBytes( - ArrayPool.Shared, - out Memory headerBytes, - segmentNum: 1, - _innerStream.Length); - headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); - _currentRegionPosition += read; - - if (_currentRegionPosition == _segmentHeaderLength) - { - _currentRegion = SMRegion.SegmentContent; - _currentRegionPosition = 0; - } - - return read; - } - - private int ReadFromSegmentFooter(Span buffer) - { - int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); - if (read < 0) - { - return 0; - } - - using IDisposable _ = StructuredMessage.V1_0.GetSegmentFooterBytes( - ArrayPool.Shared, - out Memory headerBytes, - new ReadOnlySpan(_crc, 0, StructuredMessage.Crc64Length)); - headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); - _currentRegionPosition += read; - - if (_currentRegionPosition == _segmentFooterLength) - { - _currentRegion = _innerStream.Position == _innerStream.Length - ? SMRegion.StreamFooter : SMRegion.SegmentHeader; - _currentRegionPosition = 0; - } - - return read; - } - #endregion - - #region ReadUnderlyingStream - private void CleanupContentSegment() - { - if (_innerStream.Position >= _innerStream.Length) - { - _currentRegion = SMRegion.SegmentFooter; - _currentRegionPosition = 0; - } - } - - private async ValueTask ReadFromInnerStreamInternal( - byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) - { - int read = async - ? await _innerStream.ReadAsync(buffer, offset, count).ConfigureAwait(false) - : _innerStream.Read(buffer, offset, count); - _currentRegionPosition += read; - CleanupContentSegment(); - return read; - } - -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - private int ReadFromInnerStream(Span buffer) - { - int read = _innerStream.Read(buffer); - _currentRegionPosition += read; - CleanupContentSegment(); - return read; - } - - private async ValueTask ReadFromInnerStreamAsync(Memory buffer, CancellationToken cancellationToken) - { - int read = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); - _currentRegionPosition += read; - CleanupContentSegment(); - return read; - } -#endif - #endregion - - // don't allow stream to seek too far forward. track how far the stream has been naturally read. - private void UpdateLatestPosition() - { - if (_maxSeekPosition < Position) - { - _maxSeekPosition = Position; - } - } - #endregion - - public override long Seek(long offset, SeekOrigin origin) - { - switch (origin) - { - case SeekOrigin.Begin: - Position = offset; - break; - case SeekOrigin.Current: - Position += offset; - break; - case SeekOrigin.End: - Position = Length + offset; - break; - } - return Position; - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - - if (_disposed) - { - return; - } - - if (disposing) - { - ArrayPool.Shared.Return(_crc); - _innerStream.Dispose(); - _disposed = true; - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs index 763d385240383..af21588b4ae09 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs @@ -9,7 +9,14 @@ public static StorageChecksumAlgorithm ResolveAuto(this StorageChecksumAlgorithm { if (checksumAlgorithm == StorageChecksumAlgorithm.Auto) { +#if BlobSDK || DataLakeSDK || CommonSDK return StorageChecksumAlgorithm.StorageCrc64; +#elif FileSDK // file shares don't support crc64 + return StorageChecksumAlgorithm.MD5; +#else + throw new System.NotSupportedException( + $"{typeof(TransferValidationOptionsExtensions).FullName}.{nameof(ResolveAuto)} is not supported."); +#endif } return checksumAlgorithm; } diff --git a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj index 2863b85f6feb2..5db86ebee984b 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj +++ b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj @@ -13,12 +13,9 @@ - - - @@ -31,7 +28,6 @@ - @@ -50,11 +46,6 @@ - - - - - diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs index f4e4b92ed73c4..7411eb1499312 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs @@ -15,7 +15,6 @@ internal class FaultyStream : Stream private readonly Exception _exceptionToRaise; private int _remainingExceptions; private Action _onFault; - private long _position = 0; public FaultyStream( Stream innerStream, @@ -41,7 +40,7 @@ public FaultyStream( public override long Position { - get => CanSeek ? _innerStream.Position : _position; + get => _innerStream.Position; set => _innerStream.Position = value; } @@ -54,9 +53,7 @@ public override int Read(byte[] buffer, int offset, int count) { if (_remainingExceptions == 0 || Position + count <= _raiseExceptionAt || _raiseExceptionAt >= _innerStream.Length) { - int read = _innerStream.Read(buffer, offset, count); - _position += read; - return read; + return _innerStream.Read(buffer, offset, count); } else { @@ -64,13 +61,11 @@ public override int Read(byte[] buffer, int offset, int count) } } - public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { if (_remainingExceptions == 0 || Position + count <= _raiseExceptionAt || _raiseExceptionAt >= _innerStream.Length) { - int read = await _innerStream.ReadAsync(buffer, offset, count, cancellationToken); - _position += read; - return read; + return _innerStream.ReadAsync(buffer, offset, count, cancellationToken); } else { diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs deleted file mode 100644 index 828c41179bba3..0000000000000 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.IO; -using Azure.Core; -using Azure.Core.Pipeline; -using Azure.Storage.Shared; - -namespace Azure.Storage.Test.Shared -{ - internal class ObserveStructuredMessagePolicy : HttpPipelineSynchronousPolicy - { - private readonly HashSet _requestScopes = new(); - - private readonly HashSet _responseScopes = new(); - - public ObserveStructuredMessagePolicy() - { - } - - public override void OnSendingRequest(HttpMessage message) - { - if (_requestScopes.Count > 0) - { - byte[] encodedContent; - byte[] underlyingContent; - StructuredMessageDecodingStream.RawDecodedData decodedData; - using (MemoryStream ms = new()) - { - message.Request.Content.WriteTo(ms, default); - encodedContent = ms.ToArray(); - using (MemoryStream ms2 = new()) - { - (Stream s, decodedData) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedContent)); - s.CopyTo(ms2); - underlyingContent = ms2.ToArray(); - } - } - } - } - - public override void OnReceivedResponse(HttpMessage message) - { - } - - public IDisposable CheckRequestScope() => CheckMessageScope.CheckRequestScope(this); - - public IDisposable CheckResponseScope() => CheckMessageScope.CheckResponseScope(this); - - private class CheckMessageScope : IDisposable - { - private bool _isRequestScope; - private ObserveStructuredMessagePolicy _policy; - - public static CheckMessageScope CheckRequestScope(ObserveStructuredMessagePolicy policy) - { - CheckMessageScope result = new() - { - _isRequestScope = true, - _policy = policy - }; - result._policy._requestScopes.Add(result); - return result; - } - - public static CheckMessageScope CheckResponseScope(ObserveStructuredMessagePolicy policy) - { - CheckMessageScope result = new() - { - _isRequestScope = false, - _policy = policy - }; - result._policy._responseScopes.Add(result); - return result; - } - - public void Dispose() - { - (_isRequestScope ? _policy._requestScopes : _policy._responseScopes).Remove(this); - } - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs deleted file mode 100644 index ad395e862f827..0000000000000 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System.Linq; -using System.Text; -using Azure.Core; -using NUnit.Framework; - -namespace Azure.Storage; - -public static partial class RequestExtensions -{ - public static string AssertHeaderPresent(this Request request, string headerName) - { - if (request.Headers.TryGetValue(headerName, out string value)) - { - return headerName == Constants.StructuredMessage.StructuredMessageHeader ? null : value; - } - StringBuilder sb = new StringBuilder() - .AppendLine($"`{headerName}` expected on request but was not found.") - .AppendLine($"{request.Method} {request.Uri}") - .AppendLine(string.Join("\n", request.Headers.Select(h => $"{h.Name}: {h.Value}s"))) - ; - Assert.Fail(sb.ToString()); - return null; - } -} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs index 7e6c78117f53b..f4198e9dfd532 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs @@ -14,7 +14,7 @@ internal class TamperStreamContentsPolicy : HttpPipelineSynchronousPolicy /// /// Default tampering that changes the first byte of the stream. /// - private static Func GetTamperByteStreamTransform(long position) => stream => + private static readonly Func _defaultStreamTransform = stream => { if (stream is not MemoryStream) { @@ -23,10 +23,10 @@ private static Func GetTamperByteStreamTransform(long position) stream = buffer; } - stream.Position = position; + stream.Position = 0; var firstByte = stream.ReadByte(); - stream.Position = position; + stream.Position = 0; stream.WriteByte((byte)((firstByte + 1) % byte.MaxValue)); stream.Position = 0; @@ -37,12 +37,9 @@ private static Func GetTamperByteStreamTransform(long position) public TamperStreamContentsPolicy(Func streamTransform = default) { - _streamTransform = streamTransform ?? GetTamperByteStreamTransform(0); + _streamTransform = streamTransform ?? _defaultStreamTransform; } - public static TamperStreamContentsPolicy TamperByteAt(long position) - => new(GetTamperByteStreamTransform(position)); - public bool TransformRequestBody { get; set; } public bool TransformResponseBody { get; set; } diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index 248acf8811960..c18492d2fb4dd 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -5,13 +5,10 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Security.Cryptography; using System.Threading.Tasks; using Azure.Core; -using Azure.Core.Diagnostics; -using Azure.Core.Pipeline; using Azure.Core.TestFramework; -using Azure.Storage.Shared; +using FastSerialization; using NUnit.Framework; namespace Azure.Storage.Test.Shared @@ -193,15 +190,21 @@ protected string GetNewResourceName() /// The actual checksum value expected to be on the request, if known. Defaults to no specific value expected or checked. /// /// An assertion to put into a pipeline policy. - internal static Action GetRequestChecksumHeaderAssertion(StorageChecksumAlgorithm algorithm, Func isChecksumExpected = default, byte[] expectedChecksum = default) + internal static Action GetRequestChecksumAssertion(StorageChecksumAlgorithm algorithm, Func isChecksumExpected = default, byte[] expectedChecksum = default) { // action to assert a request header is as expected - void AssertChecksum(Request req, string headerName) + void AssertChecksum(RequestHeaders headers, string headerName) { - string checksum = req.AssertHeaderPresent(headerName); - if (expectedChecksum != default) + if (headers.TryGetValue(headerName, out string checksum)) { - Assert.AreEqual(Convert.ToBase64String(expectedChecksum), checksum); + if (expectedChecksum != default) + { + Assert.AreEqual(Convert.ToBase64String(expectedChecksum), checksum); + } + } + else + { + Assert.Fail($"{headerName} expected on request but was not found."); } }; @@ -216,39 +219,14 @@ void AssertChecksum(Request req, string headerName) switch (algorithm.ResolveAuto()) { case StorageChecksumAlgorithm.MD5: - AssertChecksum(request, "Content-MD5"); + AssertChecksum(request.Headers, "Content-MD5"); break; case StorageChecksumAlgorithm.StorageCrc64: - AssertChecksum(request, Constants.StructuredMessage.StructuredMessageHeader); + AssertChecksum(request.Headers, "x-ms-content-crc64"); break; default: - throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); - } - }; - } - - internal static Action GetRequestStructuredMessageAssertion( - StructuredMessage.Flags flags, - Func isStructuredMessageExpected = default, - long? structuredContentSegmentLength = default) - { - return request => - { - // filter some requests out with predicate - if (isStructuredMessageExpected != default && !isStructuredMessageExpected(request)) - { - return; + throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumAssertion)}."); } - - Assert.That(request.Headers.TryGetValue("x-ms-structured-body", out string structuredBody)); - Assert.That(structuredBody, Does.Contain("XSM/1.0")); - if (flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) - { - Assert.That(structuredBody, Does.Contain("crc64")); - } - - Assert.That(request.Headers.TryGetValue("Content-Length", out string contentLength)); - Assert.That(request.Headers.TryGetValue("x-ms-structured-content-length", out string structuredContentLength)); }; } @@ -300,66 +278,32 @@ void AssertChecksum(ResponseHeaders headers, string headerName) AssertChecksum(response.Headers, "Content-MD5"); break; case StorageChecksumAlgorithm.StorageCrc64: - AssertChecksum(response.Headers, Constants.StructuredMessage.StructuredMessageHeader); + AssertChecksum(response.Headers, "x-ms-content-crc64"); break; default: - throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); + throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumAssertion)}."); } }; } - internal static Action GetResponseStructuredMessageAssertion( - StructuredMessage.Flags flags, - Func isStructuredMessageExpected = default) - { - return response => - { - // filter some requests out with predicate - if (isStructuredMessageExpected != default && !isStructuredMessageExpected(response)) - { - return; - } - - Assert.That(response.Headers.TryGetValue("x-ms-structured-body", out string structuredBody)); - Assert.That(structuredBody, Does.Contain("XSM/1.0")); - if (flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) - { - Assert.That(structuredBody, Does.Contain("crc64")); - } - - Assert.That(response.Headers.TryGetValue("Content-Length", out string contentLength)); - Assert.That(response.Headers.TryGetValue("x-ms-structured-content-length", out string structuredContentLength)); - }; - } - /// /// Asserts the service returned an error that expected checksum did not match checksum on upload. /// /// Async action to upload data to service. /// Checksum algorithm used. - internal static void AssertWriteChecksumMismatch( - AsyncTestDelegate writeAction, - StorageChecksumAlgorithm algorithm, - bool expectStructuredMessage = false) + internal static void AssertWriteChecksumMismatch(AsyncTestDelegate writeAction, StorageChecksumAlgorithm algorithm) { var exception = ThrowsOrInconclusiveAsync(writeAction); - if (expectStructuredMessage) - { - Assert.That(exception.ErrorCode, Is.EqualTo("Crc64Mismatch")); - } - else + switch (algorithm.ResolveAuto()) { - switch (algorithm.ResolveAuto()) - { - case StorageChecksumAlgorithm.MD5: - Assert.That(exception.ErrorCode, Is.EqualTo("Md5Mismatch")); - break; - case StorageChecksumAlgorithm.StorageCrc64: - Assert.That(exception.ErrorCode, Is.EqualTo("Crc64Mismatch")); - break; - default: - throw new ArgumentException("Test arguments contain bad algorithm specifier."); - } + case StorageChecksumAlgorithm.MD5: + Assert.AreEqual("Md5Mismatch", exception.ErrorCode); + break; + case StorageChecksumAlgorithm.StorageCrc64: + Assert.AreEqual("Crc64Mismatch", exception.ErrorCode); + break; + default: + throw new ArgumentException("Test arguments contain bad algorithm specifier."); } } #endregion @@ -404,7 +348,6 @@ public virtual async Task UploadPartitionSuccessfulHashComputation(StorageChecks await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); // Arrange - bool expectStructuredMessage = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64; const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var validationOptions = new UploadTransferValidationOptions @@ -413,10 +356,7 @@ public virtual async Task UploadPartitionSuccessfulHashComputation(StorageChecks }; // make pipeline assertion for checking checksum was present on upload - var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 - ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) - : GetRequestChecksumHeaderAssertion(algorithm); - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -466,11 +406,7 @@ public virtual async Task UploadPartitionUsePrecalculatedHash(StorageChecksumAlg }; // make pipeline assertion for checking precalculated checksum was present on upload - // precalculated partition upload will never use structured message. always check header - var assertion = GetRequestChecksumHeaderAssertion( - algorithm, - expectedChecksum: algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? default : precalculatedChecksum); - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm, expectedChecksum: precalculatedChecksum)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -487,12 +423,12 @@ public virtual async Task UploadPartitionUsePrecalculatedHash(StorageChecksumAlg AsyncTestDelegate operation = async () => await UploadPartitionAsync(client, stream, validationOptions); // Assert - AssertWriteChecksumMismatch(operation, algorithm, algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64); + AssertWriteChecksumMismatch(operation, algorithm); } } [TestCaseSource(nameof(GetValidationAlgorithms))] - public virtual async Task UploadPartitionTamperedStreamThrows(StorageChecksumAlgorithm algorithm) + public virtual async Task UploadPartitionMismatchedHashThrows(StorageChecksumAlgorithm algorithm) { await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); @@ -505,7 +441,7 @@ public virtual async Task UploadPartitionTamperedStreamThrows(StorageChecksumAlg }; // Tamper with stream contents in the pipeline to simulate silent failure in the transit layer - var streamTamperPolicy = TamperStreamContentsPolicy.TamperByteAt(100); + var streamTamperPolicy = new TamperStreamContentsPolicy(); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(streamTamperPolicy, HttpPipelinePosition.PerCall); @@ -520,10 +456,9 @@ public virtual async Task UploadPartitionTamperedStreamThrows(StorageChecksumAlg // Act streamTamperPolicy.TransformRequestBody = true; AsyncTestDelegate operation = async () => await UploadPartitionAsync(client, stream, validationOptions); - using var listener = AzureEventSourceListener.CreateConsoleLogger(); + // Assert - AssertWriteChecksumMismatch(operation, algorithm, - expectStructuredMessage: algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64); + AssertWriteChecksumMismatch(operation, algorithm); } } @@ -538,10 +473,7 @@ public virtual async Task UploadPartitionUsesDefaultClientValidationOptions( var data = GetRandomBuffer(dataLength); // make pipeline assertion for checking checksum was present on upload - var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 - ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) - : GetRequestChecksumHeaderAssertion(clientAlgorithm); - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(clientAlgorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -580,10 +512,7 @@ public virtual async Task UploadPartitionOverwritesDefaultClientValidationOption }; // make pipeline assertion for checking checksum was present on upload - var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 - ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) - : GetRequestChecksumHeaderAssertion(overrideAlgorithm); - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(overrideAlgorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -626,14 +555,10 @@ public virtual async Task UploadPartitionDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) + if (request.Headers.Contains("x-ms-content-crc64")) { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains("x-ms-structured-body")) - { - Assert.Fail($"Structured body used when none expected."); - } }); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -676,11 +601,9 @@ public virtual async Task OpenWriteSuccessfulHashComputation( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(algorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); - //ObserveStructuredMessagePolicy observe = new(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); - //clientOptions.AddPolicy(observe, HttpPipelinePosition.BeforeTransport); var client = await GetResourceClientAsync( disposingContainer.Container, @@ -693,7 +616,6 @@ public virtual async Task OpenWriteSuccessfulHashComputation( using var writeStream = await OpenWriteAsync(client, validationOptions, streamBufferSize); // Assert - //using var obsv = observe.CheckRequestScope(); using (checksumPipelineAssertion.CheckRequestScope()) { foreach (var _ in Enumerable.Range(0, streamWrites)) @@ -722,7 +644,7 @@ public virtual async Task OpenWriteMismatchedHashThrows(StorageChecksumAlgorithm // Tamper with stream contents in the pipeline to simulate silent failure in the transit layer var clientOptions = ClientBuilder.GetOptions(); - var tamperPolicy = TamperStreamContentsPolicy.TamperByteAt(100); + var tamperPolicy = new TamperStreamContentsPolicy(); clientOptions.AddPolicy(tamperPolicy, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync( @@ -760,7 +682,7 @@ public virtual async Task OpenWriteUsesDefaultClientValidationOptions( var data = GetRandomBuffer(dataLength); // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(clientAlgorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(clientAlgorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -804,7 +726,7 @@ public virtual async Task OpenWriteOverwritesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(overrideAlgorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(overrideAlgorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -852,7 +774,7 @@ public virtual async Task OpenWriteDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) + if (request.Headers.Contains("x-ms-content-crc64")) { Assert.Fail($"Hash found when none expected."); } @@ -964,7 +886,7 @@ public virtual async Task ParallelUploadSplitSuccessfulHashComputation(StorageCh // make pipeline assertion for checking checksum was present on upload var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); + checkRequest: GetRequestChecksumAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1001,10 +923,8 @@ public virtual async Task ParallelUploadOneShotSuccessfulHashComputation(Storage }; // make pipeline assertion for checking checksum was present on upload - var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 - ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) - : GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); + var checksumPipelineAssertion = new AssertMessageContentsPolicy( + checkRequest: GetRequestChecksumAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1061,7 +981,7 @@ public virtual async Task ParallelUploadPrecalculatedComposableHashAccepted(Stor PrecalculatedChecksum = hash }; - var client = await GetResourceClientAsync(disposingContainer.Container, dataLength, createResource: true); + var client = await GetResourceClientAsync(disposingContainer.Container, dataLength); // Act await DoesNotThrowOrInconclusiveAsync( @@ -1091,10 +1011,8 @@ public virtual async Task ParallelUploadUsesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload - var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split - ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) - : GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion( + clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1145,10 +1063,8 @@ public virtual async Task ParallelUploadOverwritesDefaultClientValidationOptions }; // make pipeline assertion for checking checksum was present on upload - var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split - ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) - : GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion( + overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1203,7 +1119,7 @@ public virtual async Task ParallelUploadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) + if (request.Headers.Contains("x-ms-content-crc64")) { Assert.Fail($"Hash found when none expected."); } @@ -1268,17 +1184,15 @@ public virtual async Task ParallelDownloadSuccessfulHashVerification( }; // Act - byte[] dest; - using (MemoryStream ms = new()) + var dest = new MemoryStream(); using (checksumPipelineAssertion.CheckRequestScope()) { - await ParallelDownloadAsync(client, ms, validationOptions, transferOptions); - dest = ms.ToArray(); + await ParallelDownloadAsync(client, dest, validationOptions, transferOptions); } // Assert // Assertion was in the pipeline and the SDK not throwing means the checksum was validated - Assert.IsTrue(dest.SequenceEqual(data)); + Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } [Test] @@ -1443,7 +1357,7 @@ public virtual async Task ParallelDownloadDisablesDefaultClientValidationOptions { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) + if (response.Headers.Contains("x-ms-content-crc64")) { Assert.Fail($"Hash found when none expected."); } @@ -1651,7 +1565,7 @@ public virtual async Task OpenReadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) + if (response.Headers.Contains("x-ms-content-crc64")) { Assert.Fail($"Hash found when none expected."); } @@ -1701,7 +1615,7 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; // Act - using var dest = new MemoryStream(); + var dest = new MemoryStream(); var response = await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); // Assert @@ -1712,71 +1626,13 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); + Assert.True(response.Headers.Contains("x-ms-content-crc64")); break; default: Assert.Fail("Test can't validate given algorithm type."); break; } - var result = dest.ToArray(); - Assert.IsTrue(result.SequenceEqual(data)); - } - - [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader, false, false)] - [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader-1, false, false)] - [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader+1, true, false)] - [TestCase(StorageChecksumAlgorithm.MD5, Constants.StructuredMessage.MaxDownloadCrcWithHeader+1, false, true)] - public virtual async Task DownloadApporpriatelyUsesStructuredMessage( - StorageChecksumAlgorithm algorithm, - int? downloadLen, - bool expectStructuredMessage, - bool expectThrow) - { - await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); - - // Arrange - const int dataLength = Constants.KB; - var data = GetRandomBuffer(dataLength); - - var resourceName = GetNewResourceName(); - var client = await GetResourceClientAsync( - disposingContainer.Container, - resourceLength: dataLength, - createResource: true, - resourceName: resourceName); - await SetupDataAsync(client, new MemoryStream(data)); - - // make pipeline assertion for checking checksum was present on download - HttpPipelinePolicy checksumPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: expectStructuredMessage - ? GetResponseStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64) - : GetResponseChecksumAssertion(algorithm)); - TClientOptions clientOptions = ClientBuilder.GetOptions(); - clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); - - client = await GetResourceClientAsync( - disposingContainer.Container, - resourceLength: dataLength, - resourceName: resourceName, - createResource: false, - downloadAlgorithm: algorithm, - options: clientOptions); - - var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; - - // Act - var dest = new MemoryStream(); - AsyncTestDelegate operation = async () => await DownloadPartitionAsync( - client, dest, validationOptions, downloadLen.HasValue ? new HttpRange(length: downloadLen.Value) : default); - // Assert (policies checked use of content validation) - if (expectThrow) - { - Assert.That(operation, Throws.TypeOf()); - } - else - { - Assert.That(operation, Throws.Nothing); - Assert.IsTrue(dest.ToArray().SequenceEqual(data)); - } + Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } [Test, Combinatorial] @@ -1802,9 +1658,7 @@ public virtual async Task DownloadHashMismatchThrows( // alter response contents in pipeline, forcing a checksum mismatch on verification step var clientOptions = ClientBuilder.GetOptions(); - var tamperPolicy = TamperStreamContentsPolicy.TamperByteAt(50); - tamperPolicy.TransformResponseBody = true; - clientOptions.AddPolicy(tamperPolicy, HttpPipelinePosition.PerCall); + clientOptions.AddPolicy(new TamperStreamContentsPolicy() { TransformResponseBody = true }, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource: false, @@ -1816,7 +1670,7 @@ public virtual async Task DownloadHashMismatchThrows( AsyncTestDelegate operation = async () => await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); // Assert - if (validate || algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) + if (validate) { // SDK responsible for finding bad checksum. Throw. ThrowsOrInconclusiveAsync(operation); @@ -1874,7 +1728,7 @@ public virtual async Task DownloadUsesDefaultClientValidationOptions( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); + Assert.True(response.Headers.Contains("x-ms-content-crc64")); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -1934,7 +1788,7 @@ public virtual async Task DownloadOverwritesDefaultClientValidationOptions( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); + Assert.True(response.Headers.Contains("x-ms-content-crc64")); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -1973,7 +1827,7 @@ public virtual async Task DownloadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) + if (response.Headers.Contains("x-ms-content-crc64")) { Assert.Fail($"Hash found when none expected."); } @@ -1996,54 +1850,7 @@ public virtual async Task DownloadDisablesDefaultClientValidationOptions( // Assert // no policies this time; just check response headers Assert.False(response.Headers.Contains("Content-MD5")); - Assert.False(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)); - Assert.IsTrue(dest.ToArray().SequenceEqual(data)); - } - - [Test] - public virtual async Task DownloadRecoversFromInterruptWithValidation( - [ValueSource(nameof(GetValidationAlgorithms))] StorageChecksumAlgorithm algorithm) - { - using var _ = AzureEventSourceListener.CreateConsoleLogger(); - int dataLen = algorithm.ResolveAuto() switch { - StorageChecksumAlgorithm.StorageCrc64 => 5 * Constants.MB, // >4MB for multisegment - _ => Constants.KB, - }; - - await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); - - // Arrange - var data = GetRandomBuffer(dataLen); - - TClientOptions options = ClientBuilder.GetOptions(); - options.AddPolicy(new FaultyDownloadPipelinePolicy(dataLen - 512, new IOException(), () => { }), HttpPipelinePosition.BeforeTransport); - var client = await GetResourceClientAsync( - disposingContainer.Container, - resourceLength: dataLen, - createResource: true, - options: options); - await SetupDataAsync(client, new MemoryStream(data)); - - var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; - - // Act - var dest = new MemoryStream(); - var response = await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); - - // Assert - // no policies this time; just check response headers - switch (algorithm.ResolveAuto()) - { - case StorageChecksumAlgorithm.MD5: - Assert.True(response.Headers.Contains("Content-MD5")); - break; - case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); - break; - default: - Assert.Fail("Test can't validate given algorithm type."); - break; - } + Assert.False(response.Headers.Contains("x-ms-content-crc64")); Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } #endregion @@ -2084,7 +1891,7 @@ public async Task RoundtripWIthDefaults() // make pipeline assertion for checking checksum was present on upload AND download var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumHeaderAssertion(expectedAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected), + checkRequest: GetRequestChecksumAssertion(expectedAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected), checkResponse: GetResponseChecksumAssertion(expectedAlgorithm)); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs deleted file mode 100644 index a0f9158040b11..0000000000000 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers.Binary; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Azure.Core; -using Azure.Storage.Shared; -using Azure.Storage.Test.Shared; -using Microsoft.Diagnostics.Tracing.Parsers.AspNet; -using Moq; -using NUnit.Framework; - -namespace Azure.Storage.Tests; - -[TestFixture(true)] -[TestFixture(false)] -public class StructuredMessageDecodingRetriableStreamTests -{ - public bool Async { get; } - - public StructuredMessageDecodingRetriableStreamTests(bool async) - { - Async = async; - } - - private Mock AllExceptionsRetry() - { - Mock mock = new(MockBehavior.Strict); - mock.Setup(rc => rc.IsRetriableException(It.IsAny())).Returns(true); - return mock; - } - - [Test] - public async ValueTask UninterruptedStream() - { - byte[] data = new Random().NextBytesInline(4 * Constants.KB).ToArray(); - byte[] dest = new byte[data.Length]; - - // mock with a simple MemoryStream rather than an actual StructuredMessageDecodingStream - using (Stream src = new MemoryStream(data)) - using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream(src, new(), default, default, default, default, default, 1)) - using (Stream dst = new MemoryStream(dest)) - { - await retriableSrc.CopyToInternal(dst, Async, default); - } - - Assert.AreEqual(data, dest); - } - - [Test] - public async Task Interrupt_DataIntact([Values(true, false)] bool multipleInterrupts) - { - const int segments = 4; - const int segmentLen = Constants.KB; - const int readLen = 128; - const int interruptPos = segmentLen + (3 * readLen) + 10; - - Random r = new(); - byte[] data = r.NextBytesInline(segments * Constants.KB).ToArray(); - byte[] dest = new byte[data.Length]; - - // Mock a decoded data for the mocked StructuredMessageDecodingStream - StructuredMessageDecodingStream.RawDecodedData initialDecodedData = new() - { - TotalSegments = segments, - InnerStreamLength = data.Length, - Flags = StructuredMessage.Flags.StorageCrc64 - }; - // for test purposes, initialize a DecodedData, since we are not actively decoding in this test - initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); - - (Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData) Factory(long offset, bool faulty) - { - Stream stream = new MemoryStream(data, (int)offset, data.Length - (int)offset); - if (faulty) - { - stream = new FaultyStream(stream, interruptPos, 1, new Exception(), () => { }); - } - // Mock a decoded data for the mocked StructuredMessageDecodingStream - StructuredMessageDecodingStream.RawDecodedData decodedData = new() - { - TotalSegments = segments, - InnerStreamLength = data.Length, - Flags = StructuredMessage.Flags.StorageCrc64, - }; - // for test purposes, initialize a DecodedData, since we are not actively decoding in this test - initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); - return (stream, decodedData); - } - - // mock with a simple MemoryStream rather than an actual StructuredMessageDecodingStream - using (Stream src = new MemoryStream(data)) - using (Stream faultySrc = new FaultyStream(src, interruptPos, 1, new Exception(), () => { })) - using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream( - faultySrc, - initialDecodedData, - default, - offset => Factory(offset, multipleInterrupts), - offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)>(Factory(offset, multipleInterrupts)), - null, - AllExceptionsRetry().Object, - int.MaxValue)) - using (Stream dst = new MemoryStream(dest)) - { - await retriableSrc.CopyToInternal(dst, readLen, Async, default); - } - - Assert.AreEqual(data, dest); - } - - [Test] - public async Task Interrupt_AppropriateRewind() - { - const int segments = 2; - const int segmentLen = Constants.KB; - const int dataLen = segments * segmentLen; - const int readLen = segmentLen / 4; - const int interruptOffset = 10; - const int interruptPos = segmentLen + (2 * readLen) + interruptOffset; - Random r = new(); - - // Mock a decoded data for the mocked StructuredMessageDecodingStream - StructuredMessageDecodingStream.RawDecodedData initialDecodedData = new() - { - TotalSegments = segments, - InnerStreamLength = segments * segmentLen, - Flags = StructuredMessage.Flags.StorageCrc64, - }; - // By the time of interrupt, there will be one segment reported - initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); - - Mock mock = new(MockBehavior.Strict); - mock.SetupGet(s => s.CanRead).Returns(true); - mock.SetupGet(s => s.CanSeek).Returns(false); - if (Async) - { - mock.SetupSequence(s => s.ReadAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) - .Returns(Task.FromResult(readLen)) // start first segment - .Returns(Task.FromResult(readLen)) - .Returns(Task.FromResult(readLen)) - .Returns(Task.FromResult(readLen)) // finish first segment - .Returns(Task.FromResult(readLen)) // start second segment - .Returns(Task.FromResult(readLen)) - // faulty stream interrupt - .Returns(Task.FromResult(readLen * 2)) // restart second segment. fast-forward uses an internal 4KB buffer, so it will leap the 512 byte catchup all at once - .Returns(Task.FromResult(readLen)) - .Returns(Task.FromResult(readLen)) // end second segment - .Returns(Task.FromResult(0)) // signal end of stream - .Returns(Task.FromResult(0)) // second signal needed for stream wrapping reasons - ; - } - else - { - mock.SetupSequence(s => s.Read(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(readLen) // start first segment - .Returns(readLen) - .Returns(readLen) - .Returns(readLen) // finish first segment - .Returns(readLen) // start second segment - .Returns(readLen) - // faulty stream interrupt - .Returns(readLen * 2) // restart second segment. fast-forward uses an internal 4KB buffer, so it will leap the 512 byte catchup all at once - .Returns(readLen) - .Returns(readLen) // end second segment - .Returns(0) // signal end of stream - .Returns(0) // second signal needed for stream wrapping reasons - ; - } - Stream faultySrc = new FaultyStream(mock.Object, interruptPos, 1, new Exception(), default); - Stream retriableSrc = new StructuredMessageDecodingRetriableStream( - faultySrc, - initialDecodedData, - default, - offset => (mock.Object, new()), - offset => new(Task.FromResult((mock.Object, new StructuredMessageDecodingStream.RawDecodedData()))), - null, - AllExceptionsRetry().Object, - 1); - - int totalRead = 0; - int read = 0; - byte[] buf = new byte[readLen]; - if (Async) - { - while ((read = await retriableSrc.ReadAsync(buf, 0, buf.Length)) > 0) - { - totalRead += read; - } - } - else - { - while ((read = retriableSrc.Read(buf, 0, buf.Length)) > 0) - { - totalRead += read; - } - } - await retriableSrc.CopyToInternal(Stream.Null, readLen, Async, default); - - // Asserts we read exactly the data length, excluding the fastforward of the inner stream - Assert.That(totalRead, Is.EqualTo(dataLen)); - } - - [Test] - public async Task Interrupt_ProperDecode([Values(true, false)] bool multipleInterrupts) - { - // decoding stream inserts a buffered layer of 4 KB. use larger sizes to avoid interference from it. - const int segments = 4; - const int segmentLen = 128 * Constants.KB; - const int readLen = 8 * Constants.KB; - const int interruptPos = segmentLen + (3 * readLen) + 10; - - Random r = new(); - byte[] data = r.NextBytesInline(segments * Constants.KB).ToArray(); - byte[] dest = new byte[data.Length]; - - (Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData) Factory(long offset, bool faulty) - { - Stream stream = new MemoryStream(data, (int)offset, data.Length - (int)offset); - stream = new StructuredMessageEncodingStream(stream, segmentLen, StructuredMessage.Flags.StorageCrc64); - if (faulty) - { - stream = new FaultyStream(stream, interruptPos, 1, new Exception(), () => { }); - } - return StructuredMessageDecodingStream.WrapStream(stream); - } - - (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = Factory(0, true); - using Stream retriableSrc = new StructuredMessageDecodingRetriableStream( - decodingStream, - decodedData, - default, - offset => Factory(offset, multipleInterrupts), - offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)>(Factory(offset, multipleInterrupts)), - null, - AllExceptionsRetry().Object, - int.MaxValue); - using Stream dst = new MemoryStream(dest); - - await retriableSrc.CopyToInternal(dst, readLen, Async, default); - - Assert.AreEqual(data, dest); - } -} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs deleted file mode 100644 index 2789672df4976..0000000000000 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers.Binary; -using System.Dynamic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Azure.Storage.Blobs.Tests; -using Azure.Storage.Shared; -using NUnit.Framework; -using static Azure.Storage.Shared.StructuredMessage; - -namespace Azure.Storage.Tests -{ - [TestFixture(ReadMethod.SyncArray)] - [TestFixture(ReadMethod.AsyncArray)] -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - [TestFixture(ReadMethod.SyncSpan)] - [TestFixture(ReadMethod.AsyncMemory)] -#endif - public class StructuredMessageDecodingStreamTests - { - // Cannot just implement as passthru in the stream - // Must test each one - public enum ReadMethod - { - SyncArray, - AsyncArray, -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - SyncSpan, - AsyncMemory -#endif - } - - public ReadMethod Method { get; } - - public StructuredMessageDecodingStreamTests(ReadMethod method) - { - Method = method; - } - - private class CopyStreamException : Exception - { - public long TotalCopied { get; } - - public CopyStreamException(Exception inner, long totalCopied) - : base($"Failed read after {totalCopied}-many bytes.", inner) - { - TotalCopied = totalCopied; - } - } - private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl - { - byte[] buf = new byte[bufferSize]; - int read; - long totalRead = 0; - try - { - switch (Method) - { - case ReadMethod.SyncArray: - while ((read = source.Read(buf, 0, bufferSize)) > 0) - { - totalRead += read; - destination.Write(buf, 0, read); - } - break; - case ReadMethod.AsyncArray: - while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) - { - totalRead += read; - await destination.WriteAsync(buf, 0, read); - } - break; -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - case ReadMethod.SyncSpan: - while ((read = source.Read(new Span(buf))) > 0) - { - totalRead += read; - destination.Write(new Span(buf, 0, read)); - } - break; - case ReadMethod.AsyncMemory: - while ((read = await source.ReadAsync(new Memory(buf))) > 0) - { - totalRead += read; - await destination.WriteAsync(new Memory(buf, 0, read)); - } - break; -#endif - } - destination.Flush(); - } - catch (Exception ex) - { - throw new CopyStreamException(ex, totalRead); - } - return totalRead; - } - - [Test] - [Pairwise] - public async Task DecodesData( - [Values(2048, 2005)] int dataLength, - [Values(default, 512)] int? seglen, - [Values(8*Constants.KB, 512, 530, 3)] int readLen, - [Values(true, false)] bool useCrc) - { - int segmentContentLength = seglen ?? int.MaxValue; - Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; - - byte[] originalData = new byte[dataLength]; - new Random().NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); - - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); - byte[] decodedData; - using (MemoryStream dest = new()) - { - await CopyStream(decodingStream, dest, readLen); - decodedData = dest.ToArray(); - } - - Assert.That(new Span(decodedData).SequenceEqual(originalData)); - } - - [Test] - public void BadStreamBadVersion() - { - byte[] originalData = new byte[1024]; - new Random().NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); - - encodedData[0] = byte.MaxValue; - - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); - Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); - } - - [Test] - public async Task BadSegmentCrcThrows() - { - const int segmentLength = 256; - Random r = new(); - - byte[] originalData = new byte[2048]; - r.NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentLength, Flags.StorageCrc64); - - const int badBytePos = 1024; - encodedData[badBytePos] = (byte)~encodedData[badBytePos]; - - MemoryStream encodedDataStream = new(encodedData); - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(encodedDataStream); - - // manual try/catch to validate the proccess failed mid-stream rather than the end - const int copyBufferSize = 4; - bool caught = false; - try - { - await CopyStream(decodingStream, Stream.Null, copyBufferSize); - } - catch (CopyStreamException ex) - { - caught = true; - Assert.That(ex.TotalCopied, Is.LessThanOrEqualTo(badBytePos)); - } - Assert.That(caught); - } - - [Test] - public void BadStreamCrcThrows() - { - const int segmentLength = 256; - Random r = new(); - - byte[] originalData = new byte[2048]; - r.NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentLength, Flags.StorageCrc64); - - encodedData[originalData.Length - 1] = (byte)~encodedData[originalData.Length - 1]; - - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); - Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); - } - - [Test] - public void BadStreamWrongContentLength() - { - byte[] originalData = new byte[1024]; - new Random().NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); - - BinaryPrimitives.WriteInt64LittleEndian(new Span(encodedData, V1_0.StreamHeaderMessageLengthOffset, 8), 123456789L); - - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); - Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); - } - - [TestCase(-1)] - [TestCase(1)] - public void BadStreamWrongSegmentCount(int difference) - { - const int dataSize = 1024; - const int segmentSize = 256; - const int numSegments = 4; - - byte[] originalData = new byte[dataSize]; - new Random().NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentSize, Flags.StorageCrc64); - - // rewrite the segment count to be different than the actual number of segments - BinaryPrimitives.WriteInt16LittleEndian( - new Span(encodedData, V1_0.StreamHeaderSegmentCountOffset, 2), (short)(numSegments + difference)); - - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); - Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); - } - - [Test] - public void BadStreamWrongSegmentNum() - { - byte[] originalData = new byte[1024]; - new Random().NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); - - BinaryPrimitives.WriteInt16LittleEndian( - new Span(encodedData, V1_0.StreamHeaderLength + V1_0.SegmentHeaderNumOffset, 2), 123); - - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); - Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); - } - - [Test] - [Combinatorial] - public async Task BadStreamWrongContentLength( - [Values(-1, 1)] int difference, - [Values(true, false)] bool lengthProvided) - { - byte[] originalData = new byte[1024]; - new Random().NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); - - BinaryPrimitives.WriteInt64LittleEndian( - new Span(encodedData, V1_0.StreamHeaderMessageLengthOffset, 8), - encodedData.Length + difference); - - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream( - new MemoryStream(encodedData), - lengthProvided ? (long?)encodedData.Length : default); - - // manual try/catch with tiny buffer to validate the proccess failed mid-stream rather than the end - const int copyBufferSize = 4; - bool caught = false; - try - { - await CopyStream(decodingStream, Stream.Null, copyBufferSize); - } - catch (CopyStreamException ex) - { - caught = true; - if (lengthProvided) - { - Assert.That(ex.TotalCopied, Is.EqualTo(0)); - } - else - { - Assert.That(ex.TotalCopied, Is.EqualTo(originalData.Length)); - } - } - Assert.That(caught); - } - - [Test] - public void BadStreamMissingExpectedStreamFooter() - { - byte[] originalData = new byte[1024]; - new Random().NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); - - byte[] brokenData = new byte[encodedData.Length - Crc64Length]; - new Span(encodedData, 0, encodedData.Length - Crc64Length).CopyTo(brokenData); - - (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(brokenData)); - Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); - } - - [Test] - public void NoSeek() - { - (Stream stream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream()); - - Assert.That(stream.CanSeek, Is.False); - Assert.That(() => stream.Length, Throws.TypeOf()); - Assert.That(() => stream.Position, Throws.TypeOf()); - Assert.That(() => stream.Position = 0, Throws.TypeOf()); - Assert.That(() => stream.Seek(0, SeekOrigin.Begin), Throws.TypeOf()); - } - - [Test] - public void NoWrite() - { - (Stream stream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream()); - byte[] data = new byte[1024]; - new Random().NextBytes(data); - - Assert.That(stream.CanWrite, Is.False); - Assert.That(() => stream.Write(data, 0, data.Length), - Throws.TypeOf()); - Assert.That(async () => await stream.WriteAsync(data, 0, data.Length, CancellationToken.None), - Throws.TypeOf()); -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - Assert.That(() => stream.Write(new Span(data)), - Throws.TypeOf()); - Assert.That(async () => await stream.WriteAsync(new Memory(data), CancellationToken.None), - Throws.TypeOf()); -#endif - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs deleted file mode 100644 index e0f91dee7de3a..0000000000000 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers.Binary; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Azure.Storage.Blobs.Tests; -using Azure.Storage.Shared; -using NUnit.Framework; -using static Azure.Storage.Shared.StructuredMessage; - -namespace Azure.Storage.Tests -{ - [TestFixture(ReadMethod.SyncArray)] - [TestFixture(ReadMethod.AsyncArray)] -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - [TestFixture(ReadMethod.SyncSpan)] - [TestFixture(ReadMethod.AsyncMemory)] -#endif - public class StructuredMessageEncodingStreamTests - { - // Cannot just implement as passthru in the stream - // Must test each one - public enum ReadMethod - { - SyncArray, - AsyncArray, -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - SyncSpan, - AsyncMemory -#endif - } - - public ReadMethod Method { get; } - - public StructuredMessageEncodingStreamTests(ReadMethod method) - { - Method = method; - } - - private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl - { - byte[] buf = new byte[bufferSize]; - int read; - switch (Method) - { - case ReadMethod.SyncArray: - while ((read = source.Read(buf, 0, bufferSize)) > 0) - { - destination.Write(buf, 0, read); - } - break; - case ReadMethod.AsyncArray: - while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) - { - await destination.WriteAsync(buf, 0, read); - } - break; -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - case ReadMethod.SyncSpan: - while ((read = source.Read(new Span(buf))) > 0) - { - destination.Write(new Span(buf, 0, read)); - } - break; - case ReadMethod.AsyncMemory: - while ((read = await source.ReadAsync(new Memory(buf))) > 0) - { - await destination.WriteAsync(new Memory(buf, 0, read)); - } - break; -#endif - } - destination.Flush(); - } - - [Test] - [Pairwise] - public async Task EncodesData( - [Values(2048, 2005)] int dataLength, - [Values(default, 512)] int? seglen, - [Values(8 * Constants.KB, 512, 530, 3)] int readLen, - [Values(true, false)] bool useCrc) - { - int segmentContentLength = seglen ?? int.MaxValue; - Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; - - byte[] originalData = new byte[dataLength]; - new Random().NextBytes(originalData); - byte[] expectedEncodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); - - Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(originalData), segmentContentLength, flags); - byte[] encodedData; - using (MemoryStream dest = new()) - { - await CopyStream(encodingStream, dest, readLen); - encodedData = dest.ToArray(); - } - - Assert.That(new Span(encodedData).SequenceEqual(expectedEncodedData)); - } - - [TestCase(0, 0)] // start - [TestCase(5, 0)] // partway through stream header - [TestCase(V1_0.StreamHeaderLength, 0)] // start of segment - [TestCase(V1_0.StreamHeaderLength + 3, 0)] // partway through segment header - [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength, 0)] // start of segment content - [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 123, 123)] // partway through segment content - [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 512, 512)] // start of segment footer - [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 515, 512)] // partway through segment footer - [TestCase(V1_0.StreamHeaderLength + 3*V1_0.SegmentHeaderLength + 2*Crc64Length + 1500, 1500)] // partway through not first segment content - public async Task Seek(int targetRewindOffset, int expectedInnerStreamPosition) - { - const int segmentLength = 512; - const int dataLength = 2055; - byte[] data = new byte[dataLength]; - new Random().NextBytes(data); - - MemoryStream dataStream = new(data); - StructuredMessageEncodingStream encodingStream = new(dataStream, segmentLength, Flags.StorageCrc64); - - // no support for seeking past existing read, need to consume whole stream before seeking - await CopyStream(encodingStream, Stream.Null); - - encodingStream.Position = targetRewindOffset; - Assert.That(encodingStream.Position, Is.EqualTo(targetRewindOffset)); - Assert.That(dataStream.Position, Is.EqualTo(expectedInnerStreamPosition)); - } - - [TestCase(0)] // start - [TestCase(5)] // partway through stream header - [TestCase(V1_0.StreamHeaderLength)] // start of segment - [TestCase(V1_0.StreamHeaderLength + 3)] // partway through segment header - [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength)] // start of segment content - [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 123)] // partway through segment content - [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 512)] // start of segment footer - [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 515)] // partway through segment footer - [TestCase(V1_0.StreamHeaderLength + 2 * V1_0.SegmentHeaderLength + Crc64Length + 1500)] // partway through not first segment content - public async Task SupportsRewind(int targetRewindOffset) - { - const int segmentLength = 512; - const int dataLength = 2055; - byte[] data = new byte[dataLength]; - new Random().NextBytes(data); - - Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); - byte[] encodedData1; - using (MemoryStream dest = new()) - { - await CopyStream(encodingStream, dest); - encodedData1 = dest.ToArray(); - } - encodingStream.Position = targetRewindOffset; - byte[] encodedData2; - using (MemoryStream dest = new()) - { - await CopyStream(encodingStream, dest); - encodedData2 = dest.ToArray(); - } - - Assert.That(new Span(encodedData1).Slice(targetRewindOffset).SequenceEqual(encodedData2)); - } - - [Test] - public async Task SupportsFastForward() - { - const int segmentLength = 512; - const int dataLength = 2055; - byte[] data = new byte[dataLength]; - new Random().NextBytes(data); - - // must have read stream to fastforward. so read whole stream upfront & save result to check later - Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); - byte[] encodedData; - using (MemoryStream dest = new()) - { - await CopyStream(encodingStream, dest); - encodedData = dest.ToArray(); - } - - encodingStream.Position = 0; - - bool skip = false; - const int increment = 499; - while (encodingStream.Position < encodingStream.Length) - { - if (skip) - { - encodingStream.Position = Math.Min(dataLength, encodingStream.Position + increment); - skip = !skip; - continue; - } - ReadOnlyMemory expected = new(encodedData, (int)encodingStream.Position, - (int)Math.Min(increment, encodedData.Length - encodingStream.Position)); - ReadOnlyMemory actual; - using (MemoryStream dest = new(increment)) - { - await CopyStream(WindowStream.GetWindow(encodingStream, increment), dest); - actual = dest.ToArray(); - } - Assert.That(expected.Span.SequenceEqual(actual.Span)); - skip = !skip; - } - } - - [Test] - public void NotSupportsFastForwardBeyondLatestRead() - { - const int segmentLength = 512; - const int dataLength = 2055; - byte[] data = new byte[dataLength]; - new Random().NextBytes(data); - - Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); - - Assert.That(() => encodingStream.Position = 123, Throws.TypeOf()); - } - - [Test] - [Pairwise] - public async Task WrapperStreamCorrectData( - [Values(2048, 2005)] int dataLength, - [Values(8 * Constants.KB, 512, 530, 3)] int readLen) - { - int segmentContentLength = dataLength; - Flags flags = Flags.StorageCrc64; - - byte[] originalData = new byte[dataLength]; - new Random().NextBytes(originalData); - byte[] crc = CrcInline(originalData); - byte[] expectedEncodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); - - Stream encodingStream = new StructuredMessagePrecalculatedCrcWrapperStream(new MemoryStream(originalData), crc); - byte[] encodedData; - using (MemoryStream dest = new()) - { - await CopyStream(encodingStream, dest, readLen); - encodedData = dest.ToArray(); - } - - Assert.That(new Span(encodedData).SequenceEqual(expectedEncodedData)); - } - - private static void AssertExpectedStreamHeader(ReadOnlySpan actual, int originalDataLength, Flags flags, int expectedSegments) - { - int expectedFooterLen = flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; - - Assert.That(actual.Length, Is.EqualTo(V1_0.StreamHeaderLength)); - Assert.That(actual[0], Is.EqualTo(1)); - Assert.That(BinaryPrimitives.ReadInt64LittleEndian(actual.Slice(1, 8)), - Is.EqualTo(V1_0.StreamHeaderLength + expectedSegments * (V1_0.SegmentHeaderLength + expectedFooterLen) + originalDataLength)); - Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(9, 2)), Is.EqualTo((short)flags)); - Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(11, 2)), Is.EqualTo((short)expectedSegments)); - } - - private static void AssertExpectedSegmentHeader(ReadOnlySpan actual, int segmentNum, long contentLength) - { - Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(0, 2)), Is.EqualTo((short) segmentNum)); - Assert.That(BinaryPrimitives.ReadInt64LittleEndian(actual.Slice(2, 8)), Is.EqualTo(contentLength)); - } - - private static byte[] CrcInline(ReadOnlySpan data) - { - var crc = StorageCrc64HashAlgorithm.Create(); - crc.Append(data); - return crc.GetCurrentHash(); - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs deleted file mode 100644 index 59e80320d96a0..0000000000000 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Azure.Storage.Shared; -using static Azure.Storage.Shared.StructuredMessage; - -namespace Azure.Storage.Blobs.Tests -{ - internal class StructuredMessageHelper - { - public static byte[] MakeEncodedData(ReadOnlySpan data, long segmentContentLength, Flags flags) - { - int segmentCount = (int) Math.Ceiling(data.Length / (double)segmentContentLength); - int segmentFooterLen = flags.HasFlag(Flags.StorageCrc64) ? 8 : 0; - int streamFooterLen = flags.HasFlag(Flags.StorageCrc64) ? 8 : 0; - - byte[] encodedData = new byte[ - V1_0.StreamHeaderLength + - segmentCount*(V1_0.SegmentHeaderLength + segmentFooterLen) + - streamFooterLen + - data.Length]; - V1_0.WriteStreamHeader( - new Span(encodedData, 0, V1_0.StreamHeaderLength), - encodedData.Length, - flags, - segmentCount); - - int i = V1_0.StreamHeaderLength; - int j = 0; - foreach (int seg in Enumerable.Range(1, segmentCount)) - { - int segContentLen = Math.Min((int)segmentContentLength, data.Length - j); - V1_0.WriteSegmentHeader( - new Span(encodedData, i, V1_0.SegmentHeaderLength), - seg, - segContentLen); - i += V1_0.SegmentHeaderLength; - - data.Slice(j, segContentLen) - .CopyTo(new Span(encodedData).Slice(i)); - i += segContentLen; - - if (flags.HasFlag(Flags.StorageCrc64)) - { - var crc = StorageCrc64HashAlgorithm.Create(); - crc.Append(data.Slice(j, segContentLen)); - crc.GetCurrentHash(new Span(encodedData, i, Crc64Length)); - i += Crc64Length; - } - j += segContentLen; - } - - if (flags.HasFlag(Flags.StorageCrc64)) - { - var crc = StorageCrc64HashAlgorithm.Create(); - crc.Append(data); - crc.GetCurrentHash(new Span(encodedData, i, Crc64Length)); - } - - return encodedData; - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs deleted file mode 100644 index 61583aa1ebe4e..0000000000000 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Azure.Storage.Shared; -using NUnit.Framework; -using static Azure.Storage.Shared.StructuredMessage; - -namespace Azure.Storage.Tests -{ - [TestFixture(ReadMethod.SyncArray)] - [TestFixture(ReadMethod.AsyncArray)] -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - [TestFixture(ReadMethod.SyncSpan)] - [TestFixture(ReadMethod.AsyncMemory)] -#endif - public class StructuredMessageStreamRoundtripTests - { - // Cannot just implement as passthru in the stream - // Must test each one - public enum ReadMethod - { - SyncArray, - AsyncArray, -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - SyncSpan, - AsyncMemory -#endif - } - - public ReadMethod Method { get; } - - public StructuredMessageStreamRoundtripTests(ReadMethod method) - { - Method = method; - } - - private class CopyStreamException : Exception - { - public long TotalCopied { get; } - - public CopyStreamException(Exception inner, long totalCopied) - : base($"Failed read after {totalCopied}-many bytes.", inner) - { - TotalCopied = totalCopied; - } - } - private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl - { - byte[] buf = new byte[bufferSize]; - int read; - long totalRead = 0; - try - { - switch (Method) - { - case ReadMethod.SyncArray: - while ((read = source.Read(buf, 0, bufferSize)) > 0) - { - totalRead += read; - destination.Write(buf, 0, read); - } - break; - case ReadMethod.AsyncArray: - while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) - { - totalRead += read; - await destination.WriteAsync(buf, 0, read); - } - break; -#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - case ReadMethod.SyncSpan: - while ((read = source.Read(new Span(buf))) > 0) - { - totalRead += read; - destination.Write(new Span(buf, 0, read)); - } - break; - case ReadMethod.AsyncMemory: - while ((read = await source.ReadAsync(new Memory(buf))) > 0) - { - totalRead += read; - await destination.WriteAsync(new Memory(buf, 0, read)); - } - break; -#endif - } - destination.Flush(); - } - catch (Exception ex) - { - throw new CopyStreamException(ex, totalRead); - } - return totalRead; - } - - [Test] - [Pairwise] - public async Task RoundTrip( - [Values(2048, 2005)] int dataLength, - [Values(default, 512)] int? seglen, - [Values(8 * Constants.KB, 512, 530, 3)] int readLen, - [Values(true, false)] bool useCrc) - { - int segmentLength = seglen ?? int.MaxValue; - Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; - - byte[] originalData = new byte[dataLength]; - new Random().NextBytes(originalData); - - byte[] roundtripData; - using (MemoryStream source = new(originalData)) - using (Stream encode = new StructuredMessageEncodingStream(source, segmentLength, flags)) - using (Stream decode = StructuredMessageDecodingStream.WrapStream(encode).DecodedStream) - using (MemoryStream dest = new()) - { - await CopyStream(source, dest, readLen); - roundtripData = dest.ToArray(); - } - - Assert.That(originalData.SequenceEqual(roundtripData)); - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs deleted file mode 100644 index b4f1dfe178246..0000000000000 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Buffers.Binary; -using System.Collections.Generic; -using NUnit.Framework; -using static Azure.Storage.Shared.StructuredMessage; - -namespace Azure.Storage.Tests -{ - public class StructuredMessageTests - { - [TestCase(1024, Flags.None, 2)] - [TestCase(2000, Flags.StorageCrc64, 4)] - public void EncodeStreamHeader(int messageLength, int flags, int numSegments) - { - Span encoding = new(new byte[V1_0.StreamHeaderLength]); - V1_0.WriteStreamHeader(encoding, messageLength, (Flags)flags, numSegments); - - Assert.That(encoding[0], Is.EqualTo((byte)1)); - Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(encoding.Slice(1, 8)), Is.EqualTo(messageLength)); - Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(9, 2)), Is.EqualTo(flags)); - Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(11, 2)), Is.EqualTo(numSegments)); - } - - [TestCase(V1_0.StreamHeaderLength)] - [TestCase(V1_0.StreamHeaderLength + 1)] - [TestCase(V1_0.StreamHeaderLength - 1)] - public void EncodeStreamHeaderRejectBadBufferSize(int bufferSize) - { - Random r = new(); - byte[] encoding = new byte[bufferSize]; - - void Action() => V1_0.WriteStreamHeader(encoding, r.Next(2, int.MaxValue), Flags.StorageCrc64, r.Next(2, int.MaxValue)); - if (bufferSize < V1_0.StreamHeaderLength) - { - Assert.That(Action, Throws.ArgumentException); - } - else - { - Assert.That(Action, Throws.Nothing); - } - } - - [TestCase(1, 1024)] - [TestCase(5, 39578)] - public void EncodeSegmentHeader(int segmentNum, int contentLength) - { - Span encoding = new(new byte[V1_0.SegmentHeaderLength]); - V1_0.WriteSegmentHeader(encoding, segmentNum, contentLength); - - Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(0, 2)), Is.EqualTo(segmentNum)); - Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(encoding.Slice(2, 8)), Is.EqualTo(contentLength)); - } - - [TestCase(V1_0.SegmentHeaderLength)] - [TestCase(V1_0.SegmentHeaderLength + 1)] - [TestCase(V1_0.SegmentHeaderLength - 1)] - public void EncodeSegmentHeaderRejectBadBufferSize(int bufferSize) - { - Random r = new(); - byte[] encoding = new byte[bufferSize]; - - void Action() => V1_0.WriteSegmentHeader(encoding, r.Next(1, int.MaxValue), r.Next(2, int.MaxValue)); - if (bufferSize < V1_0.SegmentHeaderLength) - { - Assert.That(Action, Throws.ArgumentException); - } - else - { - Assert.That(Action, Throws.Nothing); - } - } - - [TestCase(true)] - [TestCase(false)] - public void EncodeSegmentFooter(bool useCrc) - { - Span encoding = new(new byte[Crc64Length]); - Span crc = useCrc ? new Random().NextBytesInline(Crc64Length) : default; - V1_0.WriteSegmentFooter(encoding, crc); - - if (useCrc) - { - Assert.That(encoding.SequenceEqual(crc), Is.True); - } - else - { - Assert.That(encoding.SequenceEqual(new Span(new byte[Crc64Length])), Is.True); - } - } - - [TestCase(Crc64Length)] - [TestCase(Crc64Length + 1)] - [TestCase(Crc64Length - 1)] - public void EncodeSegmentFooterRejectBadBufferSize(int bufferSize) - { - byte[] encoding = new byte[bufferSize]; - byte[] crc = new byte[Crc64Length]; - new Random().NextBytes(crc); - - void Action() => V1_0.WriteSegmentFooter(encoding, crc); - if (bufferSize < Crc64Length) - { - Assert.That(Action, Throws.ArgumentException); - } - else - { - Assert.That(Action, Throws.Nothing); - } - } - } -} diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Azure.Storage.DataMovement.Blobs.Samples.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Azure.Storage.DataMovement.Blobs.Samples.Tests.csproj index 30d4b1f79daaf..7ab901e963e03 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Azure.Storage.DataMovement.Blobs.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Azure.Storage.DataMovement.Blobs.Samples.Tests.csproj @@ -11,7 +11,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj index 921dec1e469ee..66520948815e4 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj @@ -37,7 +37,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs index 2c6864f511571..84d60b3bc37c4 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs @@ -99,7 +99,7 @@ internal static StorageResourceItemProperties ToStorageResourceItemProperties(th ContentRange contentRange = !string.IsNullOrWhiteSpace(result?.Details?.ContentRange) ? ContentRange.Parse(result.Details.ContentRange) : default; if (contentRange != default) { - size = contentRange.TotalResourceLength; + size = contentRange.Size; } return new StorageResourceItemProperties( @@ -151,7 +151,7 @@ internal static StorageResourceReadStreamResult ToReadStreamStorageResourceInfo( if (contentRange != default) { range = ContentRange.ToHttpRange(contentRange); - size = contentRange.TotalResourceLength; + size = contentRange.Size; } else if (result.Details.ContentLength > 0) { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj index 753475c1adc47..83e1264a0290a 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj @@ -22,15 +22,11 @@ - - - - @@ -44,7 +40,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj index 66a9fea0861a2..a6abde432473f 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj @@ -35,7 +35,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj index 6a472b9f74158..9cde066f64eb7 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/samples/Azure.Storage.DataMovement.Files.Shares.Samples.Tests.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks) Microsoft Azure.Storage.DataMovement.Files.Shares client library samples @@ -11,7 +11,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs index 16a164f61b060..9cb7d338fcb60 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs @@ -335,14 +335,14 @@ internal static StorageResourceReadStreamResult ToStorageResourceReadStreamResul ContentRange contentRange = !string.IsNullOrWhiteSpace(info?.Details?.ContentRange) ? ContentRange.Parse(info.Details.ContentRange) : default; if (contentRange != default) { - size = contentRange.TotalResourceLength; + size = contentRange.Size; } return new StorageResourceReadStreamResult( content: info?.Content, range: ContentRange.ToHttpRange(contentRange), properties: new StorageResourceItemProperties( - resourceLength: contentRange.TotalResourceLength, + resourceLength: contentRange.Size, eTag: info.Details.ETag, lastModifiedTime: info.Details.LastModified, properties: properties)); diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj index d75775beceafd..8e574bca36a48 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj @@ -27,7 +27,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj b/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj index 3845bc4da8b4a..fde61bae96877 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj +++ b/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks);net6.0 diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj index 7a40eb8026443..b5e3c42359976 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj @@ -34,7 +34,6 @@ - diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 5127ea7e0c4db..4a64b8398f656 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_48a38da58a" + "Tag": "net/storage/Azure.Storage.Files.DataLake_d74597f1e3" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Azure.Storage.Files.DataLake.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Files.DataLake/samples/Azure.Storage.Files.DataLake.Samples.Tests.csproj index eecbe0543fe87..c230f2ed8fa20 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Azure.Storage.Files.DataLake.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Azure.Storage.Files.DataLake.Samples.Tests.csproj @@ -15,7 +15,6 @@ - diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj index 559839b1adf8b..89a7d0d737796 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj @@ -42,7 +42,6 @@ - @@ -82,10 +81,6 @@ - - - - diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 46a555458acb6..e755faff2f5a7 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -16,7 +16,6 @@ using Azure.Storage.Common; using Azure.Storage.Files.DataLake.Models; using Azure.Storage.Sas; -using Azure.Storage.Shared; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Files.DataLake @@ -2333,39 +2332,13 @@ internal virtual async Task AppendInternal( using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(DataLakeFileClient))) { // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = null; - long contentLength = (content?.Length - content?.Position) ?? 0; - long? structuredContentLength = default; - string structuredBodyType = null; - if (content != null && - validationOptions != null && - validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) - { - // report progress in terms of caller bytes, not encoded bytes - structuredContentLength = contentLength; - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - content = content.WithNoDispose().WithProgress(progressHandler); - content = validationOptions.PrecalculatedChecksum.IsEmpty - ? new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64) - : new StructuredMessagePrecalculatedCrcWrapperStream( - content, - validationOptions.PrecalculatedChecksum.Span); - contentLength = content.Length - content.Position; - } - else - { - // compute hash BEFORE attaching progress handler - hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - content = content?.WithNoDispose().WithProgress(progressHandler); - } + ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content?.WithNoDispose().WithProgress(progressHandler); ClientConfiguration.Pipeline.LogMethodEnter( nameof(DataLakeFileClient), message: @@ -2400,8 +2373,6 @@ internal virtual async Task AppendInternal( encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, leaseId: leaseId, leaseAction: leaseAction, leaseDuration: leaseDurationLong, @@ -2421,8 +2392,6 @@ internal virtual async Task AppendInternal( encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, leaseId: leaseId, leaseAction: leaseAction, leaseDuration: leaseDurationLong, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md index a8340f1092bcb..ec9675a014f70 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md @@ -23,7 +23,7 @@ directive: if (property.includes('/{filesystem}/{path}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/FileSystem") && false == param['$ref'].endsWith("#/parameters/Path"))}); - } + } else if (property.includes('/{filesystem}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/FileSystem"))}); @@ -127,7 +127,7 @@ directive: } $[newName] = $[oldName]; delete $[oldName]; - } + } else if (property.includes('/{filesystem}')) { var oldName = property; diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj b/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj index 1fa78690077be..bef13bb21a1c6 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj @@ -6,9 +6,6 @@ Microsoft Azure.Storage.Files.DataLake client library tests false - - DataLakeSDK - diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs index 5067f98517bd2..4bdefdbf756cd 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs @@ -34,10 +34,7 @@ protected override async Task> Get StorageChecksumAlgorithm uploadAlgorithm = StorageChecksumAlgorithm.None, StorageChecksumAlgorithm downloadAlgorithm = StorageChecksumAlgorithm.None) { - var disposingFileSystem = await ClientBuilder.GetNewFileSystem( - service: service, - fileSystemName: containerName, - publicAccessType: PublicAccessType.None); + var disposingFileSystem = await ClientBuilder.GetNewFileSystem(service: service, fileSystemName: containerName); disposingFileSystem.FileSystem.ClientConfiguration.TransferValidation.Upload.ChecksumAlgorithm = uploadAlgorithm; disposingFileSystem.FileSystem.ClientConfiguration.TransferValidation.Download.ChecksumAlgorithm = downloadAlgorithm; diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs index c49119c1f9f12..430c0b16d2ebe 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs @@ -809,7 +809,6 @@ public partial class ShareFileDownloadInfo : System.IDisposable { internal ShareFileDownloadInfo() { } public System.IO.Stream Content { get { throw null; } } - public byte[] ContentCrc { get { throw null; } } public byte[] ContentHash { get { throw null; } } public long ContentLength { get { throw null; } } public string ContentType { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs index c49119c1f9f12..430c0b16d2ebe 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs @@ -809,7 +809,6 @@ public partial class ShareFileDownloadInfo : System.IDisposable { internal ShareFileDownloadInfo() { } public System.IO.Stream Content { get { throw null; } } - public byte[] ContentCrc { get { throw null; } } public byte[] ContentHash { get { throw null; } } public long ContentLength { get { throw null; } } public string ContentType { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index c33c8bb335398..c2b5c3d31e6a2 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_4b545ae555" + "Tag": "net/storage/Azure.Storage.Files.Shares_df67d82d59" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/samples/Azure.Storage.Files.Shares.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Files.Shares/samples/Azure.Storage.Files.Shares.Samples.Tests.csproj index d1efeca0c2da2..0bcec423c144d 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/samples/Azure.Storage.Files.Shares.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/samples/Azure.Storage.Files.Shares.Samples.Tests.csproj @@ -16,7 +16,6 @@ - PreserveNewest diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj index 00b9fabda4394..740f463a07c5b 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks);net6.0 @@ -42,7 +42,6 @@ - @@ -86,11 +85,6 @@ - - - - - diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs index 4037cbdfd875e..0165af94435a0 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs @@ -38,12 +38,6 @@ public partial class ShareFileDownloadInfo : IDisposable, IDownloadedContent public byte[] ContentHash { get; internal set; } #pragma warning restore CA1819 // Properties should not return arrays - /// - /// When requested using , this value contains the CRC for the download blob range. - /// This value may only become populated once the network stream is fully consumed. - /// - public byte[] ContentCrc { get; internal set; } - /// /// Details returned when downloading a file /// diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs index 0b27510aaa6c4..f776384d06add 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs @@ -17,5 +17,20 @@ public static InvalidOperationException FileOrShareMissing( string fileClient, string shareClient) => new InvalidOperationException($"{leaseClient} requires either a {fileClient} or {shareClient}"); + + public static void AssertAlgorithmSupport(StorageChecksumAlgorithm? algorithm) + { + StorageChecksumAlgorithm resolved = (algorithm ?? StorageChecksumAlgorithm.None).ResolveAuto(); + switch (resolved) + { + case StorageChecksumAlgorithm.None: + case StorageChecksumAlgorithm.MD5: + return; + case StorageChecksumAlgorithm.StorageCrc64: + throw new ArgumentException("Azure File Shares do not support CRC-64."); + default: + throw new ArgumentException($"{nameof(StorageChecksumAlgorithm)} does not support value {Enum.GetName(typeof(StorageChecksumAlgorithm), resolved) ?? ((int)resolved).ToString(CultureInfo.InvariantCulture)}."); + } + } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs index ea3f8554b944d..f713200a524de 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs @@ -2397,70 +2397,51 @@ private async Task> DownloadInternal( // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - async ValueTask> Factory(long offset, bool async, CancellationToken cancellationToken) - { - (Response response, Stream contentStream) = await StartDownloadAsync( - range, - validationOptions, - conditions, - offset, - async, - cancellationToken).ConfigureAwait(false); - if (etag != response.GetRawResponse().Headers.ETag) + initialResponse.Value.Content = RetriableStream.Create( + stream, + startOffset => { - throw new ShareFileModifiedException( - "File has been modified concurrently", - Uri, etag, response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); - } - return response; - } - async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)> StructuredMessageFactory( - long offset, bool async, CancellationToken cancellationToken) - { - Response result = await Factory(offset, async, cancellationToken).ConfigureAwait(false); - return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.ContentLength); - } - - if (initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) - { - (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( - initialResponse.Value.Content, initialResponse.Value.ContentLength); - initialResponse.Value.Content = new StructuredMessageDecodingRetriableStream( - decodingStream, - decodedData, - StructuredMessage.Flags.StorageCrc64, - startOffset => StructuredMessageFactory(startOffset, async: false, cancellationToken) - .EnsureCompleted(), - async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) - .ConfigureAwait(false), - decodedData => + (Response Response, Stream ContentStream) = StartDownloadAsync( + range, + validationOptions, + conditions, + startOffset, + async, + cancellationToken) + .EnsureCompleted(); + if (etag != Response.GetRawResponse().Headers.ETag) { - initialResponse.Value.ContentCrc = new byte[StructuredMessage.Crc64Length]; - decodedData.Crc.WriteCrc64(initialResponse.Value.ContentCrc); - }, - ClientConfiguration.Pipeline.ResponseClassifier, - Constants.MaxReliabilityRetries); - } - else - { - initialResponse.Value.Content = RetriableStream.Create( - initialResponse.Value.Content, - startOffset => Factory(startOffset, async: false, cancellationToken) - .EnsureCompleted().Value.Content, - async startOffset => (await Factory(startOffset, async: true, cancellationToken) - .ConfigureAwait(false)).Value.Content, - ClientConfiguration.Pipeline.ResponseClassifier, - Constants.MaxReliabilityRetries); - } + throw new ShareFileModifiedException( + "File has been modified concurrently", + Uri, etag, Response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); + } + return ContentStream; + }, + async startOffset => + { + (Response Response, Stream ContentStream) = await StartDownloadAsync( + range, + validationOptions, + conditions, + startOffset, + async, + cancellationToken) + .ConfigureAwait(false); + if (etag != Response.GetRawResponse().Headers.ETag) + { + throw new ShareFileModifiedException( + "File has been modified concurrently", + Uri, etag, Response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); + } + return ContentStream; + }, + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); // buffer response stream and ensure it matches the transactional hash if any // Storage will not return a hash for payload >4MB, so this buffer is capped similarly // hashing is opt-in, so this buffer is part of that opt-in - if (validationOptions != default && - validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && - validationOptions.AutoValidateChecksum && - // structured message decoding does the validation for us - !initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) + if (validationOptions != default && validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && validationOptions.AutoValidateChecksum) { // safe-buffer; transactional hash download limit well below maxInt var readDestStream = new MemoryStream((int)initialResponse.Value.ContentLength); @@ -2543,6 +2524,8 @@ await ContentHasher.AssertResponseHashMatchInternal( bool async = true, CancellationToken cancellationToken = default) { + ShareErrors.AssertAlgorithmSupport(transferValidationOverride?.ChecksumAlgorithm); + // calculation gets illegible with null coalesce; just pre-initialize var pageRange = range; pageRange = new HttpRange( @@ -2552,27 +2535,13 @@ await ContentHasher.AssertResponseHashMatchInternal( (long?)null); ClientConfiguration.Pipeline.LogTrace($"Download {Uri} with range: {pageRange}"); - bool? rangeGetContentMD5 = null; - string structuredBodyType = null; - switch (transferValidationOverride?.ChecksumAlgorithm.ResolveAuto()) - { - case StorageChecksumAlgorithm.MD5: - rangeGetContentMD5 = true; - break; - case StorageChecksumAlgorithm.StorageCrc64: - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - break; - default: - break; - } - ResponseWithHeaders response; + if (async) { response = await FileRestClient.DownloadAsync( range: pageRange == default ? null : pageRange.ToString(), - rangeGetContentMD5: rangeGetContentMD5, - structuredBodyType: structuredBodyType, + rangeGetContentMD5: transferValidationOverride?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, shareFileRequestConditions: conditions, cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -2581,8 +2550,7 @@ await ContentHasher.AssertResponseHashMatchInternal( { response = FileRestClient.Download( range: pageRange == default ? null : pageRange.ToString(), - rangeGetContentMD5: rangeGetContentMD5, - structuredBodyType: structuredBodyType, + rangeGetContentMD5: transferValidationOverride?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, shareFileRequestConditions: conditions, cancellationToken: cancellationToken); } @@ -4662,6 +4630,7 @@ internal async Task> UploadRangeInternal( CancellationToken cancellationToken) { UploadTransferValidationOptions validationOptions = transferValidationOverride ?? ClientConfiguration.TransferValidation.Upload; + ShareErrors.AssertAlgorithmSupport(validationOptions?.ChecksumAlgorithm); using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(ShareFileClient))) { @@ -4677,38 +4646,14 @@ internal async Task> UploadRangeInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - ContentHasher.GetHashResult hashResult = null; - long contentLength = (content?.Length - content?.Position) ?? 0; - long? structuredContentLength = default; - string structuredBodyType = null; - if (validationOptions != null && - validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) - { - // report progress in terms of caller bytes, not encoded bytes - structuredContentLength = contentLength; - contentLength = (content?.Length - content?.Position) ?? 0; - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - content = content.WithNoDispose().WithProgress(progressHandler); - content = validationOptions.PrecalculatedChecksum.IsEmpty - ? new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64) - : new StructuredMessagePrecalculatedCrcWrapperStream( - content, - validationOptions.PrecalculatedChecksum.Span); - contentLength = (content?.Length - content?.Position) ?? 0; - } - else - { - // compute hash BEFORE attaching progress handler - hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - content = content.WithNoDispose().WithProgress(progressHandler); - } + // compute hash BEFORE attaching progress handler + ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + + content = content.WithNoDispose().WithProgress(progressHandler); ResponseWithHeaders response; @@ -4721,8 +4666,6 @@ internal async Task> UploadRangeInternal( fileLastWrittenMode: fileLastWrittenMode, optionalbody: content, contentMD5: hashResult?.MD5AsArray, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, shareFileRequestConditions: conditions, cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -4736,8 +4679,6 @@ internal async Task> UploadRangeInternal( fileLastWrittenMode: fileLastWrittenMode, optionalbody: content, contentMD5: hashResult?.MD5AsArray, - structuredBodyType: structuredBodyType, - structuredContentLength: structuredContentLength, shareFileRequestConditions: conditions, cancellationToken: cancellationToken); } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md index c688a6c0d1093..ed634ae302734 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md @@ -25,7 +25,7 @@ directive: if (property.includes('/{shareName}/{directory}/{fileName}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ShareName") && false == param['$ref'].endsWith("#/parameters/DirectoryPath") && false == param['$ref'].endsWith("#/parameters/FilePath"))}); - } + } else if (property.includes('/{shareName}/{directory}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ShareName") && false == param['$ref'].endsWith("#/parameters/DirectoryPath"))}); @@ -46,7 +46,7 @@ directive: $.Metrics.type = "object"; ``` -### Times aren't required +### Times aren't required ``` yaml directive: - from: swagger-document diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj index d09dd8fe8949f..398a4b6367489 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj @@ -17,7 +17,6 @@ - PreserveNewest diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs index 9fd8905e388b1..3dcdb21f27b36 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs @@ -64,6 +64,10 @@ protected override async Task GetResourceClientAsync( private void AssertSupportsHashAlgorithm(StorageChecksumAlgorithm algorithm) { + if (algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) + { + TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "Azure File Share does not support CRC64."); + } } protected override async Task UploadPartitionAsync(ShareFileClient client, Stream source, UploadTransferValidationOptions transferValidation) @@ -143,44 +147,8 @@ protected override async Task SetupDataAsync(ShareFileClient client, Stream data public override void TestAutoResolve() { Assert.AreEqual( - StorageChecksumAlgorithm.StorageCrc64, + StorageChecksumAlgorithm.MD5, TransferValidationOptionsExtensions.ResolveAuto(StorageChecksumAlgorithm.Auto)); } - - [Test] - public async Task StructuredMessagePopulatesCrcDownloadStreaming() - { - await using DisposingShare disposingContainer = await ClientBuilder.GetTestShareAsync(); - - const int dataLength = Constants.KB; - byte[] data = GetRandomBuffer(dataLength); - byte[] dataCrc = new byte[8]; - StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); - - ShareFileClient file = disposingContainer.Container.GetRootDirectoryClient().GetFileClient(GetNewResourceName()); - await file.CreateAsync(data.Length); - await file.UploadAsync(new MemoryStream(data)); - - Response response = await file.DownloadAsync(new ShareFileDownloadOptions() - { - TransferValidation = new DownloadTransferValidationOptions - { - ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 - } - }); - - // crc is not present until response stream is consumed - Assert.That(response.Value.ContentCrc, Is.Null); - - byte[] downloadedData; - using (MemoryStream ms = new()) - { - await response.Value.Content.CopyToAsync(ms); - downloadedData = ms.ToArray(); - } - - Assert.That(response.Value.ContentCrc, Is.EqualTo(dataCrc)); - Assert.That(downloadedData, Is.EqualTo(data)); - } } } diff --git a/sdk/storage/Azure.Storage.Queues/samples/Azure.Storage.Queues.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Queues/samples/Azure.Storage.Queues.Samples.Tests.csproj index 12794e190f4e1..f9ed70da2e75d 100644 --- a/sdk/storage/Azure.Storage.Queues/samples/Azure.Storage.Queues.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/samples/Azure.Storage.Queues.Samples.Tests.csproj @@ -16,7 +16,6 @@ - PreserveNewest diff --git a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj index 4d0334255f041..e0a6fab3c753b 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj @@ -21,7 +21,6 @@ - From 8df441914122d1582433947f2c3132a161eb76f2 Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:01:52 -0800 Subject: [PATCH 04/10] NFS over REST (#46519) --- .../api/Azure.Storage.Files.Shares.net6.0.cs | 74 +++ ...ure.Storage.Files.Shares.netstandard2.0.cs | 74 +++ .../Azure.Storage.Files.Shares/assets.json | 2 +- .../src/Generated/DirectoryCreateHeaders.cs | 9 + .../DirectoryGetPropertiesHeaders.cs | 9 + .../src/Generated/DirectoryRestClient.cs | 100 ++-- .../DirectorySetPropertiesHeaders.cs | 6 + .../Generated/FileCreateHardLinkHeaders.cs | 46 ++ .../src/Generated/FileCreateHeaders.cs | 9 + .../FileCreateSymbolicLinkHeaders.cs | 44 ++ .../src/Generated/FileDeleteHeaders.cs | 2 + .../src/Generated/FileDownloadHeaders.cs | 8 + .../src/Generated/FileGetPropertiesHeaders.cs | 10 + .../Generated/FileGetSymbolicLinkHeaders.cs | 27 + .../src/Generated/FileRestClient.cs | 391 +++++++++++-- .../Generated/FileSetHttpHeadersHeaders.cs | 8 + .../Models/ModeCopyMode.Serialization.cs | 28 + .../src/Generated/Models/NfsFileType.cs | 54 ++ .../Models/OwnerCopyMode.Serialization.cs | 28 + .../src/Generated/ServiceRestClient.cs | 2 +- .../src/Generated/ShareRestClient.cs | 2 +- .../src/Models/FilePosixProperties.cs | 69 +++ .../src/Models/Internal/ModeCopyMode.cs | 14 + .../src/Models/Internal/OwnerCopyMode.cs | 14 + .../src/Models/NfsFileMode.cs | 205 +++++++ .../src/Models/RolePermissionExtensions.cs | 165 ++++++ .../src/Models/RolePermissions.cs | 34 ++ .../src/Models/ShareDirectoryCreateOptions.cs | 8 + .../src/Models/ShareDirectoryInfo.cs | 24 + .../src/Models/ShareDirectoryProperties.cs | 29 + .../ShareDirectorySetHttpHeadersOptions.cs | 6 + .../src/Models/ShareFileCopyOptions.cs | 5 + .../src/Models/ShareFileCreateOptions.cs | 7 + .../ShareFileCreateSymbolicLinkOptions.cs | 46 ++ .../src/Models/ShareFileDownloadDetails.cs | 63 ++- .../src/Models/ShareFileInfo.cs | 51 ++ .../src/Models/ShareFileProperties.cs | 55 ++ .../Models/ShareFileSetHttpHeadersOptions.cs | 6 + .../src/Models/ShareFileSymbolicLinkInfo.cs | 50 ++ .../src/ShareClient.cs | 4 + .../src/ShareDirectoryClient.cs | 79 ++- .../src/ShareExtensions.cs | 137 ++++- .../src/ShareFileClient.cs | 526 +++++++++++++++++- .../src/Shared/ShareModelExtensions.cs | 9 +- .../src/autorest.md | 2 +- .../tests/ClientBuilderExtensions.cs | 27 +- .../tests/DirectoryClientTests.cs | 93 ++++ .../tests/DisposingShare.cs | 8 +- .../tests/FileClientTests.cs | 355 ++++++++++++ .../tests/FileTestBase.cs | 3 + .../tests/NfsFileModeTests.cs | 50 ++ 51 files changed, 2920 insertions(+), 157 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateHardLinkHeaders.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateSymbolicLinkHeaders.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileGetSymbolicLinkHeaders.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/ModeCopyMode.Serialization.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/NfsFileType.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/OwnerCopyMode.Serialization.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Models/FilePosixProperties.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Models/Internal/ModeCopyMode.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Models/Internal/OwnerCopyMode.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Models/NfsFileMode.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissionExtensions.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissions.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateSymbolicLinkOptions.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSymbolicLinkInfo.cs create mode 100644 sdk/storage/Azure.Storage.Files.Shares/tests/NfsFileModeTests.cs diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs index 430c0b16d2ebe..5fc271b35181b 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs @@ -268,6 +268,8 @@ public ShareFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredent public virtual System.Threading.Tasks.Task> CreateAsync(long maxSize, Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders httpHeaders, System.Collections.Generic.IDictionary metadata, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties, string filePermission, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions, System.Threading.CancellationToken cancellationToken) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> CreateAsync(long maxSize, Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders httpHeaders, System.Collections.Generic.IDictionary metadata, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties, string filePermission, System.Threading.CancellationToken cancellationToken) { throw null; } + public virtual Azure.Response CreateHardLink(string targetFile, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> CreateHardLinkAsync(string targetFile, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response Delete(Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Delete(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -495,6 +497,15 @@ public enum FilePermissionFormat Sddl = 0, Binary = 1, } + public partial class FilePosixProperties + { + public FilePosixProperties() { } + public Azure.Storage.Files.Shares.Models.NfsFileMode FileMode { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.NfsFileType? FileType { get { throw null; } } + public string Group { get { throw null; } set { } } + public long? LinkCount { get { throw null; } } + public string Owner { get { throw null; } set { } } + } public partial class FileSmbProperties { public FileSmbProperties() { } @@ -512,16 +523,59 @@ public FileSmbProperties() { } } public static partial class FilesModelFactory { + public static Azure.Storage.Files.Shares.Models.FilePosixProperties FileNfsProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileItem ShareFileItem(bool isDirectory = false, string name = null, long? fileSize = default(long?), string id = null, Azure.Storage.Files.Shares.Models.ShareFileItemProperties properties = null, Azure.Storage.Files.Shares.Models.NtfsFileAttributes? fileAttributes = default(Azure.Storage.Files.Shares.Models.NtfsFileAttributes?), string permissionKey = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata, Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileDownloadInfo StorageFileDownloadInfo(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletionTime = default(System.DateTimeOffset), string copyStatusDescription = null, string contentDisposition = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServerEncrypted = false, string cacheControl = null, string fileAttributes = null, System.Collections.Generic.IEnumerable contentEncoding = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), byte[] contentHash = null, System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string contentRange = null, string filePermissionKey = null, string contentType = null, string fileId = null, long contentLength = (long)0, string fileParentId = null, System.Collections.Generic.IDictionary metadata = null, System.IO.Stream content = null, string copyId = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentType, string contentRange, Azure.ETag eTag, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, string acceptRanges, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, byte[] fileContentHash, bool isServiceEncrypted) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, string filePermissionKey = null, string fileAttributes = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string fileId = null, string fileParentId = null, Azure.Storage.Files.Shares.Models.NfsFileMode nfsFileMode = null, string owner = null, string group = null, Azure.Storage.Files.Shares.Models.NfsFileType nfsFileType = default(Azure.Storage.Files.Shares.Models.NfsFileType)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileItem StorageFileItem(bool isDirectory, string name, long? fileSize) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, Azure.Storage.Files.Shares.Models.NtfsFileAttributes fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } } + public partial class NfsFileMode + { + public NfsFileMode() { } + public bool EffectiveGroupIdentity { get { throw null; } set { } } + public bool EffectiveUserIdentity { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Group { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Other { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Owner { get { throw null; } set { } } + public bool StickyBit { get { throw null; } set { } } + public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseOctalFileMode(string modeString) { throw null; } + public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseSymbolicFileMode(string modeString) { throw null; } + public string ToOctalFileMode() { throw null; } + public string ToSymbolicFileMode() { throw null; } + } + [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] + public readonly partial struct NfsFileType : System.IEquatable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public NfsFileType(string value) { throw null; } + public static Azure.Storage.Files.Shares.Models.NfsFileType Directory { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType Regular { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType Symlink { get { throw null; } } + public bool Equals(Azure.Storage.Files.Shares.Models.NfsFileType other) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override bool Equals(object obj) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override int GetHashCode() { throw null; } + public static bool operator ==(Azure.Storage.Files.Shares.Models.NfsFileType left, Azure.Storage.Files.Shares.Models.NfsFileType right) { throw null; } + public static implicit operator Azure.Storage.Files.Shares.Models.NfsFileType (string value) { throw null; } + public static bool operator !=(Azure.Storage.Files.Shares.Models.NfsFileType left, Azure.Storage.Files.Shares.Models.NfsFileType right) { throw null; } + public override string ToString() { throw null; } + } [System.FlagsAttribute] public enum NtfsFileAttributes { @@ -546,6 +600,14 @@ public partial class PermissionInfo internal PermissionInfo() { } public string FilePermissionKey { get { throw null; } } } + [System.FlagsAttribute] + public enum RolePermissions + { + None = 0, + Execute = 1, + Write = 2, + Read = 4, + } public partial class ShareAccessPolicy { public ShareAccessPolicy() { } @@ -630,6 +692,7 @@ public partial class ShareDirectoryCreateOptions public ShareDirectoryCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryGetFilesAndDirectoriesOptions @@ -644,6 +707,7 @@ public partial class ShareDirectoryInfo internal ShareDirectoryInfo() { } public Azure.ETag ETag { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryProperties @@ -653,12 +717,14 @@ internal ShareDirectoryProperties() { } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectorySetHttpHeadersOptions { public ShareDirectorySetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -768,6 +834,7 @@ public ShareFileCopyOptions() { } public Azure.Storage.Files.Shares.Models.PermissionCopyMode? FilePermissionCopyMode { get { throw null; } set { } } public bool? IgnoreReadOnly { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FilePermissionFormat? PermissionFormat { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.CopyableFileSmbProperties SmbPropertiesToCopy { get { throw null; } set { } } @@ -778,6 +845,7 @@ public ShareFileCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadDetails @@ -803,6 +871,7 @@ internal ShareFileDownloadDetails() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadInfo : System.IDisposable @@ -880,6 +949,7 @@ internal ShareFileInfo() { } public Azure.ETag ETag { get { throw null; } } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileItem @@ -967,6 +1037,7 @@ internal ShareFileProperties() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileRangeInfo @@ -1008,6 +1079,7 @@ public ShareFileSetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public long? NewSize { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.FlagsAttribute] @@ -1227,6 +1299,8 @@ public ShareSmbSettings() { } public static partial class SharesModelFactory { public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } } public partial class ShareSnapshotInfo diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs index 430c0b16d2ebe..5fc271b35181b 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs @@ -268,6 +268,8 @@ public ShareFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredent public virtual System.Threading.Tasks.Task> CreateAsync(long maxSize, Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders httpHeaders, System.Collections.Generic.IDictionary metadata, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties, string filePermission, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions, System.Threading.CancellationToken cancellationToken) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> CreateAsync(long maxSize, Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders httpHeaders, System.Collections.Generic.IDictionary metadata, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties, string filePermission, System.Threading.CancellationToken cancellationToken) { throw null; } + public virtual Azure.Response CreateHardLink(string targetFile, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> CreateHardLinkAsync(string targetFile, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response Delete(Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Delete(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -495,6 +497,15 @@ public enum FilePermissionFormat Sddl = 0, Binary = 1, } + public partial class FilePosixProperties + { + public FilePosixProperties() { } + public Azure.Storage.Files.Shares.Models.NfsFileMode FileMode { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.NfsFileType? FileType { get { throw null; } } + public string Group { get { throw null; } set { } } + public long? LinkCount { get { throw null; } } + public string Owner { get { throw null; } set { } } + } public partial class FileSmbProperties { public FileSmbProperties() { } @@ -512,16 +523,59 @@ public FileSmbProperties() { } } public static partial class FilesModelFactory { + public static Azure.Storage.Files.Shares.Models.FilePosixProperties FileNfsProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileItem ShareFileItem(bool isDirectory = false, string name = null, long? fileSize = default(long?), string id = null, Azure.Storage.Files.Shares.Models.ShareFileItemProperties properties = null, Azure.Storage.Files.Shares.Models.NtfsFileAttributes? fileAttributes = default(Azure.Storage.Files.Shares.Models.NtfsFileAttributes?), string permissionKey = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata, Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileDownloadInfo StorageFileDownloadInfo(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletionTime = default(System.DateTimeOffset), string copyStatusDescription = null, string contentDisposition = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServerEncrypted = false, string cacheControl = null, string fileAttributes = null, System.Collections.Generic.IEnumerable contentEncoding = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), byte[] contentHash = null, System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string contentRange = null, string filePermissionKey = null, string contentType = null, string fileId = null, long contentLength = (long)0, string fileParentId = null, System.Collections.Generic.IDictionary metadata = null, System.IO.Stream content = null, string copyId = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentType, string contentRange, Azure.ETag eTag, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, string acceptRanges, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, byte[] fileContentHash, bool isServiceEncrypted) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, string filePermissionKey = null, string fileAttributes = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string fileId = null, string fileParentId = null, Azure.Storage.Files.Shares.Models.NfsFileMode nfsFileMode = null, string owner = null, string group = null, Azure.Storage.Files.Shares.Models.NfsFileType nfsFileType = default(Azure.Storage.Files.Shares.Models.NfsFileType)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileItem StorageFileItem(bool isDirectory, string name, long? fileSize) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, Azure.Storage.Files.Shares.Models.NtfsFileAttributes fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } } + public partial class NfsFileMode + { + public NfsFileMode() { } + public bool EffectiveGroupIdentity { get { throw null; } set { } } + public bool EffectiveUserIdentity { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Group { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Other { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Owner { get { throw null; } set { } } + public bool StickyBit { get { throw null; } set { } } + public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseOctalFileMode(string modeString) { throw null; } + public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseSymbolicFileMode(string modeString) { throw null; } + public string ToOctalFileMode() { throw null; } + public string ToSymbolicFileMode() { throw null; } + } + [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] + public readonly partial struct NfsFileType : System.IEquatable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public NfsFileType(string value) { throw null; } + public static Azure.Storage.Files.Shares.Models.NfsFileType Directory { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType Regular { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType Symlink { get { throw null; } } + public bool Equals(Azure.Storage.Files.Shares.Models.NfsFileType other) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override bool Equals(object obj) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override int GetHashCode() { throw null; } + public static bool operator ==(Azure.Storage.Files.Shares.Models.NfsFileType left, Azure.Storage.Files.Shares.Models.NfsFileType right) { throw null; } + public static implicit operator Azure.Storage.Files.Shares.Models.NfsFileType (string value) { throw null; } + public static bool operator !=(Azure.Storage.Files.Shares.Models.NfsFileType left, Azure.Storage.Files.Shares.Models.NfsFileType right) { throw null; } + public override string ToString() { throw null; } + } [System.FlagsAttribute] public enum NtfsFileAttributes { @@ -546,6 +600,14 @@ public partial class PermissionInfo internal PermissionInfo() { } public string FilePermissionKey { get { throw null; } } } + [System.FlagsAttribute] + public enum RolePermissions + { + None = 0, + Execute = 1, + Write = 2, + Read = 4, + } public partial class ShareAccessPolicy { public ShareAccessPolicy() { } @@ -630,6 +692,7 @@ public partial class ShareDirectoryCreateOptions public ShareDirectoryCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryGetFilesAndDirectoriesOptions @@ -644,6 +707,7 @@ public partial class ShareDirectoryInfo internal ShareDirectoryInfo() { } public Azure.ETag ETag { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryProperties @@ -653,12 +717,14 @@ internal ShareDirectoryProperties() { } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectorySetHttpHeadersOptions { public ShareDirectorySetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -768,6 +834,7 @@ public ShareFileCopyOptions() { } public Azure.Storage.Files.Shares.Models.PermissionCopyMode? FilePermissionCopyMode { get { throw null; } set { } } public bool? IgnoreReadOnly { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FilePermissionFormat? PermissionFormat { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.CopyableFileSmbProperties SmbPropertiesToCopy { get { throw null; } set { } } @@ -778,6 +845,7 @@ public ShareFileCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadDetails @@ -803,6 +871,7 @@ internal ShareFileDownloadDetails() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadInfo : System.IDisposable @@ -880,6 +949,7 @@ internal ShareFileInfo() { } public Azure.ETag ETag { get { throw null; } } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileItem @@ -967,6 +1037,7 @@ internal ShareFileProperties() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileRangeInfo @@ -1008,6 +1079,7 @@ public ShareFileSetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public long? NewSize { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.FlagsAttribute] @@ -1227,6 +1299,8 @@ public ShareSmbSettings() { } public static partial class SharesModelFactory { public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } } public partial class ShareSnapshotInfo diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index c2b5c3d31e6a2..be24571a39a8d 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_df67d82d59" + "Tag": "net/storage/Azure.Storage.Files.Shares_0600fdfad4" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryCreateHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryCreateHeaders.cs index 2893eee8248cc..2eaa2796f382f 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryCreateHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryCreateHeaders.cs @@ -7,6 +7,7 @@ using System; using Azure.Core; +using Azure.Storage.Files.Shares.Models; namespace Azure.Storage.Files.Shares { @@ -37,5 +38,13 @@ public DirectoryCreateHeaders(Response response) public string FileId => _response.Headers.TryGetValue("x-ms-file-id", out string value) ? value : null; /// The parent fileId of the directory. public string FileParentId => _response.Headers.TryGetValue("x-ms-file-parent-id", out string value) ? value : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; + /// NFS only. Type of the file or directory. + public NfsFileType? NfsFileType => _response.Headers.TryGetValue("x-ms-file-file-type", out string value) ? new NfsFileType(value) : (NfsFileType?)null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryGetPropertiesHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryGetPropertiesHeaders.cs index 43cac7bf49396..c7706aa600673 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryGetPropertiesHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryGetPropertiesHeaders.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using Azure.Core; +using Azure.Storage.Files.Shares.Models; namespace Azure.Storage.Files.Shares { @@ -40,5 +41,13 @@ public DirectoryGetPropertiesHeaders(Response response) public string FileId => _response.Headers.TryGetValue("x-ms-file-id", out string value) ? value : null; /// The parent fileId of the directory. public string FileParentId => _response.Headers.TryGetValue("x-ms-file-parent-id", out string value) ? value : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; + /// NFS only. Type of the file or directory. + public NfsFileType? NfsFileType => _response.Headers.TryGetValue("x-ms-file-file-type", out string value) ? new NfsFileType(value) : (NfsFileType?)null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryRestClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryRestClient.cs index 8a2edb8b99134..927f91370d88a 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryRestClient.cs @@ -33,7 +33,7 @@ internal partial class DirectoryRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, share, directory or file that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". + /// Specifies the version of the operation to use for this request. The default value is "2025-05-05". /// If true, the trailing dot will not be trimmed from the target URI. /// Valid value is backup. /// If true, the trailing dot will not be trimmed from the source URI. @@ -49,7 +49,7 @@ public DirectoryRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pip _allowSourceTrailingDot = allowSourceTrailingDot; } - internal HttpMessage CreateCreateRequest(string fileAttributes, int? timeout, IDictionary metadata, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string fileCreationTime, string fileLastWriteTime, string fileChangeTime) + internal HttpMessage CreateCreateRequest(int? timeout, IDictionary metadata, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string fileAttributes, string fileCreationTime, string fileLastWriteTime, string fileChangeTime, string owner, string group, string fileMode) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -83,7 +83,10 @@ internal HttpMessage CreateCreateRequest(string fileAttributes, int? timeout, ID { request.Headers.Add("x-ms-file-permission-key", filePermissionKey); } - request.Headers.Add("x-ms-file-attributes", fileAttributes); + if (fileAttributes != null) + { + request.Headers.Add("x-ms-file-attributes", fileAttributes); + } if (fileCreationTime != null) { request.Headers.Add("x-ms-file-creation-time", fileCreationTime); @@ -100,30 +103,39 @@ internal HttpMessage CreateCreateRequest(string fileAttributes, int? timeout, ID { request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); } + if (owner != null) + { + request.Headers.Add("x-ms-owner", owner); + } + if (group != null) + { + request.Headers.Add("x-ms-group", group); + } + if (fileMode != null) + { + request.Headers.Add("x-ms-mode", fileMode); + } request.Headers.Add("Accept", "application/xml"); return message; } /// Creates a new directory under the specified share or parent directory. - /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// A name-value pair to associate with a file storage object. /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// Creation time for the file/directory. Default value: Now. /// Last write time for the file/directory. Default value: Now. /// Change time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. /// The cancellation token to use. - /// is null. - public async Task> CreateAsync(string fileAttributes, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, CancellationToken cancellationToken = default) + public async Task> CreateAsync(int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileAttributes = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, string owner = null, string group = null, string fileMode = null, CancellationToken cancellationToken = default) { - if (fileAttributes == null) - { - throw new ArgumentNullException(nameof(fileAttributes)); - } - - using var message = CreateCreateRequest(fileAttributes, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, fileCreationTime, fileLastWriteTime, fileChangeTime); + using var message = CreateCreateRequest(timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, fileAttributes, fileCreationTime, fileLastWriteTime, fileChangeTime, owner, group, fileMode); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new DirectoryCreateHeaders(message.Response); switch (message.Response.Status) @@ -136,25 +148,22 @@ public async Task> CreateAsync(strin } /// Creates a new directory under the specified share or parent directory. - /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// A name-value pair to associate with a file storage object. /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// Creation time for the file/directory. Default value: Now. /// Last write time for the file/directory. Default value: Now. /// Change time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. /// The cancellation token to use. - /// is null. - public ResponseWithHeaders Create(string fileAttributes, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders Create(int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileAttributes = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, string owner = null, string group = null, string fileMode = null, CancellationToken cancellationToken = default) { - if (fileAttributes == null) - { - throw new ArgumentNullException(nameof(fileAttributes)); - } - - using var message = CreateCreateRequest(fileAttributes, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, fileCreationTime, fileLastWriteTime, fileChangeTime); + using var message = CreateCreateRequest(timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, fileAttributes, fileCreationTime, fileLastWriteTime, fileChangeTime, owner, group, fileMode); _pipeline.Send(message, cancellationToken); var headers = new DirectoryCreateHeaders(message.Response); switch (message.Response.Status) @@ -292,7 +301,7 @@ public ResponseWithHeaders Delete(int? timeout = null, C } } - internal HttpMessage CreateSetPropertiesRequest(string fileAttributes, int? timeout, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string fileCreationTime, string fileLastWriteTime, string fileChangeTime) + internal HttpMessage CreateSetPropertiesRequest(int? timeout, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string fileAttributes, string fileCreationTime, string fileLastWriteTime, string fileChangeTime, string owner, string group, string fileMode) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -319,7 +328,10 @@ internal HttpMessage CreateSetPropertiesRequest(string fileAttributes, int? time { request.Headers.Add("x-ms-file-permission-key", filePermissionKey); } - request.Headers.Add("x-ms-file-attributes", fileAttributes); + if (fileAttributes != null) + { + request.Headers.Add("x-ms-file-attributes", fileAttributes); + } if (fileCreationTime != null) { request.Headers.Add("x-ms-file-creation-time", fileCreationTime); @@ -340,29 +352,38 @@ internal HttpMessage CreateSetPropertiesRequest(string fileAttributes, int? time { request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); } + if (owner != null) + { + request.Headers.Add("x-ms-owner", owner); + } + if (group != null) + { + request.Headers.Add("x-ms-group", group); + } + if (fileMode != null) + { + request.Headers.Add("x-ms-mode", fileMode); + } request.Headers.Add("Accept", "application/xml"); return message; } /// Sets properties on the directory. - /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// Creation time for the file/directory. Default value: Now. /// Last write time for the file/directory. Default value: Now. /// Change time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. /// The cancellation token to use. - /// is null. - public async Task> SetPropertiesAsync(string fileAttributes, int? timeout = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, CancellationToken cancellationToken = default) + public async Task> SetPropertiesAsync(int? timeout = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileAttributes = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, string owner = null, string group = null, string fileMode = null, CancellationToken cancellationToken = default) { - if (fileAttributes == null) - { - throw new ArgumentNullException(nameof(fileAttributes)); - } - - using var message = CreateSetPropertiesRequest(fileAttributes, timeout, filePermission, filePermissionFormat, filePermissionKey, fileCreationTime, fileLastWriteTime, fileChangeTime); + using var message = CreateSetPropertiesRequest(timeout, filePermission, filePermissionFormat, filePermissionKey, fileAttributes, fileCreationTime, fileLastWriteTime, fileChangeTime, owner, group, fileMode); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new DirectorySetPropertiesHeaders(message.Response); switch (message.Response.Status) @@ -375,24 +396,21 @@ public async Task> SetPropert } /// Sets properties on the directory. - /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// Creation time for the file/directory. Default value: Now. /// Last write time for the file/directory. Default value: Now. /// Change time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. /// The cancellation token to use. - /// is null. - public ResponseWithHeaders SetProperties(string fileAttributes, int? timeout = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders SetProperties(int? timeout = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileAttributes = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, string owner = null, string group = null, string fileMode = null, CancellationToken cancellationToken = default) { - if (fileAttributes == null) - { - throw new ArgumentNullException(nameof(fileAttributes)); - } - - using var message = CreateSetPropertiesRequest(fileAttributes, timeout, filePermission, filePermissionFormat, filePermissionKey, fileCreationTime, fileLastWriteTime, fileChangeTime); + using var message = CreateSetPropertiesRequest(timeout, filePermission, filePermissionFormat, filePermissionKey, fileAttributes, fileCreationTime, fileLastWriteTime, fileChangeTime, owner, group, fileMode); _pipeline.Send(message, cancellationToken); var headers = new DirectorySetPropertiesHeaders(message.Response); switch (message.Response.Status) diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectorySetPropertiesHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectorySetPropertiesHeaders.cs index 122579967f528..70c784d9fcd7c 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectorySetPropertiesHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectorySetPropertiesHeaders.cs @@ -37,5 +37,11 @@ public DirectorySetPropertiesHeaders(Response response) public string FileId => _response.Headers.TryGetValue("x-ms-file-id", out string value) ? value : null; /// The parent fileId of the directory. public string FileParentId => _response.Headers.TryGetValue("x-ms-file-parent-id", out string value) ? value : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateHardLinkHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateHardLinkHeaders.cs new file mode 100644 index 0000000000000..0760e0e32c2a7 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateHardLinkHeaders.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; +using Azure.Core; +using Azure.Storage.Files.Shares.Models; + +namespace Azure.Storage.Files.Shares +{ + internal partial class FileCreateHardLinkHeaders + { + private readonly Response _response; + public FileCreateHardLinkHeaders(Response response) + { + _response = response; + } + /// Returns the date and time the share was last modified. Any operation that modifies the directory or its properties updates the last modified time. Operations on files do not affect the last modified time of the directory. + public DateTimeOffset? LastModified => _response.Headers.TryGetValue("Last-Modified", out DateTimeOffset? value) ? value : null; + /// Indicates the version of the File service used to execute the request. + public string Version => _response.Headers.TryGetValue("x-ms-version", out string value) ? value : null; + /// Creation time for the file. + public DateTimeOffset? FileCreationTime => _response.Headers.TryGetValue("x-ms-file-creation-time", out DateTimeOffset? value) ? value : null; + /// Last write time for the file. + public DateTimeOffset? FileLastWriteTime => _response.Headers.TryGetValue("x-ms-file-last-write-time", out DateTimeOffset? value) ? value : null; + /// Change time for the file. + public DateTimeOffset? FileChangeTime => _response.Headers.TryGetValue("x-ms-file-change-time", out DateTimeOffset? value) ? value : null; + /// The fileId of the file. + public string FileId => _response.Headers.TryGetValue("x-ms-file-id", out string value) ? value : null; + /// The parent fileId of the directory. + public string FileParentId => _response.Headers.TryGetValue("x-ms-file-parent-id", out string value) ? value : null; + /// NFS only. The link count of the file or directory. + public long? LinkCount => _response.Headers.TryGetValue("x-ms-link-count", out long? value) ? value : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; + /// NFS only. Type of the file or directory. + public NfsFileType? NfsFileType => _response.Headers.TryGetValue("x-ms-file-file-type", out string value) ? new NfsFileType(value) : (NfsFileType?)null; + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateHeaders.cs index 69e1975f86c7b..5a28bfff5489e 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateHeaders.cs @@ -7,6 +7,7 @@ using System; using Azure.Core; +using Azure.Storage.Files.Shares.Models; namespace Azure.Storage.Files.Shares { @@ -37,5 +38,13 @@ public FileCreateHeaders(Response response) public string FileId => _response.Headers.TryGetValue("x-ms-file-id", out string value) ? value : null; /// The parent fileId of the file. public string FileParentId => _response.Headers.TryGetValue("x-ms-file-parent-id", out string value) ? value : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; + /// NFS only. Type of the file or directory. + public NfsFileType? NfsFileType => _response.Headers.TryGetValue("x-ms-file-file-type", out string value) ? new NfsFileType(value) : (NfsFileType?)null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateSymbolicLinkHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateSymbolicLinkHeaders.cs new file mode 100644 index 0000000000000..bda9480cdb734 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileCreateSymbolicLinkHeaders.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; +using Azure.Core; +using Azure.Storage.Files.Shares.Models; + +namespace Azure.Storage.Files.Shares +{ + internal partial class FileCreateSymbolicLinkHeaders + { + private readonly Response _response; + public FileCreateSymbolicLinkHeaders(Response response) + { + _response = response; + } + /// Returns the date and time the share was last modified. Any operation that modifies the directory or its properties updates the last modified time. Operations on files do not affect the last modified time of the directory. + public DateTimeOffset? LastModified => _response.Headers.TryGetValue("Last-Modified", out DateTimeOffset? value) ? value : null; + /// Indicates the version of the File service used to execute the request. + public string Version => _response.Headers.TryGetValue("x-ms-version", out string value) ? value : null; + /// Creation time for the file. + public DateTimeOffset? FileCreationTime => _response.Headers.TryGetValue("x-ms-file-creation-time", out DateTimeOffset? value) ? value : null; + /// Last write time for the file. + public DateTimeOffset? FileLastWriteTime => _response.Headers.TryGetValue("x-ms-file-last-write-time", out DateTimeOffset? value) ? value : null; + /// Change time for the file. + public DateTimeOffset? FileChangeTime => _response.Headers.TryGetValue("x-ms-file-change-time", out DateTimeOffset? value) ? value : null; + /// The fileId of the file. + public string FileId => _response.Headers.TryGetValue("x-ms-file-id", out string value) ? value : null; + /// The parent fileId of the directory. + public string FileParentId => _response.Headers.TryGetValue("x-ms-file-parent-id", out string value) ? value : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; + /// NFS only. Type of the file or directory. + public NfsFileType? NfsFileType => _response.Headers.TryGetValue("x-ms-file-file-type", out string value) ? new NfsFileType(value) : (NfsFileType?)null; + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDeleteHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDeleteHeaders.cs index b1db3dbf3963c..131cec8d9c146 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDeleteHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDeleteHeaders.cs @@ -18,5 +18,7 @@ public FileDeleteHeaders(Response response) } /// Indicates the version of the File service used to execute the request. public string Version => _response.Headers.TryGetValue("x-ms-version", out string value) ? value : null; + /// NFS only. The link count of the file or directory. + public long? LinkCount => _response.Headers.TryGetValue("x-ms-link-count", out long? value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDownloadHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDownloadHeaders.cs index c4d7056a5cfa3..174d478602c24 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDownloadHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDownloadHeaders.cs @@ -83,5 +83,13 @@ public FileDownloadHeaders(Response response) public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; /// The length of the blob/file content inside the message body when the response body is returned as a structured message. Will always be smaller than Content-Length. public long? StructuredContentLength => _response.Headers.TryGetValue("x-ms-structured-content-length", out long? value) ? value : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; + /// NFS only. The link count of the file or directory. + public long? LinkCount => _response.Headers.TryGetValue("x-ms-link-count", out long? value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileGetPropertiesHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileGetPropertiesHeaders.cs index 4e2050c13c06d..29e642249adda 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileGetPropertiesHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileGetPropertiesHeaders.cs @@ -75,5 +75,15 @@ public FileGetPropertiesHeaders(Response response) public ShareLeaseState? LeaseState => _response.Headers.TryGetValue("x-ms-lease-state", out string value) ? value.ToShareLeaseState() : null; /// The current lease status of the file. public ShareLeaseStatus? LeaseStatus => _response.Headers.TryGetValue("x-ms-lease-status", out string value) ? value.ToShareLeaseStatus() : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; + /// NFS only. The link count of the file or directory. + public long? LinkCount => _response.Headers.TryGetValue("x-ms-link-count", out long? value) ? value : null; + /// NFS only. Type of the file or directory. + public NfsFileType? NfsFileType => _response.Headers.TryGetValue("x-ms-file-file-type", out string value) ? new NfsFileType(value) : (NfsFileType?)null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileGetSymbolicLinkHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileGetSymbolicLinkHeaders.cs new file mode 100644 index 0000000000000..255a50bc98f9e --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileGetSymbolicLinkHeaders.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; +using Azure.Core; + +namespace Azure.Storage.Files.Shares +{ + internal partial class FileGetSymbolicLinkHeaders + { + private readonly Response _response; + public FileGetSymbolicLinkHeaders(Response response) + { + _response = response; + } + /// Returns the date and time the share was last modified. Any operation that modifies the directory or its properties updates the last modified time. Operations on files do not affect the last modified time of the directory. + public DateTimeOffset? LastModified => _response.Headers.TryGetValue("Last-Modified", out DateTimeOffset? value) ? value : null; + /// Indicates the version of the File service used to execute the request. + public string Version => _response.Headers.TryGetValue("x-ms-version", out string value) ? value : null; + /// The path to the original file, the symbolic link is pointing to. The path is of type string which is not resolved and is stored as is. The path can be absolute path or the relative path depending on the content stored in the symbolic link file. + public string LinkText => _response.Headers.TryGetValue("x-ms-link-text", out string value) ? value : null; + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileRestClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileRestClient.cs index 07f88af545aec..0800d2818937b 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileRestClient.cs @@ -34,7 +34,7 @@ internal partial class FileRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, share, directory or file that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". + /// Specifies the version of the operation to use for this request. The default value is "2025-05-05". /// Only update is supported: - Update: Writes the bytes downloaded from the source url into the specified range. The default value is "update". /// If true, the trailing dot will not be trimmed from the target URI. /// Valid value is backup. @@ -52,7 +52,7 @@ public FileRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline _allowSourceTrailingDot = allowSourceTrailingDot; } - internal HttpMessage CreateCreateRequest(long fileContentLength, string fileAttributes, int? timeout, IDictionary metadata, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string fileCreationTime, string fileLastWriteTime, string fileChangeTime, FileHttpHeaders fileHttpHeaders, ShareFileRequestConditions shareFileRequestConditions) + internal HttpMessage CreateCreateRequest(long fileContentLength, int? timeout, IDictionary metadata, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string fileAttributes, string fileCreationTime, string fileLastWriteTime, string fileChangeTime, string owner, string group, string fileMode, NfsFileType? nfsFileType, FileHttpHeaders fileHttpHeaders, ShareFileRequestConditions shareFileRequestConditions) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -111,7 +111,10 @@ internal HttpMessage CreateCreateRequest(long fileContentLength, string fileAttr { request.Headers.Add("x-ms-file-permission-key", filePermissionKey); } - request.Headers.Add("x-ms-file-attributes", fileAttributes); + if (fileAttributes != null) + { + request.Headers.Add("x-ms-file-attributes", fileAttributes); + } if (fileCreationTime != null) { request.Headers.Add("x-ms-file-creation-time", fileCreationTime); @@ -132,33 +135,47 @@ internal HttpMessage CreateCreateRequest(long fileContentLength, string fileAttr { request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); } + if (owner != null) + { + request.Headers.Add("x-ms-owner", owner); + } + if (group != null) + { + request.Headers.Add("x-ms-group", group); + } + if (fileMode != null) + { + request.Headers.Add("x-ms-mode", fileMode); + } + if (nfsFileType != null) + { + request.Headers.Add("x-ms-file-file-type", nfsFileType.Value.ToString()); + } request.Headers.Add("Accept", "application/xml"); return message; } /// Creates a new file or replaces a file. Note it only initializes the file with no content. /// Specifies the maximum size for the file, up to 4 TB. - /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// A name-value pair to associate with a file storage object. /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// Creation time for the file/directory. Default value: Now. /// Last write time for the file/directory. Default value: Now. /// Change time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. + /// Optional, NFS only. Type of the file or directory. /// Parameter group. /// Parameter group. /// The cancellation token to use. - /// is null. - public async Task> CreateAsync(long fileContentLength, string fileAttributes, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, FileHttpHeaders fileHttpHeaders = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public async Task> CreateAsync(long fileContentLength, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileAttributes = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, string owner = null, string group = null, string fileMode = null, NfsFileType? nfsFileType = null, FileHttpHeaders fileHttpHeaders = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { - if (fileAttributes == null) - { - throw new ArgumentNullException(nameof(fileAttributes)); - } - - using var message = CreateCreateRequest(fileContentLength, fileAttributes, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, fileCreationTime, fileLastWriteTime, fileChangeTime, fileHttpHeaders, shareFileRequestConditions); + using var message = CreateCreateRequest(fileContentLength, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, fileAttributes, fileCreationTime, fileLastWriteTime, fileChangeTime, owner, group, fileMode, nfsFileType, fileHttpHeaders, shareFileRequestConditions); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new FileCreateHeaders(message.Response); switch (message.Response.Status) @@ -172,27 +189,25 @@ public async Task> CreateAsync(long fileC /// Creates a new file or replaces a file. Note it only initializes the file with no content. /// Specifies the maximum size for the file, up to 4 TB. - /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// A name-value pair to associate with a file storage object. /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// Creation time for the file/directory. Default value: Now. /// Last write time for the file/directory. Default value: Now. /// Change time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. + /// Optional, NFS only. Type of the file or directory. /// Parameter group. /// Parameter group. /// The cancellation token to use. - /// is null. - public ResponseWithHeaders Create(long fileContentLength, string fileAttributes, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, FileHttpHeaders fileHttpHeaders = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders Create(long fileContentLength, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileAttributes = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, string owner = null, string group = null, string fileMode = null, NfsFileType? nfsFileType = null, FileHttpHeaders fileHttpHeaders = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { - if (fileAttributes == null) - { - throw new ArgumentNullException(nameof(fileAttributes)); - } - - using var message = CreateCreateRequest(fileContentLength, fileAttributes, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, fileCreationTime, fileLastWriteTime, fileChangeTime, fileHttpHeaders, shareFileRequestConditions); + using var message = CreateCreateRequest(fileContentLength, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, fileAttributes, fileCreationTime, fileLastWriteTime, fileChangeTime, owner, group, fileMode, nfsFileType, fileHttpHeaders, shareFileRequestConditions); _pipeline.Send(message, cancellationToken); var headers = new FileCreateHeaders(message.Response); switch (message.Response.Status) @@ -432,7 +447,7 @@ public ResponseWithHeaders Delete(int? timeout = null, ShareF } } - internal HttpMessage CreateSetHttpHeadersRequest(string fileAttributes, int? timeout, long? fileContentLength, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string fileCreationTime, string fileLastWriteTime, string fileChangeTime, FileHttpHeaders fileHttpHeaders, ShareFileRequestConditions shareFileRequestConditions) + internal HttpMessage CreateSetHttpHeadersRequest(int? timeout, long? fileContentLength, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string fileAttributes, string fileCreationTime, string fileLastWriteTime, string fileChangeTime, string owner, string group, string fileMode, FileHttpHeaders fileHttpHeaders, ShareFileRequestConditions shareFileRequestConditions) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -486,7 +501,10 @@ internal HttpMessage CreateSetHttpHeadersRequest(string fileAttributes, int? tim { request.Headers.Add("x-ms-file-permission-key", filePermissionKey); } - request.Headers.Add("x-ms-file-attributes", fileAttributes); + if (fileAttributes != null) + { + request.Headers.Add("x-ms-file-attributes", fileAttributes); + } if (fileCreationTime != null) { request.Headers.Add("x-ms-file-creation-time", fileCreationTime); @@ -511,32 +529,41 @@ internal HttpMessage CreateSetHttpHeadersRequest(string fileAttributes, int? tim { request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); } + if (owner != null) + { + request.Headers.Add("x-ms-owner", owner); + } + if (group != null) + { + request.Headers.Add("x-ms-group", group); + } + if (fileMode != null) + { + request.Headers.Add("x-ms-mode", fileMode); + } request.Headers.Add("Accept", "application/xml"); return message; } /// Sets HTTP headers on the file. - /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// Resizes a file to the specified size. If the specified byte value is less than the current size of the file, then all ranges above the specified byte value are cleared. /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// Creation time for the file/directory. Default value: Now. /// Last write time for the file/directory. Default value: Now. /// Change time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. /// Parameter group. /// Parameter group. /// The cancellation token to use. - /// is null. - public async Task> SetHttpHeadersAsync(string fileAttributes, int? timeout = null, long? fileContentLength = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, FileHttpHeaders fileHttpHeaders = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public async Task> SetHttpHeadersAsync(int? timeout = null, long? fileContentLength = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileAttributes = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, string owner = null, string group = null, string fileMode = null, FileHttpHeaders fileHttpHeaders = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { - if (fileAttributes == null) - { - throw new ArgumentNullException(nameof(fileAttributes)); - } - - using var message = CreateSetHttpHeadersRequest(fileAttributes, timeout, fileContentLength, filePermission, filePermissionFormat, filePermissionKey, fileCreationTime, fileLastWriteTime, fileChangeTime, fileHttpHeaders, shareFileRequestConditions); + using var message = CreateSetHttpHeadersRequest(timeout, fileContentLength, filePermission, filePermissionFormat, filePermissionKey, fileAttributes, fileCreationTime, fileLastWriteTime, fileChangeTime, owner, group, fileMode, fileHttpHeaders, shareFileRequestConditions); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new FileSetHttpHeadersHeaders(message.Response); switch (message.Response.Status) @@ -549,27 +576,24 @@ public async Task> SetHttpHeaders } /// Sets HTTP headers on the file. - /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// Resizes a file to the specified size. If the specified byte value is less than the current size of the file, then all ranges above the specified byte value are cleared. /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// If specified, the provided file attributes shall be set. Default value: ‘Archive’ for file and ‘Directory’ for directory. ‘None’ can also be specified as default. /// Creation time for the file/directory. Default value: Now. /// Last write time for the file/directory. Default value: Now. /// Change time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. /// Parameter group. /// Parameter group. /// The cancellation token to use. - /// is null. - public ResponseWithHeaders SetHttpHeaders(string fileAttributes, int? timeout = null, long? fileContentLength = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, FileHttpHeaders fileHttpHeaders = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders SetHttpHeaders(int? timeout = null, long? fileContentLength = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string fileAttributes = null, string fileCreationTime = null, string fileLastWriteTime = null, string fileChangeTime = null, string owner = null, string group = null, string fileMode = null, FileHttpHeaders fileHttpHeaders = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { - if (fileAttributes == null) - { - throw new ArgumentNullException(nameof(fileAttributes)); - } - - using var message = CreateSetHttpHeadersRequest(fileAttributes, timeout, fileContentLength, filePermission, filePermissionFormat, filePermissionKey, fileCreationTime, fileLastWriteTime, fileChangeTime, fileHttpHeaders, shareFileRequestConditions); + using var message = CreateSetHttpHeadersRequest(timeout, fileContentLength, filePermission, filePermissionFormat, filePermissionKey, fileAttributes, fileCreationTime, fileLastWriteTime, fileChangeTime, owner, group, fileMode, fileHttpHeaders, shareFileRequestConditions); _pipeline.Send(message, cancellationToken); var headers = new FileSetHttpHeadersHeaders(message.Response); switch (message.Response.Status) @@ -1308,7 +1332,7 @@ public ResponseWithHeaders GetRange } } - internal HttpMessage CreateStartCopyRequest(string copySource, int? timeout, IDictionary metadata, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, CopyFileSmbInfo copyFileSmbInfo, ShareFileRequestConditions shareFileRequestConditions) + internal HttpMessage CreateStartCopyRequest(string copySource, int? timeout, IDictionary metadata, string filePermission, FilePermissionFormat? filePermissionFormat, string filePermissionKey, string owner, string group, string fileMode, ModeCopyMode? fileModeCopyMode, OwnerCopyMode? fileOwnerCopyMode, CopyFileSmbInfo copyFileSmbInfo, ShareFileRequestConditions shareFileRequestConditions) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -1382,6 +1406,26 @@ internal HttpMessage CreateStartCopyRequest(string copySource, int? timeout, IDi { request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); } + if (owner != null) + { + request.Headers.Add("x-ms-owner", owner); + } + if (group != null) + { + request.Headers.Add("x-ms-group", group); + } + if (fileMode != null) + { + request.Headers.Add("x-ms-mode", fileMode); + } + if (fileModeCopyMode != null) + { + request.Headers.Add("x-ms-file-mode-copy-mode", fileModeCopyMode.Value.ToSerialString()); + } + if (fileOwnerCopyMode != null) + { + request.Headers.Add("x-ms-file-owner-copy-mode", fileOwnerCopyMode.Value.ToSerialString()); + } request.Headers.Add("Accept", "application/xml"); return message; } @@ -1393,18 +1437,23 @@ internal HttpMessage CreateStartCopyRequest(string copySource, int? timeout, IDi /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. + /// NFS only. Applicable only when the copy source is a File. Determines the copy behavior of the mode bits of the file. source: The mode on the destination file is copied from the source file. override: The mode on the destination file is determined via the x-ms-mode header. + /// NFS only. Determines the copy behavior of the owner user identifier (UID) and group identifier (GID) of the file. source: The owner user identifier (UID) and group identifier (GID) on the destination file is copied from the source file. override: The owner user identifier (UID) and group identifier (GID) on the destination file is determined via the x-ms-owner and x-ms-group headers. /// Parameter group. /// Parameter group. /// The cancellation token to use. /// is null. - public async Task> StartCopyAsync(string copySource, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, CopyFileSmbInfo copyFileSmbInfo = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public async Task> StartCopyAsync(string copySource, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string owner = null, string group = null, string fileMode = null, ModeCopyMode? fileModeCopyMode = null, OwnerCopyMode? fileOwnerCopyMode = null, CopyFileSmbInfo copyFileSmbInfo = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { if (copySource == null) { throw new ArgumentNullException(nameof(copySource)); } - using var message = CreateStartCopyRequest(copySource, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, copyFileSmbInfo, shareFileRequestConditions); + using var message = CreateStartCopyRequest(copySource, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, owner, group, fileMode, fileModeCopyMode, fileOwnerCopyMode, copyFileSmbInfo, shareFileRequestConditions); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new FileStartCopyHeaders(message.Response); switch (message.Response.Status) @@ -1423,18 +1472,23 @@ public async Task> StartCopyAsync(stri /// If specified the permission (security descriptor) shall be set for the directory/file. This header can be used if Permission size is <= 8KB, else x-ms-file-permission-key header shall be used. Default value: Inherit. If SDDL is specified as input, it must have owner, group and dacl. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. /// Optional. Available for version 2023-06-01 and later. Specifies the format in which the permission is returned. Acceptable values are SDDL or binary. If x-ms-file-permission-format is unspecified or explicitly set to SDDL, the permission is returned in SDDL format. If x-ms-file-permission-format is explicitly set to binary, the permission is returned as a base64 string representing the binary encoding of the permission. /// Key of the permission to be set for the directory/file. Note: Only one of the x-ms-file-permission or x-ms-file-permission-key should be specified. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Optional, NFS only. The file mode of the file or directory. + /// NFS only. Applicable only when the copy source is a File. Determines the copy behavior of the mode bits of the file. source: The mode on the destination file is copied from the source file. override: The mode on the destination file is determined via the x-ms-mode header. + /// NFS only. Determines the copy behavior of the owner user identifier (UID) and group identifier (GID) of the file. source: The owner user identifier (UID) and group identifier (GID) on the destination file is copied from the source file. override: The owner user identifier (UID) and group identifier (GID) on the destination file is determined via the x-ms-owner and x-ms-group headers. /// Parameter group. /// Parameter group. /// The cancellation token to use. /// is null. - public ResponseWithHeaders StartCopy(string copySource, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, CopyFileSmbInfo copyFileSmbInfo = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders StartCopy(string copySource, int? timeout = null, IDictionary metadata = null, string filePermission = null, FilePermissionFormat? filePermissionFormat = null, string filePermissionKey = null, string owner = null, string group = null, string fileMode = null, ModeCopyMode? fileModeCopyMode = null, OwnerCopyMode? fileOwnerCopyMode = null, CopyFileSmbInfo copyFileSmbInfo = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { if (copySource == null) { throw new ArgumentNullException(nameof(copySource)); } - using var message = CreateStartCopyRequest(copySource, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, copyFileSmbInfo, shareFileRequestConditions); + using var message = CreateStartCopyRequest(copySource, timeout, metadata, filePermission, filePermissionFormat, filePermissionKey, owner, group, fileMode, fileModeCopyMode, fileOwnerCopyMode, copyFileSmbInfo, shareFileRequestConditions); _pipeline.Send(message, cancellationToken); var headers = new FileStartCopyHeaders(message.Response); switch (message.Response.Status) @@ -1859,5 +1913,250 @@ public ResponseWithHeaders Rename(string renameSource, int? t throw new RequestFailedException(message.Response); } } + + internal HttpMessage CreateCreateSymbolicLinkRequest(string linkText, int? timeout, IDictionary metadata, string fileCreationTime, string fileLastWriteTime, string owner, string group, ShareFileRequestConditions shareFileRequestConditions) + { + var message = _pipeline.CreateMessage(); + var request = message.Request; + request.Method = RequestMethod.Put; + var uri = new RawRequestUriBuilder(); + uri.AppendRaw(_url, false); + uri.AppendQuery("restype", "symboliclink", true); + if (timeout != null) + { + uri.AppendQuery("timeout", timeout.Value, true); + } + request.Uri = uri; + request.Headers.Add("x-ms-version", _version); + if (metadata != null) + { + request.Headers.Add("x-ms-meta-", metadata); + } + if (fileCreationTime != null) + { + request.Headers.Add("x-ms-file-creation-time", fileCreationTime); + } + if (fileLastWriteTime != null) + { + request.Headers.Add("x-ms-file-last-write-time", fileLastWriteTime); + } + if (shareFileRequestConditions?.LeaseId != null) + { + request.Headers.Add("x-ms-lease-id", shareFileRequestConditions.LeaseId); + } + if (owner != null) + { + request.Headers.Add("x-ms-owner", owner); + } + if (group != null) + { + request.Headers.Add("x-ms-group", group); + } + request.Headers.Add("x-ms-link-text", linkText); + if (_fileRequestIntent != null) + { + request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); + } + request.Headers.Add("Accept", "application/xml"); + return message; + } + + /// Creates a symbolic link. + /// NFS only. Required. The path to the original file, the symbolic link is pointing to. The path is of type string which is not resolved and is stored as is. The path can be absolute path or the relative path depending on the content stored in the symbolic link file. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. + /// A name-value pair to associate with a file storage object. + /// Creation time for the file/directory. Default value: Now. + /// Last write time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Parameter group. + /// The cancellation token to use. + /// is null. + public async Task> CreateSymbolicLinkAsync(string linkText, int? timeout = null, IDictionary metadata = null, string fileCreationTime = null, string fileLastWriteTime = null, string owner = null, string group = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + { + if (linkText == null) + { + throw new ArgumentNullException(nameof(linkText)); + } + + using var message = CreateCreateSymbolicLinkRequest(linkText, timeout, metadata, fileCreationTime, fileLastWriteTime, owner, group, shareFileRequestConditions); + await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); + var headers = new FileCreateSymbolicLinkHeaders(message.Response); + switch (message.Response.Status) + { + case 201: + return ResponseWithHeaders.FromValue(headers, message.Response); + default: + throw new RequestFailedException(message.Response); + } + } + + /// Creates a symbolic link. + /// NFS only. Required. The path to the original file, the symbolic link is pointing to. The path is of type string which is not resolved and is stored as is. The path can be absolute path or the relative path depending on the content stored in the symbolic link file. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. + /// A name-value pair to associate with a file storage object. + /// Creation time for the file/directory. Default value: Now. + /// Last write time for the file/directory. Default value: Now. + /// Optional, NFS only. The owner of the file or directory. + /// Optional, NFS only. The owning group of the file or directory. + /// Parameter group. + /// The cancellation token to use. + /// is null. + public ResponseWithHeaders CreateSymbolicLink(string linkText, int? timeout = null, IDictionary metadata = null, string fileCreationTime = null, string fileLastWriteTime = null, string owner = null, string group = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + { + if (linkText == null) + { + throw new ArgumentNullException(nameof(linkText)); + } + + using var message = CreateCreateSymbolicLinkRequest(linkText, timeout, metadata, fileCreationTime, fileLastWriteTime, owner, group, shareFileRequestConditions); + _pipeline.Send(message, cancellationToken); + var headers = new FileCreateSymbolicLinkHeaders(message.Response); + switch (message.Response.Status) + { + case 201: + return ResponseWithHeaders.FromValue(headers, message.Response); + default: + throw new RequestFailedException(message.Response); + } + } + + internal HttpMessage CreateGetSymbolicLinkRequest(int? timeout, string sharesnapshot) + { + var message = _pipeline.CreateMessage(); + var request = message.Request; + request.Method = RequestMethod.Get; + var uri = new RawRequestUriBuilder(); + uri.AppendRaw(_url, false); + uri.AppendQuery("restype", "symboliclink", true); + if (timeout != null) + { + uri.AppendQuery("timeout", timeout.Value, true); + } + if (sharesnapshot != null) + { + uri.AppendQuery("sharesnapshot", sharesnapshot, true); + } + request.Uri = uri; + request.Headers.Add("x-ms-version", _version); + if (_fileRequestIntent != null) + { + request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); + } + request.Headers.Add("Accept", "application/xml"); + return message; + } + + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. + /// The snapshot parameter is an opaque DateTime value that, when present, specifies the share snapshot to query. + /// The cancellation token to use. + public async Task> GetSymbolicLinkAsync(int? timeout = null, string sharesnapshot = null, CancellationToken cancellationToken = default) + { + using var message = CreateGetSymbolicLinkRequest(timeout, sharesnapshot); + await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); + var headers = new FileGetSymbolicLinkHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + return ResponseWithHeaders.FromValue(headers, message.Response); + default: + throw new RequestFailedException(message.Response); + } + } + + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. + /// The snapshot parameter is an opaque DateTime value that, when present, specifies the share snapshot to query. + /// The cancellation token to use. + public ResponseWithHeaders GetSymbolicLink(int? timeout = null, string sharesnapshot = null, CancellationToken cancellationToken = default) + { + using var message = CreateGetSymbolicLinkRequest(timeout, sharesnapshot); + _pipeline.Send(message, cancellationToken); + var headers = new FileGetSymbolicLinkHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + return ResponseWithHeaders.FromValue(headers, message.Response); + default: + throw new RequestFailedException(message.Response); + } + } + + internal HttpMessage CreateCreateHardLinkRequest(string targetFile, int? timeout, ShareFileRequestConditions shareFileRequestConditions) + { + var message = _pipeline.CreateMessage(); + var request = message.Request; + request.Method = RequestMethod.Put; + var uri = new RawRequestUriBuilder(); + uri.AppendRaw(_url, false); + uri.AppendQuery("restype", "hardlink", true); + if (timeout != null) + { + uri.AppendQuery("timeout", timeout.Value, true); + } + request.Uri = uri; + request.Headers.Add("x-ms-version", _version); + request.Headers.Add("x-ms-type", "file"); + if (shareFileRequestConditions?.LeaseId != null) + { + request.Headers.Add("x-ms-lease-id", shareFileRequestConditions.LeaseId); + } + request.Headers.Add("x-ms-file-target-file", targetFile); + if (_fileRequestIntent != null) + { + request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); + } + request.Headers.Add("Accept", "application/xml"); + return message; + } + + /// Creates a hard link. + /// NFS only. Required. Specifies the path of the target file to which the link will be created, up to 2 KiB in length. It should be full path of the target from the root.The target file must be in the same share and hence the same storage account. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. + /// Parameter group. + /// The cancellation token to use. + /// is null. + public async Task> CreateHardLinkAsync(string targetFile, int? timeout = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + { + if (targetFile == null) + { + throw new ArgumentNullException(nameof(targetFile)); + } + + using var message = CreateCreateHardLinkRequest(targetFile, timeout, shareFileRequestConditions); + await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); + var headers = new FileCreateHardLinkHeaders(message.Response); + switch (message.Response.Status) + { + case 201: + return ResponseWithHeaders.FromValue(headers, message.Response); + default: + throw new RequestFailedException(message.Response); + } + } + + /// Creates a hard link. + /// NFS only. Required. Specifies the path of the target file to which the link will be created, up to 2 KiB in length. It should be full path of the target from the root.The target file must be in the same share and hence the same storage account. + /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. + /// Parameter group. + /// The cancellation token to use. + /// is null. + public ResponseWithHeaders CreateHardLink(string targetFile, int? timeout = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + { + if (targetFile == null) + { + throw new ArgumentNullException(nameof(targetFile)); + } + + using var message = CreateCreateHardLinkRequest(targetFile, timeout, shareFileRequestConditions); + _pipeline.Send(message, cancellationToken); + var headers = new FileCreateHardLinkHeaders(message.Response); + switch (message.Response.Status) + { + case 201: + return ResponseWithHeaders.FromValue(headers, message.Response); + default: + throw new RequestFailedException(message.Response); + } + } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileSetHttpHeadersHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileSetHttpHeadersHeaders.cs index ff2030851669b..cb6c9cb38d062 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileSetHttpHeadersHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileSetHttpHeadersHeaders.cs @@ -37,5 +37,13 @@ public FileSetHttpHeadersHeaders(Response response) public string FileId => _response.Headers.TryGetValue("x-ms-file-id", out string value) ? value : null; /// The parent fileId of the directory. public string FileParentId => _response.Headers.TryGetValue("x-ms-file-parent-id", out string value) ? value : null; + /// NFS only. The mode of the file or directory. + public string FileMode => _response.Headers.TryGetValue("x-ms-mode", out string value) ? value : null; + /// NFS only. The owner of the file or directory. + public string Owner => _response.Headers.TryGetValue("x-ms-owner", out string value) ? value : null; + /// NFS only. The owning group of the file or directory. + public string Group => _response.Headers.TryGetValue("x-ms-group", out string value) ? value : null; + /// NFS only. The link count of the file or directory. + public long? LinkCount => _response.Headers.TryGetValue("x-ms-link-count", out long? value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/ModeCopyMode.Serialization.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/ModeCopyMode.Serialization.cs new file mode 100644 index 0000000000000..40e7ec8999e73 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/ModeCopyMode.Serialization.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; + +namespace Azure.Storage.Files.Shares.Models +{ + internal static partial class ModeCopyModeExtensions + { + public static string ToSerialString(this ModeCopyMode value) => value switch + { + ModeCopyMode.Source => "source", + ModeCopyMode.Override => "override", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown ModeCopyMode value.") + }; + + public static ModeCopyMode ToModeCopyMode(this string value) + { + if (StringComparer.OrdinalIgnoreCase.Equals(value, "source")) return ModeCopyMode.Source; + if (StringComparer.OrdinalIgnoreCase.Equals(value, "override")) return ModeCopyMode.Override; + throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown ModeCopyMode value."); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/NfsFileType.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/NfsFileType.cs new file mode 100644 index 0000000000000..2e40095b63aa6 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/NfsFileType.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; +using System.ComponentModel; + +namespace Azure.Storage.Files.Shares.Models +{ + /// The NfsFileType. + public readonly partial struct NfsFileType : IEquatable + { + private readonly string _value; + + /// Initializes a new instance of . + /// is null. + public NfsFileType(string value) + { + _value = value ?? throw new ArgumentNullException(nameof(value)); + } + + private const string RegularValue = "Regular"; + private const string DirectoryValue = "Directory"; + private const string SymlinkValue = "Symlink"; + + /// Regular. + public static NfsFileType Regular { get; } = new NfsFileType(RegularValue); + /// Directory. + public static NfsFileType Directory { get; } = new NfsFileType(DirectoryValue); + /// Symlink. + public static NfsFileType Symlink { get; } = new NfsFileType(SymlinkValue); + /// Determines if two values are the same. + public static bool operator ==(NfsFileType left, NfsFileType right) => left.Equals(right); + /// Determines if two values are not the same. + public static bool operator !=(NfsFileType left, NfsFileType right) => !left.Equals(right); + /// Converts a to a . + public static implicit operator NfsFileType(string value) => new NfsFileType(value); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override bool Equals(object obj) => obj is NfsFileType other && Equals(other); + /// + public bool Equals(NfsFileType other) => string.Equals(_value, other._value, StringComparison.InvariantCultureIgnoreCase); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override int GetHashCode() => _value != null ? StringComparer.InvariantCultureIgnoreCase.GetHashCode(_value) : 0; + /// + public override string ToString() => _value; + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/OwnerCopyMode.Serialization.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/OwnerCopyMode.Serialization.cs new file mode 100644 index 0000000000000..38bc10e4d5984 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/OwnerCopyMode.Serialization.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// + +#nullable disable + +using System; + +namespace Azure.Storage.Files.Shares.Models +{ + internal static partial class OwnerCopyModeExtensions + { + public static string ToSerialString(this OwnerCopyMode value) => value switch + { + OwnerCopyMode.Source => "source", + OwnerCopyMode.Override => "override", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown OwnerCopyMode value.") + }; + + public static OwnerCopyMode ToOwnerCopyMode(this string value) + { + if (StringComparer.OrdinalIgnoreCase.Equals(value, "source")) return OwnerCopyMode.Source; + if (StringComparer.OrdinalIgnoreCase.Equals(value, "override")) return OwnerCopyMode.Override; + throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown OwnerCopyMode value."); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ServiceRestClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ServiceRestClient.cs index fe5ea495a7a15..d3182cfe4d216 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ServiceRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ServiceRestClient.cs @@ -31,7 +31,7 @@ internal partial class ServiceRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, share, directory or file that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". + /// Specifies the version of the operation to use for this request. The default value is "2025-05-05". /// Valid value is backup. /// , , or is null. public ServiceRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version, ShareTokenIntent? fileRequestIntent = null) diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ShareRestClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ShareRestClient.cs index 69bb02404dd49..7740558db7a46 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ShareRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ShareRestClient.cs @@ -32,7 +32,7 @@ internal partial class ShareRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, share, directory or file that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". + /// Specifies the version of the operation to use for this request. The default value is "2025-05-05". /// Valid value is backup. /// , , or is null. public ShareRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version, ShareTokenIntent? fileRequestIntent = null) diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/FilePosixProperties.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/FilePosixProperties.cs new file mode 100644 index 0000000000000..a71721c738ea5 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/FilePosixProperties.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma warning disable SA1402 // File may only contain a single type + +using System.Collections.Generic; +using System; + +namespace Azure.Storage.Files.Shares.Models +{ + /// + /// NFS properties. + /// Note that these properties only apply to files or directories in + /// premium NFS file accounts. + /// + public class FilePosixProperties + { + /// + /// Optional. Version TBD and newer. The mode permissions to be set on the file or directory. + /// + public NfsFileMode FileMode { get; set; } + + /// + /// Optional. The owner user identifier (UID) to be set on the file or directory. The default value is 0 (root). + /// + public string Owner { get; set; } + + /// + /// Optional. The owner group identifier (GID) to be set on the file or directory. The default value is 0 (root group). + /// + public string Group { get; set; } + + /// + /// Optional, only applicable to files. The type of the file. The default value is . + /// + public NfsFileType? FileType { get; internal set; } + + /// + /// The link count of the file or directory. + /// + public long? LinkCount { get; internal set; } + } + + /// + /// FilesModelFactory provides utilities for mocking. + /// + public static partial class FilesModelFactory + { + /// + /// Creates a new StorageFileDownloadProperties instance for mocking. + /// + public static FilePosixProperties FileNfsProperties( + NfsFileMode fileMode, + string owner, + string group, + NfsFileType fileType, + long? linkCount) + { + return new FilePosixProperties + { + FileMode = fileMode, + Owner = owner, + Group = group, + FileType = fileType, + LinkCount = linkCount + }; + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/Internal/ModeCopyMode.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/Internal/ModeCopyMode.cs new file mode 100644 index 0000000000000..f2f43604a7d8d --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/Internal/ModeCopyMode.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Files.Shares.Models +{ + /// The ModeCopyMode. + internal enum ModeCopyMode + { + /// source. + Source, + /// override. + Override + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/Internal/OwnerCopyMode.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/Internal/OwnerCopyMode.cs new file mode 100644 index 0000000000000..31b1c6c9dc8bd --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/Internal/OwnerCopyMode.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Files.Shares.Models +{ + /// The OwnerCopyMode. + internal enum OwnerCopyMode + { + /// source. + Source, + /// override. + Override + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/NfsFileMode.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/NfsFileMode.cs new file mode 100644 index 0000000000000..3823708a0c17f --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/NfsFileMode.cs @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Globalization; +using System.Text; + +namespace Azure.Storage.Files.Shares.Models +{ + /// + /// The mode permissions of the file or directory. + /// + public class NfsFileMode + { + /// + /// Permissions the owner has over the file or directory. + /// + public RolePermissions Owner { get; set; } + + /// + /// Permissions the group has over the file or directory. + /// + public RolePermissions Group { get; set; } + + /// + /// Permissions other have over the file or directory. + /// + public RolePermissions Other { get; set; } + + /// + /// Set effective user ID (setuid) on the file or directory. + /// + public bool EffectiveUserIdentity { get; set; } + + /// + /// Set effective group ID (setgid) on the file or directory. + /// + public bool EffectiveGroupIdentity { get; set; } + + /// + /// The sticky bit may be set on directories. The files in that + /// directory may only be renamed or deleted by the file's owner, the directory's owner, or the root user. + /// + public bool StickyBit { get; set; } + + /// + /// Returns the octal represenation of this as a string. + /// + public string ToOctalFileMode() + { + // https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation + StringBuilder stringBuilder = new StringBuilder(); + + int higherOrderDigit = 0; + + if (EffectiveUserIdentity) + { + higherOrderDigit |= 4; + } + + if (EffectiveGroupIdentity) + { + higherOrderDigit |= 2; + } + + if (StickyBit) + { + higherOrderDigit |= 1; + } + + stringBuilder.Append(higherOrderDigit.ToString(CultureInfo.InvariantCulture)); + stringBuilder.Append(Owner.ToOctalRolePermissions()); + stringBuilder.Append(Group.ToOctalRolePermissions()); + stringBuilder.Append(Other.ToOctalRolePermissions()); + return stringBuilder.ToString(); + } + + /// + /// Returns a from the octal string representation. + /// + /// + /// A 4-digit octal string representation of a File Mode. + /// + public static NfsFileMode ParseOctalFileMode(string modeString) + { + // https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation + if (modeString == null) + { + return null; + } + + if (modeString.Length != 4) + { + throw Errors.InvalidFormat(nameof(modeString)); + } + + NfsFileMode nfsFileMode = new NfsFileMode + { + Owner = RolePermissionExtensions.ParseOctalRolePermissions(modeString[1]), + Group = RolePermissionExtensions.ParseOctalRolePermissions(modeString[2]), + Other = RolePermissionExtensions.ParseOctalRolePermissions(modeString[3]) + }; + + int value = (int)char.GetNumericValue(modeString[0]); + + if ((value & 4) > 0) + { + nfsFileMode.EffectiveUserIdentity = true; + } + + if ((value & 2) > 0) + { + nfsFileMode.EffectiveGroupIdentity = true; + } + + if ((value & 1) > 0) + { + nfsFileMode.StickyBit = true; + } + + return nfsFileMode; + } + + /// + /// Returns this as a string in symbolic notation. + /// + public string ToSymbolicFileMode() + { + // https://en.wikipedia.org/wiki/File-system_permissions#Symbolic_notation + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.Append(Owner.ToSymbolicRolePermissions()); + stringBuilder.Append(Group.ToSymbolicRolePermissions()); + stringBuilder.Append(Other.ToSymbolicRolePermissions()); + + if (EffectiveUserIdentity) + { + if (stringBuilder[2] == 'x') + { + stringBuilder[2] = 's'; + } + else + { + stringBuilder[2] = 'S'; + } + } + + if (EffectiveGroupIdentity) + { + if (stringBuilder[5] == 'x') + { + stringBuilder[5] = 's'; + } + else + { + stringBuilder[5] = 'S'; + } + } + + if (StickyBit) + { + if (stringBuilder[8] == 'x') + { + stringBuilder[8] = 't'; + } + else + { + stringBuilder[8] = 'T'; + } + } + return stringBuilder.ToString(); + } + /// + /// Returns a from the symbolic string representation. + /// + /// + /// A 9-character symbolic string representation of a File Mode. + /// + public static NfsFileMode ParseSymbolicFileMode(string modeString) + { + // https://en.wikipedia.org/wiki/File-system_permissions#Symbolic_notation + if (modeString == null) + { + return null; + } + + if (modeString.Length != 9) + { + throw Errors.InvalidFormat(nameof(modeString)); + } + + NfsFileMode nfsFileMode = new NfsFileMode(); + + nfsFileMode.Owner = RolePermissionExtensions.ParseSymbolicRolePermissions(modeString.Substring(0, 3), out bool effectiveUserIdentity); + nfsFileMode.EffectiveUserIdentity = effectiveUserIdentity; + + nfsFileMode.Group = RolePermissionExtensions.ParseSymbolicRolePermissions(modeString.Substring(3, 3), out bool effectiveGroupIdentity); + nfsFileMode.EffectiveGroupIdentity = effectiveGroupIdentity; + + nfsFileMode.Other = RolePermissionExtensions.ParseSymbolicRolePermissions(modeString.Substring(6, 3), out bool stickyBit); + nfsFileMode.StickyBit = stickyBit; + + return nfsFileMode; + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissionExtensions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissionExtensions.cs new file mode 100644 index 0000000000000..1f075014e8b29 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissionExtensions.cs @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Globalization; +using System.Text; + +namespace Azure.Storage.Files.Shares.Models +{ + /// + /// Extension methods for . + /// + internal static class RolePermissionExtensions + { + /// + /// Parses octal char to RolePermissions. + /// + public static RolePermissions ParseOctalRolePermissions(char c) + { + RolePermissions rolePermissions = RolePermissions.None; + + int value = (int)char.GetNumericValue(c); + + if (value < 0 || value > 7) + { + throw Errors.MustBeBetweenInclusive(nameof(c), 0, 7, value); + } + + if ((value & 4) > 0) + { + rolePermissions |= RolePermissions.Read; + } + + if ((value & 2) > 0) + { + rolePermissions |= RolePermissions.Write; + } + + if ((value & 1) > 0) + { + rolePermissions |= RolePermissions.Execute; + } + + return rolePermissions; + } + + /// + /// Returns the octal string representation of this RolePermissions. + /// + public static string ToOctalRolePermissions(this RolePermissions rolePermissions) + { + int result = 0; + + if (rolePermissions.HasFlag(RolePermissions.Read)) + { + result |= 4; + } + + if (rolePermissions.HasFlag(RolePermissions.Write)) + { + result |= 2; + } + + if (rolePermissions.HasFlag(RolePermissions.Execute)) + { + result |= 1; + } + + return result.ToString(CultureInfo.InvariantCulture); + } + + /// + /// Returns the symbolic string representation of this RolePermissions. + /// + public static string ToSymbolicRolePermissions(this RolePermissions rolePermissions) + { + StringBuilder stringBuilder = new StringBuilder(); + + if (rolePermissions.HasFlag(RolePermissions.Read)) + { + stringBuilder.Append("r"); + } + else + { + stringBuilder.Append("-"); + } + + if (rolePermissions.HasFlag(RolePermissions.Write)) + { + stringBuilder.Append("w"); + } + else + { + stringBuilder.Append("-"); + } + + if (rolePermissions.HasFlag(RolePermissions.Execute)) + { + stringBuilder.Append("x"); + } + else + { + stringBuilder.Append("-"); + } + + return stringBuilder.ToString(); + } + + public static RolePermissions ParseSymbolicRolePermissions(string s, out bool setSticky) + { + if (s == null) + { + throw new ArgumentNullException("s"); + } + if (s.Length != 3) + { + throw new FormatException($"s must be 3 characters long"); + } + + RolePermissions rolePermissions = new RolePermissions(); + setSticky = false; + + // Read character + if (s[0] == 'r') + { + rolePermissions |= RolePermissions.Read; + } + else if (s[0] != '-') + { + throw new ArgumentException($"Invalid character in symbolic role permission: {s[0]}"); + } + + // Write character + if (s[1] == 'w') + { + rolePermissions |= RolePermissions.Write; + } + else if (s[1] != '-') + { + throw new ArgumentException($"Invalid character in symbolic role permission: {s[1]}"); + } + + // Execute character + if (s[2] == 'x' || s[2] == 's' || s[2] == 't') + { + rolePermissions |= RolePermissions.Execute; + if (s[2] == 's' || s[2] == 't') + { + setSticky = true; + } + } + if (s[2] == 'S' || s[2] == 'T') + { + setSticky = true; + } + + if (s[2] != 'x' && s[2] != 's' && s[2] != 'S' && s[2] != 't' && s[2] != 'T' && s[2] != '-') + { + throw new ArgumentException($"Invalid character in symbolic role permission: {s[2]}"); + } + + return rolePermissions; + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissions.cs new file mode 100644 index 0000000000000..0d21ca0f1a5c1 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissions.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +namespace Azure.Storage.Files.Shares.Models +{ + /// + /// Represents file permissions for a specific role. + /// + [Flags] + public enum RolePermissions + { + /// + /// No permissions. + /// + None = 0, + + /// + /// The execute permission. + /// + Execute = 1, + + /// + /// The write permission. + /// + Write = 2, + + /// + /// The read permission. + /// + Read = 4, + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryCreateOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryCreateOptions.cs index 929ad0adc9e3a..93739869c18c8 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryCreateOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryCreateOptions.cs @@ -24,5 +24,13 @@ public class ShareDirectoryCreateOptions /// Optional file permission to set on the directory. /// public ShareFilePermission FilePermission { get; set; } + + /// + /// + /// Optional properties to set on NFS directories. + /// Note that this property is only applicable to directories created in NFS shares. + /// + /// + public FilePosixProperties NfsProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryInfo.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryInfo.cs index 1d9f258b79caf..5e7ab16df6cff 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryInfo.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryInfo.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Text; #pragma warning disable SA1402 // File may only contain a single type @@ -30,6 +31,12 @@ public class ShareDirectoryInfo /// public FileSmbProperties SmbProperties { get; set; } + /// + /// The directory's NFS properties. + /// Only applicable to files in a NFS share. + /// + public FilePosixProperties NfsProperties { get; internal set; } + /// /// Constructor. /// @@ -44,6 +51,23 @@ public static partial class SharesModelFactory /// /// Creates a new StorageDirectoryInfo instance for mocking. /// + public static ShareDirectoryInfo StorageDirectoryInfo( + ETag eTag = default, + DateTimeOffset lastModified = default, + FileSmbProperties smbProperties = default, + FilePosixProperties nfsProperties = default) + => new ShareDirectoryInfo + { + ETag = eTag, + LastModified = lastModified, + SmbProperties = smbProperties, + NfsProperties = nfsProperties, + }; + + /// + /// Creates a new StorageDirectoryInfo instance for mocking. + /// + [EditorBrowsable(EditorBrowsableState.Never)] public static ShareDirectoryInfo StorageDirectoryInfo( ETag eTag, DateTimeOffset lastModified, diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryProperties.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryProperties.cs index b225e512ac385..887f630ba2412 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryProperties.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryProperties.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Text; #pragma warning disable SA1402 // File may only contain a single type @@ -41,6 +42,12 @@ public class ShareDirectoryProperties /// public FileSmbProperties SmbProperties { get; set; } + /// + /// NFS properties. + /// Note that this property is only applicable to directories created in NFS shares. + /// + public FilePosixProperties NfsProperties { get; internal set; } + /// /// Constructor. /// @@ -55,6 +62,28 @@ public static partial class FilesModelFactory /// /// Creates a new StorageDirectoryProperties instance for mocking. /// + public static ShareDirectoryProperties StorageDirectoryProperties( + IDictionary metadata = default, + ETag eTag = default, + DateTimeOffset lastModified = default, + bool isServerEncrypted = default, + FileSmbProperties smbProperties = default, + FilePosixProperties nfsProperties = default + ) + => new ShareDirectoryProperties + { + Metadata = metadata, + ETag = eTag, + LastModified = lastModified, + IsServerEncrypted = isServerEncrypted, + SmbProperties = smbProperties, + NfsProperties = nfsProperties + }; + + /// + /// Creates a new StorageDirectoryProperties instance for mocking. + /// + [EditorBrowsable(EditorBrowsableState.Never)] public static ShareDirectoryProperties StorageDirectoryProperties( IDictionary metadata, ETag eTag, diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectorySetHttpHeadersOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectorySetHttpHeadersOptions.cs index 9c07466061b1f..d64c2c52f8147 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectorySetHttpHeadersOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectorySetHttpHeadersOptions.cs @@ -18,5 +18,11 @@ public class ShareDirectorySetHttpHeadersOptions /// Optional file permission to set for the directory. /// public ShareFilePermission FilePermission { get; set; } + + /// + /// Optional properties to set on NFS files. + /// Note that this property is only applicable to directories created in NFS shares. + /// + public FilePosixProperties NfsProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCopyOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCopyOptions.cs index d74113002f182..358db59256ff0 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCopyOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCopyOptions.cs @@ -64,5 +64,10 @@ public class ShareFileCopyOptions /// SMB properties to copy from the source file. /// public CopyableFileSmbProperties SmbPropertiesToCopy { get; set; } + + /// + /// Only applicable to NFS Files. NFS properties to set on the destination file. + /// + public FilePosixProperties NfsProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateOptions.cs index 9f3a517321834..02e9ad752af54 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateOptions.cs @@ -22,6 +22,7 @@ public class ShareFileCreateOptions /// /// Optional SMB properties to set for the file. + /// Note that this property is only applicable to files created in SMB shares. /// public FileSmbProperties SmbProperties { get; set; } @@ -29,5 +30,11 @@ public class ShareFileCreateOptions /// Optional file permission to set on file. /// public ShareFilePermission FilePermission { get; set; } + + /// + /// Optional properties to set on NFS files. + /// Note that this property is only applicable to files created in NFS shares. + /// + public FilePosixProperties NfsProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateSymbolicLinkOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateSymbolicLinkOptions.cs new file mode 100644 index 0000000000000..7efafb00c3e1e --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateSymbolicLinkOptions.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Metadata = System.Collections.Generic.IDictionary; + +namespace Azure.Storage.Files.Shares.Models +{ + /// + /// NFS only. Options for creating a symbolic link. + /// + /// https://github.com/Azure/azure-sdk-for-net/issues/46907 + internal class ShareFileCreateSymbolicLinkOptions + { + /// + /// Optional custom metadata to set for the symbolic link. + /// + public Metadata Metadata { get; set; } + + /// + /// The creation time of the symbolic link. + /// + public DateTimeOffset? FileCreatedOn { get; set; } + + /// + /// The last write time of the symbolic link. + /// + public DateTimeOffset? FileLastWrittenOn { get; set; } + + /// + /// Optional. The owner user identifier (UID) to be set on the symbolic link. The default value is 0 (root). + /// + public string Owner { get; set; } + + /// + /// Optional. The owner group identifier (GID) to be set on the symbolic link. The default value is 0 (root group). + /// + public string Group { get; set; } + + /// + /// Optional to add conditions + /// on creating the symbolic link. + /// + public ShareFileRequestConditions Conditions { get; set; } + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadDetails.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadDetails.cs index 8de847b910f64..5d7958ead83a6 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadDetails.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadDetails.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; #pragma warning disable SA1402 // File may only contain a single type @@ -122,10 +123,16 @@ public partial class ShareFileDownloadDetails public ShareLeaseStatus LeaseStatus { get; internal set; } /// - /// The SMB properties for the file + /// The SMB properties for the file. /// public FileSmbProperties SmbProperties { get; set; } + /// + /// NFS properties. + /// Note that this property is only applicable to files created in NFS shares. + /// + public FilePosixProperties NfsProperties { get; internal set; } + /// /// Constructor. /// @@ -140,6 +147,60 @@ public static partial class FilesModelFactory /// /// Creates a new StorageFileDownloadProperties instance for mocking. /// + public static ShareFileDownloadDetails StorageFileDownloadProperties( + DateTimeOffset lastModified = default, + IDictionary metadata = default, + string contentRange = default, + ETag eTag = default, + IEnumerable contentEncoding = default, + string cacheControl = default, + string contentDisposition = default, + IEnumerable contentLanguage = default, + string acceptRanges = default, + DateTimeOffset copyCompletedOn = default, + string copyStatusDescription = default, + string copyId = default, + string copyProgress = default, + Uri copySource = default, + CopyStatus copyStatus = default, + byte[] fileContentHash = default, + bool isServiceEncrypted = default, + ShareLeaseDuration leaseDuration = default, + ShareLeaseState leaseState = default, + ShareLeaseStatus leaseStatus = default, + FileSmbProperties smbProperties = default, + FilePosixProperties nfsProperties = default) + { + return new ShareFileDownloadDetails + { + LastModified = lastModified, + Metadata = metadata, + ContentRange = contentRange, + ETag = eTag, + ContentEncoding = contentEncoding, + CacheControl = cacheControl, + ContentDisposition = contentDisposition, + ContentLanguage = contentLanguage, + AcceptRanges = acceptRanges, + CopyCompletedOn = copyCompletedOn, + CopyStatusDescription = copyStatusDescription, + CopyId = copyId, + CopyProgress = copyProgress, + CopySource = copySource, + CopyStatus = copyStatus, + FileContentHash = fileContentHash, + IsServerEncrypted = isServiceEncrypted, + LeaseDuration = leaseDuration, + LeaseState = leaseState, + SmbProperties = smbProperties, + NfsProperties = nfsProperties, + }; + } + + /// + /// Creates a new StorageFileDownloadProperties instance for mocking. + /// + [EditorBrowsable(EditorBrowsableState.Never)] public static ShareFileDownloadDetails StorageFileDownloadProperties( DateTimeOffset lastModified, IDictionary metadata, diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileInfo.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileInfo.cs index 46212582505fe..37e9d5ab26877 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileInfo.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileInfo.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Text; #pragma warning disable SA1402 // File may only contain a single type @@ -31,9 +32,16 @@ public class ShareFileInfo /// /// The file's SMB properties. + /// Only applicable to files in a SMB share. /// public FileSmbProperties SmbProperties { get; set; } + /// + /// The file's NFS properties. + /// Only applicable to files in a NFS share. + /// + public FilePosixProperties NfsProperties { get; internal set; } + /// /// Constructor. /// @@ -48,6 +56,49 @@ public static partial class FilesModelFactory /// /// Creates a new StorageFileInfo instance for mocking. /// + public static ShareFileInfo StorageFileInfo( + ETag eTag = default, + DateTimeOffset lastModified = default, + bool isServerEncrypted = default, + string filePermissionKey = default, + string fileAttributes = default, + DateTimeOffset fileCreationTime = default, + DateTimeOffset fileLastWriteTime = default, + DateTimeOffset fileChangeTime = default, + string fileId = default, + string fileParentId = default, + NfsFileMode nfsFileMode = default, + string owner = default, + string group = default, + NfsFileType nfsFileType = default) + => new ShareFileInfo + { + ETag = eTag, + LastModified = lastModified, + IsServerEncrypted = isServerEncrypted, + SmbProperties = new FileSmbProperties + { + FileAttributes = ShareModelExtensions.ToFileAttributes(fileAttributes), + FilePermissionKey = filePermissionKey, + FileCreatedOn = fileCreationTime, + FileLastWrittenOn = fileLastWriteTime, + FileChangedOn = fileChangeTime, + FileId = fileId, + ParentId = fileParentId + }, + NfsProperties = new FilePosixProperties + { + FileMode = nfsFileMode, + Owner = owner, + Group = group, + FileType = nfsFileType, + } + }; + + /// + /// Creates a new StorageFileInfo instance for mocking. + /// + [EditorBrowsable(EditorBrowsableState.Never)] public static ShareFileInfo StorageFileInfo( ETag eTag, DateTimeOffset lastModified, diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileProperties.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileProperties.cs index 05715a8e12b6b..efd8c598a473e 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileProperties.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileProperties.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; #pragma warning disable SA1402 // File may only contain a single type @@ -119,6 +120,12 @@ public class ShareFileProperties /// public ShareLeaseStatus LeaseStatus { get; internal set; } + /// + /// NFS properties. + /// Note that this property is only applicable to files created in NFS shares. + /// + public FilePosixProperties NfsProperties { get; internal set; } + /// /// Constructor. /// @@ -133,6 +140,53 @@ public static partial class FilesModelFactory /// /// Creates a new StorageFileProperties instance for mocking. /// + public static ShareFileProperties StorageFileProperties( + DateTimeOffset lastModified = default, + IDictionary metadata = default, + long contentLength = default, + string contentType = default, + ETag eTag = default, + byte[] contentHash = default, + IEnumerable contentEncoding = default, + string cacheControl = default, + string contentDisposition = default, + IEnumerable contentLanguage = default, + DateTimeOffset copyCompletedOn = default, + string copyStatusDescription = default, + string copyId = default, + string copyProgress = default, + string copySource = default, + CopyStatus copyStatus = default, + bool isServerEncrypted = default, + FileSmbProperties smbProperties = default, + FilePosixProperties nfsProperties = default + ) => new ShareFileProperties + { + LastModified = lastModified, + Metadata = metadata, + ContentLength = contentLength, + ContentType = contentType, + ETag = eTag, + ContentHash = contentHash, + ContentEncoding = contentEncoding, + CacheControl = cacheControl, + ContentDisposition = contentDisposition, + ContentLanguage = contentLanguage, + CopyCompletedOn = copyCompletedOn, + CopyStatusDescription = copyStatusDescription, + CopyId = copyId, + CopyProgress = copyProgress, + CopySource = copySource, + CopyStatus = copyStatus, + IsServerEncrypted = isServerEncrypted, + SmbProperties = smbProperties, + NfsProperties = nfsProperties, + }; + + /// + /// Creates a new StorageFileProperties instance for mocking. + /// + [EditorBrowsable(EditorBrowsableState.Never)] public static ShareFileProperties StorageFileProperties( DateTimeOffset lastModified, IDictionary metadata, @@ -194,6 +248,7 @@ string fileParentId /// /// Creates a new StorageFileProperties instance for mocking. /// + [EditorBrowsable(EditorBrowsableState.Never)] public static ShareFileProperties StorageFileProperties( DateTimeOffset lastModified, IDictionary metadata, diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSetHttpHeadersOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSetHttpHeadersOptions.cs index 3f73e0e99e75a..af62257ceb0ba 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSetHttpHeadersOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSetHttpHeadersOptions.cs @@ -31,5 +31,11 @@ public class ShareFileSetHttpHeadersOptions /// Optional file permission to set for the file. /// public ShareFilePermission FilePermission { get; set; } + + /// + /// Optional properties to set on NFS files. + /// Note that this property is only applicable to files created in NFS shares. + /// + public FilePosixProperties NfsProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSymbolicLinkInfo.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSymbolicLinkInfo.cs new file mode 100644 index 0000000000000..990ac2ec5fce5 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSymbolicLinkInfo.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma warning disable SA1402 // File may only contain a single type + +using System; + +namespace Azure.Storage.Files.Shares.Models +{ + /// + /// Infomation about a Symbolic Link. + /// Only applicable to NFS files. + /// + /// https://github.com/Azure/azure-sdk-for-net/issues/46907 + internal class ShareFileSymbolicLinkInfo + { + /// + /// The ETag contains a value which represents the version of the file, in quotes. + /// + public ETag ETag { get; internal set; } + + /// + /// Returns the date and time the file was last modified. + /// + public DateTimeOffset LastModified { get; internal set; } + + /// + /// Text of the symbolic link. + /// + public string LinkText { get; internal set; } + } + + public static partial class SharesModelFactory + { + /// + /// Creates a new FileSymolicLinkInfo for mocking. + /// + /// https://github.com/Azure/azure-sdk-for-net/issues/46907 + internal static ShareFileSymbolicLinkInfo FileSymbolicLinkInfo( + ETag eTag = default, + DateTimeOffset lastModified = default, + string linkText = default) + => new ShareFileSymbolicLinkInfo + { + ETag = eTag, + LastModified = lastModified, + LinkText = linkText + }; + } +} diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareClient.cs index 75d4430e5b26d..3f23c2d40de16 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareClient.cs @@ -3710,6 +3710,7 @@ public virtual Response CreateDirectory( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, + nfsProperties: options?.NfsProperties, async: false, cancellationToken, operationName: $"{nameof(ShareClient)}.{nameof(CreateDirectory)}") @@ -3765,6 +3766,7 @@ public virtual Response CreateDirectory( smbProperties, filePermission, filePermissionFormat: null, + nfsProperties: null, async: false, cancellationToken, operationName: $"{nameof(ShareClient)}.{nameof(CreateDirectory)}") @@ -3809,6 +3811,7 @@ public virtual async Task> CreateDirectoryAsync( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, + nfsProperties: options?.NfsProperties, async: true, cancellationToken, operationName: $"{nameof(ShareClient)}.{nameof(CreateDirectory)}") @@ -3864,6 +3867,7 @@ public virtual async Task> CreateDirectoryAsync( smbProperties, filePermission, filePermissionFormat: null, + nfsProperties: null, async: true, cancellationToken, operationName: $"{nameof(ShareClient)}.{nameof(CreateDirectory)}") diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareDirectoryClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareDirectoryClient.cs index 1deeef8b6c6d0..7b2af0156f107 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareDirectoryClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareDirectoryClient.cs @@ -561,6 +561,7 @@ public virtual Response Create( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, + nfsProperties: options?.NfsProperties, async: false, // async cancellationToken: cancellationToken) .EnsureCompleted(); @@ -607,6 +608,7 @@ public virtual Response Create( smbProperties, filePermission, filePermissionFormat: null, + nfsProperties: null, false, // async cancellationToken) .EnsureCompleted(); @@ -642,6 +644,7 @@ await CreateInternal( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, + nfsProperties: options?.NfsProperties, async: true, cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -688,6 +691,7 @@ await CreateInternal( smbProperties, filePermission, filePermissionFormat: null, + nfsProperties: null, true, // async cancellationToken) .ConfigureAwait(false); @@ -712,6 +716,9 @@ await CreateInternal( /// /// Optional file permission format. /// + /// + /// Optional NFS properties. + /// /// /// Whether to invoke the operation asynchronously. /// @@ -735,6 +742,7 @@ internal async Task> CreateInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, + FilePosixProperties nfsProperties, bool async, CancellationToken cancellationToken, string operationName = default) @@ -754,38 +762,40 @@ internal async Task> CreateInternal( FileSmbProperties smbProps = smbProperties ?? new FileSmbProperties(); ShareExtensions.AssertValidFilePermissionAndKey(filePermission, smbProps.FilePermissionKey); - if (filePermission == null && smbProps.FilePermissionKey == null) - { - filePermission = Constants.File.FilePermissionInherit; - } ResponseWithHeaders response; if (async) { response = await DirectoryRestClient.CreateAsync( - fileAttributes: smbProps.FileAttributes?.ToAttributesString() ?? Constants.File.FileAttributesNone, - fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString() ?? Constants.File.FileTimeNow, - fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString() ?? Constants.File.FileTimeNow, + fileAttributes: smbProps?.FileAttributes.ToAttributesString(), + fileCreationTime: smbProps?.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: smbProps?.FileLastWrittenOn.ToFileDateTimeString(), metadata: metadata, filePermission: filePermission, filePermissionFormat: filePermissionFormat, - filePermissionKey: smbProps.FilePermissionKey, - fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), + filePermissionKey: smbProps?.FilePermissionKey, + fileChangeTime: smbProps?.FileChangedOn.ToFileDateTimeString(), + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode.ToOctalFileMode(), cancellationToken: cancellationToken) .ConfigureAwait(false); } else { response = DirectoryRestClient.Create( - fileAttributes: smbProps.FileAttributes?.ToAttributesString() ?? Constants.File.FileAttributesNone, - fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString() ?? Constants.File.FileTimeNow, - fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString() ?? Constants.File.FileTimeNow, + fileAttributes: smbProps?.FileAttributes.ToAttributesString(), + fileCreationTime: smbProps?.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: smbProps?.FileLastWrittenOn.ToFileDateTimeString(), metadata: metadata, filePermission: filePermission, filePermissionFormat: filePermissionFormat, - filePermissionKey: smbProps.FilePermissionKey, - fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), + filePermissionKey: smbProps?.FilePermissionKey, + fileChangeTime: smbProps?.FileChangedOn.ToFileDateTimeString(), + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode.ToOctalFileMode(), cancellationToken: cancellationToken); } @@ -841,6 +851,7 @@ public virtual Response CreateIfNotExists( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, + nfsProperties: options?.NfsProperties, async: false, cancellationToken).EnsureCompleted(); @@ -887,6 +898,7 @@ public virtual Response CreateIfNotExists( smbProperties, filePermission, filePermissionFormat: null, + nfsProperties: null, async: false, cancellationToken).EnsureCompleted(); @@ -922,6 +934,7 @@ await CreateIfNotExistsInternal( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, + nfsProperties: options?.NfsProperties, async: true, cancellationToken: cancellationToken).ConfigureAwait(false); @@ -968,6 +981,7 @@ await CreateIfNotExistsInternal( smbProperties, filePermission, filePermissionFormat: null, + nfsProperties: null, async: true, cancellationToken).ConfigureAwait(false); @@ -992,6 +1006,9 @@ await CreateIfNotExistsInternal( /// /// Optional file permission format. /// + /// + /// Optional NFS properties. + /// /// /// Whether to invoke the operation asynchronously. /// @@ -1015,6 +1032,7 @@ internal async Task> CreateIfNotExistsInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, + FilePosixProperties nfsProperties, bool async, CancellationToken cancellationToken, string operationName = default) @@ -1031,6 +1049,7 @@ internal async Task> CreateIfNotExistsInternal( smbProperties, filePermission, filePermissionFormat, + nfsProperties, async, cancellationToken, operationName: operationName ?? $"{nameof(ShareDirectoryClient)}.{nameof(CreateIfNotExists)}") @@ -1566,6 +1585,7 @@ public virtual Response SetHttpHeaders( options?.SmbProperties, options?.FilePermission?.Permission, options?.FilePermission?.PermissionFormat, + options?.NfsProperties, false, // async cancellationToken) .EnsureCompleted(); @@ -1598,6 +1618,7 @@ await SetHttpHeadersInternal( options?.SmbProperties, options?.FilePermission?.Permission, options?.FilePermission?.PermissionFormat, + options?.NfsProperties, true, // async cancellationToken) .ConfigureAwait(false); @@ -1639,6 +1660,7 @@ public virtual Response SetHttpHeaders( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, false, // async cancellationToken) .EnsureCompleted(); @@ -1678,6 +1700,7 @@ await SetHttpHeadersInternal( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, true, // async cancellationToken) .ConfigureAwait(false); @@ -1699,6 +1722,9 @@ await SetHttpHeadersInternal( /// /// Optional file permission format. /// + /// + /// Optional NFS properties. + /// /// /// Whether to invoke the operation asynchronously. /// @@ -1718,6 +1744,7 @@ private async Task> SetHttpHeadersInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, + FilePosixProperties nfsProperties, bool async, CancellationToken cancellationToken) { @@ -1736,36 +1763,38 @@ private async Task> SetHttpHeadersInternal( FileSmbProperties smbProps = smbProperties ?? new FileSmbProperties(); ShareExtensions.AssertValidFilePermissionAndKey(filePermission, smbProps.FilePermissionKey); - if (filePermission == null && smbProps.FilePermissionKey == null) - { - filePermission = Constants.File.Preserve; - } ResponseWithHeaders response; if (async) { response = await DirectoryRestClient.SetPropertiesAsync( - fileAttributes: smbProps.FileAttributes?.ToAttributesString() ?? Constants.File.Preserve, - fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString() ?? Constants.File.Preserve, - fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString() ?? Constants.File.Preserve, + fileAttributes: smbProps?.FileAttributes.ToAttributesString(), + fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString(), filePermission: filePermission, filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps.FilePermissionKey, fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode.ToOctalFileMode(), cancellationToken: cancellationToken) .ConfigureAwait(false); } else { response = DirectoryRestClient.SetProperties( - fileAttributes: smbProps.FileAttributes?.ToAttributesString() ?? Constants.File.Preserve, - fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString() ?? Constants.File.Preserve, - fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString() ?? Constants.File.Preserve, + fileAttributes: smbProps?.FileAttributes.ToAttributesString(), + fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString(), filePermission: filePermission, filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps.FilePermissionKey, fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode.ToOctalFileMode(), cancellationToken: cancellationToken); } @@ -2863,7 +2892,7 @@ private async Task> RenameInternal( CopyFileSmbInfo copyFileSmbInfo = new CopyFileSmbInfo { - FileAttributes = options?.SmbProperties?.FileAttributes?.ToAttributesString(), + FileAttributes = options?.SmbProperties?.FileAttributes.ToAttributesString(), FileCreationTime = options?.SmbProperties?.FileCreatedOn.ToFileDateTimeString(), FileLastWriteTime = options?.SmbProperties?.FileLastWrittenOn.ToFileDateTimeString(), FileChangeTime = options?.SmbProperties?.FileChangedOn.ToFileDateTimeString(), diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareExtensions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareExtensions.cs index 4d7a0950ab0d6..89fbdc06c7996 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareExtensions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareExtensions.cs @@ -6,7 +6,9 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; +using System.Text.RegularExpressions; using Azure.Core; using Azure.Storage.Files.Shares.Models; @@ -16,14 +18,14 @@ internal static partial class ShareExtensions { internal static void AssertValidFilePermissionAndKey(string filePermission, string filePermissionKey) { - if (filePermission != null && filePermissionKey != null) + if (filePermission != null && Encoding.UTF8.GetByteCount(filePermission) > Constants.File.MaxFilePermissionHeaderSize) { - throw Errors.CannotBothBeNotNull(nameof(filePermission), nameof(filePermissionKey)); + throw Errors.MustBeLessThanOrEqualTo(nameof(filePermission), Constants.File.MaxFilePermissionHeaderSize); } - if (filePermission != null && Encoding.UTF8.GetByteCount(filePermission) > Constants.File.MaxFilePermissionHeaderSize) + if (filePermission != null && filePermissionKey != null) { - throw Errors.MustBeLessThanOrEqualTo(nameof(filePermission), Constants.File.MaxFilePermissionHeaderSize); + throw new ArgumentException("filePermission and filePermissionKey cannot both be set"); } } @@ -113,6 +115,13 @@ internal static ShareDirectoryInfo ToShareDirectoryInfo(this ResponseWithHeaders FileChangedOn = response.Headers.FileChangeTime, FileId = response.Headers.FileId, ParentId = response.Headers.FileParentId + }, + NfsProperties = new FilePosixProperties + { + FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), + Owner = response.Headers.Owner, + Group = response.Headers.Group, + FileType = response.Headers.NfsFileType, } }; } @@ -138,6 +147,13 @@ internal static ShareDirectoryProperties ToShareDirectoryProperties(this Respons FileChangedOn = response.Headers.FileChangeTime, FileId = response.Headers.FileId, ParentId = response.Headers.FileParentId + }, + NfsProperties = new FilePosixProperties() + { + FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), + Owner = response.Headers.Owner, + Group = response.Headers.Group, + FileType = response.Headers.NfsFileType, } }; } @@ -161,6 +177,12 @@ internal static ShareDirectoryInfo ToShareDirectoryInfo(this ResponseWithHeaders FileChangedOn = response.Headers.FileChangeTime, FileId = response.Headers.FileId, ParentId = response.Headers.FileParentId + }, + NfsProperties = new FilePosixProperties + { + FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), + Owner = response.Headers.Owner, + Group = response.Headers.Group } }; } @@ -279,7 +301,7 @@ internal static ShareFileInfo ToShareFileInfo(this ResponseWithHeaders response) @@ -347,7 +378,15 @@ internal static ShareFileProperties ToShareFileProperties(this ResponseWithHeade }, LeaseDuration = response.Headers.LeaseDuration.GetValueOrDefault(), LeaseState = response.Headers.LeaseState.GetValueOrDefault(), - LeaseStatus = response.Headers.LeaseStatus.GetValueOrDefault() + LeaseStatus = response.Headers.LeaseStatus.GetValueOrDefault(), + NfsProperties = new FilePosixProperties() + { + FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), + Owner = response.Headers.Owner, + Group = response.Headers.Group, + FileType = response.Headers.NfsFileType, + LinkCount = response.Headers.LinkCount + } }; if (response.Headers.ContentEncoding != null) @@ -383,6 +422,13 @@ internal static ShareFileInfo ToShareFileInfo(this ResponseWithHeaders ToShareFilePermission(this Respons }, response.GetRawResponse()); } + + internal static ShareFileSymbolicLinkInfo ToFileSymbolicLinkInfo(this ResponseWithHeaders response) + { + if (response == null) + { + return null; + } + + return new ShareFileSymbolicLinkInfo + { + ETag = response.GetRawResponse().Headers.TryGetValue(Constants.HeaderNames.ETag, out string value) ? new ETag(value) : default, + LastModified = response.Headers.LastModified.GetValueOrDefault(), + LinkText = response.Headers.LinkText + }; + } + + internal static ShareFileInfo ToShareFileInfo(this ResponseWithHeaders response) + { + if (response == null) + { + return null; + } + return new ShareFileInfo + { + ETag = response.GetRawResponse().Headers.TryGetValue(Constants.HeaderNames.ETag, out string value) ? new ETag(value) : default, + LastModified = response.Headers.LastModified.GetValueOrDefault(), + SmbProperties = new FileSmbProperties + { + FileCreatedOn = response.Headers.FileCreationTime, + FileLastWrittenOn = response.Headers.FileLastWriteTime, + FileChangedOn = response.Headers.FileChangeTime, + FileId = response.Headers.FileId, + ParentId = response.Headers.FileParentId + }, + NfsProperties = new FilePosixProperties() + { + FileType = response.Headers.NfsFileType, + FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), + Owner = response.Headers.Owner, + Group = response.Headers.Group + } + }; + } + + internal static ShareFileInfo ToShareFileInfo(this ResponseWithHeaders response) + { + if (response == null) + { + return null; + } + return new ShareFileInfo + { + ETag = response.GetRawResponse().Headers.TryGetValue(Constants.HeaderNames.ETag, out string value) ? new ETag(value) : default, + LastModified = response.Headers.LastModified.GetValueOrDefault(), + SmbProperties = new FileSmbProperties + { + FileCreatedOn = response.Headers.FileCreationTime, + FileLastWrittenOn = response.Headers.FileLastWriteTime, + FileChangedOn = response.Headers.FileChangeTime, + FileId = response.Headers.FileId, + ParentId = response.Headers.FileParentId + }, + NfsProperties = new FilePosixProperties() + { + FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), + Owner = response.Headers.Owner, + Group = response.Headers.Group, + LinkCount = response.Headers.LinkCount, + FileType = response.Headers.NfsFileType + } + }; + } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs index f713200a524de..263e4b5f68973 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs @@ -15,6 +15,7 @@ using Azure.Storage.Sas; using Metadata = System.Collections.Generic.IDictionary; using Azure.Storage.Common; +using System.Net.Http.Headers; #pragma warning disable SA1402 // File may only contain a single type @@ -129,7 +130,7 @@ public virtual string Path /// public virtual bool CanGenerateSasUri => ClientConfiguration.SharedKeyCredential != null; - //const string fileType = "file"; + //const string filetype = "file"; //// FileMaxUploadRangeBytes indicates the maximum number of bytes that can be sent in a call to UploadRange. //public const Int64 FileMaxUploadRangeBytes = 4 * Constants.MB; // 4MB @@ -578,6 +579,7 @@ public virtual Response Create( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, + nfsProperties: options?.NfsProperties, conditions, async: false, cancellationToken) @@ -628,6 +630,7 @@ await CreateInternal( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, + nfsProperties: options?.NfsProperties, conditions, async: true, cancellationToken) @@ -693,6 +696,7 @@ public virtual Response Create( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, conditions, async: false, cancellationToken) @@ -753,6 +757,7 @@ public virtual Response Create( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, conditions: default, async: false, cancellationToken) @@ -818,6 +823,7 @@ await CreateInternal( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, conditions, async: true, cancellationToken) @@ -878,6 +884,7 @@ await CreateInternal( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, conditions: default, async: true, cancellationToken) @@ -912,6 +919,9 @@ await CreateInternal( /// /// Optional file permission format. /// + /// + /// Optional NFS properties. + /// /// /// Optional to add conditions /// on creating the file. @@ -941,6 +951,7 @@ private async Task> CreateInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, + FilePosixProperties nfsProperties, ShareFileRequestConditions conditions, bool async, CancellationToken cancellationToken, @@ -965,21 +976,20 @@ private async Task> CreateInternal( ShareExtensions.AssertValidFilePermissionAndKey(filePermission, smbProps.FilePermissionKey); - if (filePermission == null && smbProps.FilePermissionKey == null) - { - filePermission = Constants.File.FilePermissionInherit; - } - ResponseWithHeaders response; if (async) { response = await FileRestClient.CreateAsync( fileContentLength: maxSize, - fileAttributes: smbProps.FileAttributes?.ToAttributesString() ?? Constants.File.FileAttributesNone, - fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString() ?? Constants.File.FileTimeNow, - fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString() ?? Constants.File.FileTimeNow, + fileAttributes: smbProps.FileAttributes.ToAttributesString(), + fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString(), fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode.ToOctalFileMode(), + nfsFileType: nfsProperties?.FileType, metadata: metadata, filePermission: filePermission, filePermissionFormat: filePermissionFormat, @@ -993,10 +1003,14 @@ private async Task> CreateInternal( { response = FileRestClient.Create( fileContentLength: maxSize, - fileAttributes: smbProps.FileAttributes?.ToAttributesString() ?? Constants.File.FileAttributesNone, - fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString() ?? Constants.File.FileTimeNow, - fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString() ?? Constants.File.FileTimeNow, + fileAttributes: smbProps.FileAttributes.ToAttributesString(), + fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString(), fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode.ToOctalFileMode(), + nfsFileType: nfsProperties?.FileType, metadata: metadata, filePermission: filePermission, filePermissionFormat: filePermissionFormat, @@ -1308,6 +1322,7 @@ public virtual Response StartCopy( setArchiveAttribute: options?.Archive, conditions: options?.Conditions, copyableFileSmbProperties: options?.SmbPropertiesToCopy, + nfsProperties: options?.NfsProperties, async: false, cancellationToken: cancellationToken) .EnsureCompleted(); @@ -1385,6 +1400,7 @@ public virtual Response StartCopy( setArchiveAttribute, conditions, copyableFileSmbProperties: default, + nfsProperties: default, async: false, cancellationToken) .EnsureCompleted(); @@ -1432,6 +1448,7 @@ public virtual Response StartCopy( setArchiveAttribute: default, conditions: default, copyableFileSmbProperties: default, + nfsProperties: default, async: false, cancellationToken) .EnsureCompleted(); @@ -1476,6 +1493,7 @@ await StartCopyInternal( setArchiveAttribute: options?.Archive, conditions: options?.Conditions, copyableFileSmbProperties: options?.SmbPropertiesToCopy, + nfsProperties: options?.NfsProperties, async: true, cancellationToken: cancellationToken). ConfigureAwait(false); @@ -1553,6 +1571,7 @@ await StartCopyInternal( setArchiveAttribute, conditions, copyableFileSmbProperties: default, + nfsProperties: default, async: true, cancellationToken). ConfigureAwait(false); @@ -1600,6 +1619,7 @@ await StartCopyInternal( setArchiveAttribute: default, conditions: default, copyableFileSmbProperties: default, + nfsProperties: default, async: true, cancellationToken). ConfigureAwait(false); @@ -1647,6 +1667,9 @@ await StartCopyInternal( /// /// SMB properties to copy from the source file. /// + /// + /// NFS files only. NFS properties to set on the destination file. + /// /// /// Whether to invoke the operation asynchronously. /// @@ -1673,6 +1696,7 @@ private async Task> StartCopyInternal( bool? setArchiveAttribute, ShareFileRequestConditions conditions, CopyableFileSmbProperties? copyableFileSmbProperties, + FilePosixProperties nfsProperties, bool async, CancellationToken cancellationToken) { @@ -1723,7 +1747,7 @@ private async Task> StartCopyInternal( } else { - fileAttributes = smbProperties?.FileAttributes?.ToAttributesString(); + fileAttributes = smbProperties?.FileAttributes.ToAttributesString(); } string fileCreatedOn = null; @@ -1772,6 +1796,18 @@ private async Task> StartCopyInternal( ShareUriBuilder uriBuilder = new ShareUriBuilder(sourceUri); + ModeCopyMode? modeCopyMode = null; + if (nfsProperties?.FileMode != null) + { + modeCopyMode = ModeCopyMode.Override; + } + + OwnerCopyMode? ownerCopyMode = null; + if (nfsProperties?.Owner != null || nfsProperties?.Group != null) + { + ownerCopyMode = OwnerCopyMode.Override; + } + if (async) { response = await FileRestClient.StartCopyAsync( @@ -1780,6 +1816,11 @@ private async Task> StartCopyInternal( filePermission: filePermission, filePermissionFormat: filePermissionFormat, filePermissionKey: smbProperties?.FilePermissionKey, + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode?.ToOctalFileMode(), + fileModeCopyMode: modeCopyMode, + fileOwnerCopyMode: ownerCopyMode, copyFileSmbInfo: copyFileSmbInfo, shareFileRequestConditions: conditions, cancellationToken: cancellationToken) @@ -1793,6 +1834,11 @@ private async Task> StartCopyInternal( filePermission: filePermission, filePermissionFormat: filePermissionFormat, filePermissionKey: smbProperties?.FilePermissionKey, + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode?.ToOctalFileMode(), + fileModeCopyMode: modeCopyMode, + fileOwnerCopyMode: ownerCopyMode, copyFileSmbInfo: copyFileSmbInfo, shareFileRequestConditions: conditions, cancellationToken: cancellationToken); @@ -3395,6 +3441,7 @@ public virtual Response SetHttpHeaders( options?.SmbProperties, options?.FilePermission?.Permission, options?.FilePermission?.PermissionFormat, + options?.NfsProperties, conditions, async: false, cancellationToken) @@ -3437,6 +3484,7 @@ await SetHttpHeadersInternal( options?.SmbProperties, options?.FilePermission?.Permission, options?.FilePermission?.PermissionFormat, + options?.NfsProperties, conditions, async: true, cancellationToken) @@ -3497,6 +3545,7 @@ public virtual Response SetHttpHeaders( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, conditions, async: false, cancellationToken) @@ -3553,6 +3602,7 @@ public virtual Response SetHttpHeaders( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, conditions: default, async: false, cancellationToken) @@ -3613,6 +3663,7 @@ await SetHttpHeadersInternal( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, conditions, async: true, cancellationToken) @@ -3668,6 +3719,7 @@ await SetHttpHeadersInternal( smbProperties, filePermission, filePermissionFormat: default, + nfsProperties: default, conditions: default, async: true, cancellationToken) @@ -3699,6 +3751,9 @@ await SetHttpHeadersInternal( /// /// Optional file permission format. /// + /// + /// Optional NFS properties. + /// /// /// Optional to add conditions /// on creating the file. @@ -3724,6 +3779,7 @@ private async Task> SetHttpHeadersInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, + FilePosixProperties nfsProperties, ShareFileRequestConditions conditions, bool async, CancellationToken cancellationToken) @@ -3745,24 +3801,23 @@ private async Task> SetHttpHeadersInternal( FileSmbProperties smbProps = smbProperties ?? new FileSmbProperties(); ShareExtensions.AssertValidFilePermissionAndKey(filePermission, smbProps.FilePermissionKey); - if (filePermission == null && smbProps.FilePermissionKey == null) - { - filePermission = Constants.File.Preserve; - } ResponseWithHeaders response; if (async) { response = await FileRestClient.SetHttpHeadersAsync( - fileAttributes: smbProps.FileAttributes?.ToAttributesString() ?? Constants.File.Preserve, - fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString() ?? Constants.File.Preserve, - fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString() ?? Constants.File.Preserve, + fileAttributes: smbProps.FileAttributes.ToAttributesString(), + fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString(), fileContentLength: newSize, filePermission: filePermission, filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps.FilePermissionKey, fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode.ToOctalFileMode(), fileHttpHeaders: httpHeaders.ToFileHttpHeaders(), shareFileRequestConditions: conditions, cancellationToken: cancellationToken) @@ -3771,14 +3826,17 @@ private async Task> SetHttpHeadersInternal( else { response = FileRestClient.SetHttpHeaders( - fileAttributes: smbProps.FileAttributes?.ToAttributesString() ?? Constants.File.Preserve, - fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString() ?? Constants.File.Preserve, - fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString() ?? Constants.File.Preserve, + fileAttributes: smbProps.FileAttributes.ToAttributesString(), + fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString(), fileContentLength: newSize, filePermission: filePermission, filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps.FilePermissionKey, fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), + owner: nfsProperties?.Owner, + group: nfsProperties?.Group, + fileMode: nfsProperties?.FileMode.ToOctalFileMode(), fileHttpHeaders: httpHeaders.ToFileHttpHeaders(), shareFileRequestConditions: conditions, cancellationToken: cancellationToken); @@ -6595,7 +6653,7 @@ private async Task> RenameInternal( CopyFileSmbInfo copyFileSmbInfo = new CopyFileSmbInfo { - FileAttributes = options?.SmbProperties?.FileAttributes?.ToAttributesString(), + FileAttributes = options?.SmbProperties?.FileAttributes.ToAttributesString(), FileCreationTime = options?.SmbProperties?.FileCreatedOn.ToFileDateTimeString(), FileChangeTime = options?.SmbProperties?.FileChangedOn.ToFileDateTimeString(), FileLastWriteTime = options?.SmbProperties?.FileLastWrittenOn.ToFileDateTimeString(), @@ -6660,6 +6718,424 @@ private async Task> RenameInternal( } #endregion Rename + #region GetSymbolicLink + /// + /// Reads the value of the symbolic link. + /// Only applicable if this is pointed at an NFS symbolic link. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the symbolic link. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + /// https://github.com/Azure/azure-sdk-for-net/issues/46907 + internal virtual Response GetSymbolicLink( + CancellationToken cancellationToken = default) => + GetSymbolicLinkInternal( + async: false, + cancellationToken: cancellationToken) + .EnsureCompleted(); + + /// + /// Reads the value of the symbolic link. + /// Only applicable if this is pointed at an NFS symbolic link. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the symbolic link. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + /// https://github.com/Azure/azure-sdk-for-net/issues/46907 + internal virtual async Task> GetSymbolicLinkAsync( + CancellationToken cancellationToken = default) => + await GetSymbolicLinkInternal( + async: true, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + /// + /// Reads the value of the symbolic link. + /// Only applicable if this is pointed at an NFS symbolic link. + /// + /// + /// Whether to invoke the operation asynchronously. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the symbolic link. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + private async Task> GetSymbolicLinkInternal( + bool async, + CancellationToken cancellationToken) + { + using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(ShareFileClient))) + { + ClientConfiguration.Pipeline.LogMethodEnter(nameof(ShareFileClient), message: string.Empty); + + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(ShareFileClient)}.{nameof(GetSymbolicLink)}"); + + ResponseWithHeaders response; + + try + { + scope.Start(); + + if (async) + { + response = await FileRestClient.GetSymbolicLinkAsync( + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + else + { + response = FileRestClient.GetSymbolicLink( + cancellationToken: cancellationToken); + } + + return Response.FromValue( + response.ToFileSymbolicLinkInfo(), + response.GetRawResponse()); + } + catch (Exception ex) + { + ClientConfiguration.Pipeline.LogException(ex); + scope.Failed(ex); + throw; + } + finally + { + ClientConfiguration.Pipeline.LogMethodExit(nameof(ShareFileClient)); + scope.Dispose(); + } + } + } + #endregion + + #region CreateSymbolicLink + /// + /// NFS only. Creates a symoblic link to the file specified by path. + /// + /// + /// The absolution or relative path to the file to be linked to. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// state of the file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + /// https://github.com/Azure/azure-sdk-for-net/issues/46907 + internal virtual Response CreateSymbolicLink( + string linkText, + ShareFileCreateSymbolicLinkOptions options = default, + CancellationToken cancellationToken = default) => + CreateSymbolicLinkInternal( + linkText: linkText, + options: options, + async: false, + cancellationToken: cancellationToken) + .EnsureCompleted(); + + /// + /// NFS only. Creates a symoblic link to the file specified by path. + /// + /// + /// The absolution or relative path to the file to be linked to. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// state of the file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + /// https://github.com/Azure/azure-sdk-for-net/issues/46907 + internal virtual async Task> CreateSymbolicLinkAsync( + string linkText, + ShareFileCreateSymbolicLinkOptions options = default, + CancellationToken cancellationToken = default) => + await CreateSymbolicLinkInternal( + linkText: linkText, + options: options, + async: true, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + /// + /// NFS only. Creates a symoblic link to the file specified by path. + /// + /// + /// The absolution or relative path to the file to be linked to. + /// + /// + /// Optional parameters. + /// + /// + /// Whether to invoke the operation asynchronously. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// state of the file. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + private async Task> CreateSymbolicLinkInternal( + string linkText, + ShareFileCreateSymbolicLinkOptions options, + bool async, + CancellationToken cancellationToken) + { + using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(ShareFileClient))) + { + ClientConfiguration.Pipeline.LogMethodEnter( + nameof(ShareFileClient), + message: + $"{nameof(Uri)}: {Uri}\n"); + + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(ShareFileClient)}.{nameof(CreateSymbolicLink)}"); + + ResponseWithHeaders response; + + try + { + scope.Start(); + + if (async) + { + response = await FileRestClient.CreateSymbolicLinkAsync( + linkText: linkText, + metadata: options?.Metadata, + fileCreationTime: options?.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: options?.FileLastWrittenOn.ToFileDateTimeString(), + owner: options?.Owner, + group: options?.Group, + shareFileRequestConditions: options?.Conditions, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + else + { + response = FileRestClient.CreateSymbolicLink( + linkText: linkText, + metadata: options?.Metadata, + fileCreationTime: options?.FileCreatedOn.ToFileDateTimeString(), + fileLastWriteTime: options?.FileLastWrittenOn.ToFileDateTimeString(), + owner: options?.Owner, + group: options?.Group, + shareFileRequestConditions: options?.Conditions, + cancellationToken: cancellationToken); + } + + return Response.FromValue( + response.ToShareFileInfo(), + response.GetRawResponse()); + } + catch (Exception ex) + { + ClientConfiguration.Pipeline.LogException(ex); + scope.Failed(ex); + throw; + } + finally + { + ClientConfiguration.Pipeline.LogMethodExit(nameof(ShareFileClient)); + scope.Dispose(); + } + } + } + #endregion + + #region CreateHardLink + /// + /// NFS only. Creates a hard link to the file file specified by path. + /// + /// + /// Path of the file to create the hard link to, not including the share. + /// + /// + /// Optional to add conditions + /// on creating the hard link. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// state of the hard link. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual Response CreateHardLink( + string targetFile, + ShareFileRequestConditions conditions = default, + CancellationToken cancellationToken = default) + => CreateHardLinkInternal( + targetFile: targetFile, + conditions: conditions, + async: false, + cancellationToken: cancellationToken) + .EnsureCompleted(); + + /// + /// NFS only. Creates a hard link to the file file specified by path. + /// + /// + /// Path of the file to create the hard link to, not including the share. + /// + /// + /// Optional to add conditions + /// on creating the hard link. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// state of the hard link. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public async virtual Task> CreateHardLinkAsync( + string targetFile, + ShareFileRequestConditions conditions = default, + CancellationToken cancellationToken = default) + => await CreateHardLinkInternal( + targetFile: targetFile, + conditions: conditions, + async: true, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + /// + /// NFS only. Creates a hard link to the file file specified by path. + /// + /// + /// Path of the file to create the hard link to, not including the share. + /// + /// + /// Optional to add conditions + /// on creating the hard link. + /// + /// + /// Whether to invoke the operation asynchronously. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// state of the hard link. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + private async Task> CreateHardLinkInternal( + string targetFile, + ShareFileRequestConditions conditions, + bool async, + CancellationToken cancellationToken) + { + using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(ShareFileClient))) + { + ClientConfiguration.Pipeline.LogMethodEnter( + nameof(ShareFileClient), + message: + $"{nameof(Uri)}: {Uri}\n" + + $"{nameof(targetFile)}: {targetFile}"); + + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(ShareFileClient)}.{nameof(CreateHardLink)}"); + + try + { + scope.Start(); + + ResponseWithHeaders response; + + if (async) + { + response = await FileRestClient.CreateHardLinkAsync( + targetFile: targetFile, + shareFileRequestConditions: conditions, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + else + { + response = FileRestClient.CreateHardLink( + targetFile: targetFile, + shareFileRequestConditions: conditions, + cancellationToken: cancellationToken); + } + + return Response.FromValue( + response.ToShareFileInfo(), + response.GetRawResponse()); + } + catch (Exception ex) + { + ClientConfiguration.Pipeline.LogException(ex); + scope.Failed(ex); + throw; + } + finally + { + ClientConfiguration.Pipeline.LogMethodExit(nameof(ShareFileClient)); + scope.Dispose(); + } + } + } + #endregion + #region OpenWrite /// /// Opens a stream for writing to the file. @@ -6790,6 +7266,7 @@ private async Task OpenWriteInternal( smbProperties: default, filePermission: default, filePermissionFormat: default, + nfsProperties: default, conditions: options?.OpenConditions, async: async, cancellationToken: cancellationToken) @@ -6820,6 +7297,7 @@ private async Task OpenWriteInternal( smbProperties: default, filePermission: default, filePermissionFormat: default, + nfsProperties: default, conditions: options?.OpenConditions, async: async, cancellationToken: cancellationToken) diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Shared/ShareModelExtensions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Shared/ShareModelExtensions.cs index 21f3f36b1a13d..bcb41d34b9000 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Shared/ShareModelExtensions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Shared/ShareModelExtensions.cs @@ -17,9 +17,14 @@ internal static partial class ShareModelExtensions /// ToString /// /// string - public static string ToAttributesString(this NtfsFileAttributes attributes) + public static string ToAttributesString(this NtfsFileAttributes? attributes) { - var stringBuilder = new StringBuilder(); + if (attributes == null) + { + return null; + } + + StringBuilder stringBuilder = new StringBuilder(); if ((attributes & NtfsFileAttributes.ReadOnly) == NtfsFileAttributes.ReadOnly) { diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md index ed634ae302734..9780abf2b3633 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/ae95eb6a4701d844bada7d1c4f5ecf4a7444e5b8/specification/storage/data-plane/Microsoft.FileStorage/stable/2025-01-05/file.json + - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/9ba64409c50ecbddc416ae30432ab0d17d0d97d6/specification/storage/data-plane/Microsoft.FileStorage/stable/2025-05-05/file.json generation1-convenience-client: true # https://github.com/Azure/autorest/issues/4075 skip-semantics-validation: true diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ClientBuilderExtensions.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ClientBuilderExtensions.cs index 35f4f78c33aab..23449be0978b3 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ClientBuilderExtensions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ClientBuilderExtensions.cs @@ -59,6 +59,16 @@ public static ShareServiceClient GetServiceClient_OAuthAccount_SharedKey(this Sh public static ShareServiceClient GetServiceClient_PremiumFile(this ShareClientBuilder clientBuilder) => clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigPremiumFile); + public static ShareServiceClient GetServiceClient_PremiumFileOAuth( + this ShareClientBuilder clientBuilder, + TokenCredential tokenCredential, + ShareClientOptions options = default) + { + options ??= clientBuilder.GetOptions(); + options.ShareTokenIntent = ShareTokenIntent.Backup; + return clientBuilder.GetServiceClientFromOauthConfig(clientBuilder.Tenants.TestConfigPremiumFile, tokenCredential, options); + } + public static ShareServiceClient GetServiceClient_SoftDelete(this ShareClientBuilder clientBuilder) => clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigSoftDelete); @@ -67,13 +77,14 @@ public static async Task GetTestShareAsync( ShareServiceClient service = default, string shareName = default, IDictionary metadata = default, - ShareClientOptions options = default) + ShareClientOptions options = default, + bool nfs = false) { - service ??= clientBuilder.GetServiceClient_SharedKey(options); + service ??= nfs ? clientBuilder.GetServiceClient_PremiumFile() : clientBuilder.GetServiceClient_SharedKey(options); metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); shareName ??= clientBuilder.GetNewShareName(); ShareClient share = clientBuilder.AzureCoreRecordedTestBase.InstrumentClient(service.GetShareClient(shareName)); - return await DisposingShare.CreateAsync(share, metadata); + return await DisposingShare.CreateAsync(share, metadata, nfs); } public static async Task GetTestDirectoryAsync( @@ -81,9 +92,10 @@ public static async Task GetTestDirectoryAsync( ShareServiceClient service = default, string shareName = default, string directoryName = default, - ShareClientOptions options = default) + ShareClientOptions options = default, + bool nfs = false) { - DisposingShare test = await clientBuilder.GetTestShareAsync(service, shareName, options: options); + DisposingShare test = await clientBuilder.GetTestShareAsync(service, shareName, options: options, nfs: nfs); directoryName ??= clientBuilder.GetNewDirectoryName(); ShareDirectoryClient directory = clientBuilder.AzureCoreRecordedTestBase.InstrumentClient(test.Share.GetDirectoryClient(directoryName)); @@ -96,9 +108,10 @@ public static async Task GetTestFileAsync( string shareName = default, string directoryName = default, string fileName = default, - ShareClientOptions options = default) + ShareClientOptions options = default, + bool nfs = false) { - DisposingDirectory test = await clientBuilder.GetTestDirectoryAsync(service, shareName, directoryName, options); + DisposingDirectory test = await clientBuilder.GetTestDirectoryAsync(service, shareName, directoryName, options, nfs: nfs); fileName ??= clientBuilder.GetNewFileName(); ShareFileClient file = clientBuilder.AzureCoreRecordedTestBase.InstrumentClient(test.Directory.GetFileClient(fileName)); return await DisposingFile.CreateAsync(test, file); diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/DirectoryClientTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/DirectoryClientTests.cs index 66f7ea56a121a..bcf2ea683cc34 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/DirectoryClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/DirectoryClientTests.cs @@ -585,6 +585,44 @@ public async Task CreateAsync_TrailingDot(bool? allowTrailingDot) } } + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task CreateAsync_NFS() + { + await using DisposingDirectory test = await SharesClientBuilder.GetTestDirectoryAsync(nfs: true); + ShareClient share = test.Share; + + // Arrange + var name = GetNewDirectoryName(); + ShareDirectoryClient directory = InstrumentClient(share.GetDirectoryClient(name)); + + string owner = "345"; + string group = "123"; + string fileMode = "7777"; + + ShareDirectoryCreateOptions options = new ShareDirectoryCreateOptions + { + NfsProperties = new FilePosixProperties + { + Owner = owner, + Group = group, + FileMode = NfsFileMode.ParseOctalFileMode(fileMode) + } + }; + + // Act + Response response = await directory.CreateAsync(options); + + // Assert + Assert.AreEqual(NfsFileType.Directory, response.Value.NfsProperties.FileType); + Assert.AreEqual(owner, response.Value.NfsProperties.Owner); + Assert.AreEqual(group, response.Value.NfsProperties.Group); + Assert.AreEqual(fileMode, response.Value.NfsProperties.FileMode.ToOctalFileMode()); + + Assert.IsNull(response.Value.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); + } + [RecordedTest] public async Task CreateIfNotExists_NotExists() { @@ -982,6 +1020,27 @@ public async Task GetPropertiesAsync_TrailingDot() AssertPropertiesEqual(createResponse.Value.SmbProperties, getPropertiesResponse.Value.SmbProperties); } + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task GetPropertiesAsync_NFS() + { + // Arrange + await using DisposingDirectory test = await SharesClientBuilder.GetTestDirectoryAsync(nfs: true); + + // Act + Response response = await test.Directory.GetPropertiesAsync(); + + // Assert + Assert.AreEqual(NfsFileType.Directory, response.Value.NfsProperties.FileType); + Assert.AreEqual("0", response.Value.NfsProperties.Owner); + Assert.AreEqual("0", response.Value.NfsProperties.Group); + Assert.AreEqual("0755", response.Value.NfsProperties.FileMode.ToOctalFileMode()); + + Assert.IsNull(response.Value.NfsProperties.LinkCount); + Assert.IsNull(response.Value.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); + } + [RecordedTest] public async Task SetHttpHeadersAsync() { @@ -1221,6 +1280,40 @@ public async Task SetHttpHeadersAsync_TrailingDot() await directory.SetHttpHeadersAsync(); } + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task SetHttpHeadersAsync_NFS() + { + // Arrange + await using DisposingDirectory test = await SharesClientBuilder.GetTestDirectoryAsync(nfs: true); + + string owner = "345"; + string group = "123"; + string fileMode = "7777"; + + ShareDirectorySetHttpHeadersOptions options = new ShareDirectorySetHttpHeadersOptions + { + NfsProperties = new FilePosixProperties + { + Owner = owner, + Group = group, + FileMode = NfsFileMode.ParseOctalFileMode(fileMode) + } + }; + + // Act + Response response = await test.Directory.SetHttpHeadersAsync(options); + + // Assert + Assert.AreEqual(owner, response.Value.NfsProperties.Owner); + Assert.AreEqual(group, response.Value.NfsProperties.Group); + Assert.AreEqual(fileMode, response.Value.NfsProperties.FileMode.ToOctalFileMode()); + + Assert.IsNull(response.Value.NfsProperties.LinkCount); + Assert.IsNull(response.Value.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); + } + [RecordedTest] public async Task SetMetadataAsync() { diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/DisposingShare.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/DisposingShare.cs index 4389023663cf5..dbff1885fdbce 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/DisposingShare.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/DisposingShare.cs @@ -15,12 +15,18 @@ public class DisposingShare : IDisposingContainer public ShareClient Container { get; private set; } - public static async Task CreateAsync(ShareClient share, IDictionary metadata) + public static async Task CreateAsync(ShareClient share, IDictionary metadata, bool nfs = false) { ShareCreateOptions options = new ShareCreateOptions { Metadata = metadata }; + + if (nfs) + { + options.Protocols = ShareProtocols.Nfs; + } + await share.CreateIfNotExistsAsync(options); return new DisposingShare(share); } diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs index b1ce4cc25987a..b86ed902cfeaf 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs @@ -10,6 +10,7 @@ using System.Threading; using System.Threading.Tasks; using Azure.Core.TestFramework; +using Azure.Identity; using Azure.Storage.Files.Shares.Models; using Azure.Storage.Files.Shares.Specialized; using Azure.Storage.Sas; @@ -17,6 +18,7 @@ using Azure.Storage.Test.Shared; using Moq; using NUnit.Framework; +using NUnit.Framework.Internal; namespace Azure.Storage.Files.Shares.Tests { @@ -724,6 +726,46 @@ public async Task CreateAsync_TrailingDot(bool? allowTrailingDot) } } + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task CreateAsync_NFS() + { + await using DisposingDirectory test = await SharesClientBuilder.GetTestDirectoryAsync(nfs: true); + ShareDirectoryClient directory = test.Directory; + + // Arrange + string name = GetNewFileName(); + ShareFileClient file = InstrumentClient(directory.GetFileClient(name)); + + string owner = "345"; + string group = "123"; + string fileMode = "7777"; + + ShareFileCreateOptions options = new ShareFileCreateOptions + { + NfsProperties = new FilePosixProperties + { + Owner = owner, + Group = group, + FileMode = NfsFileMode.ParseOctalFileMode(fileMode) + } + }; + + // Act + Response response = await file.CreateAsync( + maxSize: Constants.MB, + options: options); + + // Assert + Assert.AreEqual(NfsFileType.Regular, response.Value.NfsProperties.FileType); + Assert.AreEqual(owner, response.Value.NfsProperties.Owner); + Assert.AreEqual(group, response.Value.NfsProperties.Group); + Assert.AreEqual(fileMode, response.Value.NfsProperties.FileMode.ToOctalFileMode()); + + Assert.IsNull(response.Value.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); + } + [RecordedTest] public async Task ExistsAsync_Exists() { @@ -1325,6 +1367,26 @@ public async Task GetPropertiesAsync_TrailingDot() AssertPropertiesEqual(createResponse.Value.SmbProperties, getPropertiesResponse.Value.SmbProperties); } + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task GetProperties_NFS() + { + await using DisposingFile test = await SharesClientBuilder.GetTestFileAsync(nfs: true); + + // Act + Response response = await test.File.GetPropertiesAsync(); + + // Assert + Assert.AreEqual(NfsFileType.Regular, response.Value.NfsProperties.FileType); + Assert.AreEqual("0", response.Value.NfsProperties.Owner); + Assert.AreEqual("0", response.Value.NfsProperties.Group); + Assert.AreEqual("0664", response.Value.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(1, response.Value.NfsProperties.LinkCount); + + Assert.IsNull(response.Value.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); + } + [RecordedTest] public async Task SetHttpHeadersAsync() { @@ -1712,6 +1774,39 @@ public async Task SetHttpHeadersAsync_TrailingDot() Response response = await file.SetHttpHeadersAsync(setHttpHeadersOptions); } + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task SetHttpHeadersAsync_NFS() + { + await using DisposingFile test = await SharesClientBuilder.GetTestFileAsync(nfs: true); + + string owner = "345"; + string group = "123"; + string fileMode = "7777"; + + ShareFileSetHttpHeadersOptions options = new ShareFileSetHttpHeadersOptions + { + NfsProperties = new FilePosixProperties + { + Owner = owner, + Group = group, + FileMode = NfsFileMode.ParseOctalFileMode(fileMode) + } + }; + + // Act + Response response = await test.File.SetHttpHeadersAsync(options); + + // Assert + Assert.AreEqual(owner, response.Value.NfsProperties.Owner); + Assert.AreEqual(group, response.Value.NfsProperties.Group); + Assert.AreEqual(fileMode, response.Value.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(1, response.Value.NfsProperties.LinkCount); + + Assert.IsNull(response.Value.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); + } + [RecordedTest] public async Task DeleteAsync() { @@ -2446,6 +2541,59 @@ await TestHelper.AssertExpectedExceptionAsync( } } + [RecordedTest] + [TestCase(false)] + [TestCase(true)] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task StartCopy_NFS(bool overwriteOwnerAndMode) + { + // Arrange + await using DisposingFile source = await SharesClientBuilder.GetTestFileAsync(nfs: true); + await using DisposingFile destination = await SharesClientBuilder.GetTestFileAsync(nfs: true); + + byte[] data = GetRandomBuffer(Constants.KB); + using Stream stream = new MemoryStream(data); + await source.File.UploadRangeAsync( + range: new HttpRange(0, Constants.KB), + content: stream); + + Response sourceProperties = await source.File.GetPropertiesAsync(); + + string owner; + string group; + NfsFileMode fileMode; + + ShareFileCopyOptions options = new ShareFileCopyOptions + { + NfsProperties = new FilePosixProperties() + }; + + if (overwriteOwnerAndMode) + { + owner = "54321"; + group = "12345"; + fileMode = NfsFileMode.ParseOctalFileMode("7777"); + options.NfsProperties.Owner = owner; + options.NfsProperties.Group = group; + options.NfsProperties.FileMode = fileMode; + } + else + { + owner = sourceProperties.Value.NfsProperties.Owner; + fileMode = sourceProperties.Value.NfsProperties.FileMode; + group = sourceProperties.Value.NfsProperties.Group; + } + + // Act + await destination.File.StartCopyAsync(source.File.Uri, options); + Response destinationProperties = await destination.File.GetPropertiesAsync(); + + // Assert + Assert.AreEqual(owner, destinationProperties.Value.NfsProperties.Owner); + Assert.AreEqual(group, destinationProperties.Value.NfsProperties.Group); + Assert.AreEqual(fileMode.ToOctalFileMode(), destinationProperties.Value.NfsProperties.FileMode.ToOctalFileMode()); + } + [RecordedTest] public async Task AbortCopyAsync() { @@ -3073,6 +3221,37 @@ await file.UploadRangeAsync( } } + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task DownloadAsync_NFS() + { + // Arrange + var data = GetRandomBuffer(Constants.KB); + + await using DisposingFile test = await SharesClientBuilder.GetTestFileAsync(nfs: true); + ShareFileClient file = test.File; + + using Stream stream = new MemoryStream(data); + await file.UploadRangeAsync( + range: new HttpRange(0, Constants.KB), + content: stream); + + // Act + Response response = await file.DownloadAsync(new ShareFileDownloadOptions + { + Range = new HttpRange(0, Constants.KB) + }); + + // Assert + Assert.AreEqual("0", response.Value.Details.NfsProperties.Owner); + Assert.AreEqual("0", response.Value.Details.NfsProperties.Group); + Assert.AreEqual("0664", response.Value.Details.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(1, response.Value.Details.NfsProperties.LinkCount); + + Assert.IsNull(response.Value.Details.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.Details.SmbProperties.FilePermissionKey); + } + [RecordedTest] public async Task GetRangeListAsync() { @@ -6622,6 +6801,182 @@ public async Task RenameAsync_OAuth() TestHelper.AssertSequenceEqual(data, actual.ToArray()); } + [RecordedTest] + [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/46907")] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task CreateGetSymbolicLinkAsync() + { + // Arrange + await using DisposingDirectory test = await SharesClientBuilder.GetTestDirectoryAsync(nfs: true); + ShareDirectoryClient directory = test.Directory; + + ShareFileClient source = InstrumentClient(await directory.CreateFileAsync(GetNewFileName(), maxSize: Constants.KB)); + ShareFileClient symlink = InstrumentClient(directory.GetFileClient(GetNewFileName())); + + IDictionary metdata = BuildMetadata(); + string owner = "345"; + string group = "123"; + DateTimeOffset fileCreatedOn = new DateTimeOffset(2024, 10, 15, 0, 0, 0, TimeSpan.Zero); + DateTimeOffset fileLastWrittenOn = new DateTimeOffset(2025, 5, 2, 0, 0, 0, TimeSpan.Zero); + + ShareFileCreateSymbolicLinkOptions options = new ShareFileCreateSymbolicLinkOptions + { + Metadata = metdata, + FileCreatedOn = fileCreatedOn, + FileLastWrittenOn = fileLastWrittenOn, + Owner = owner, + Group = group + }; + + // Act + Response response = await symlink.CreateSymbolicLinkAsync( + linkText: source.Uri.ToString(), + options: options); + + // Assert + Assert.AreEqual(NfsFileType.Symlink, response.Value.NfsProperties.FileType); + Assert.AreEqual(owner, response.Value.NfsProperties.Owner); + Assert.AreEqual(group, response.Value.NfsProperties.Group); + Assert.AreEqual(fileCreatedOn, response.Value.SmbProperties.FileCreatedOn); + Assert.AreEqual(fileLastWrittenOn, response.Value.SmbProperties.FileLastWrittenOn); + + Assert.IsNull(response.Value.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); + + Assert.IsNotNull(response.Value.SmbProperties.FileId); + Assert.IsNotNull(response.Value.SmbProperties.ParentId); + + // Act + Response getSymLinkResponse = await symlink.GetSymbolicLinkAsync(); + + // Assert + Assert.AreNotEqual(default, getSymLinkResponse.Value.ETag); + Assert.AreNotEqual(default, getSymLinkResponse.Value.LastModified); + Assert.AreEqual(source.Uri.ToString(), getSymLinkResponse.Value.LinkText); + } + + [RecordedTest] + [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/46907")] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task CreateGetSymbolicLinkAsync_Error() + { + // Arrange + await using DisposingShare test = await SharesClientBuilder.GetTestShareAsync(nfs: true); + // Note that the parent directory was not created in this test case. + ShareDirectoryClient directory = InstrumentClient(test.Share.GetDirectoryClient(GetNewDirectoryName())); + + ShareFileClient source = InstrumentClient(directory.GetFileClient(GetNewFileName())); + ShareFileClient symlink = InstrumentClient(directory.GetFileClient(GetNewFileName())); + + // Act + await TestHelper.AssertExpectedExceptionAsync( + symlink.CreateSymbolicLinkAsync(linkText: source.Uri.ToString()), + e => Assert.AreEqual("ParentNotFound", e.ErrorCode)); + + await TestHelper.AssertExpectedExceptionAsync( + symlink.GetSymbolicLinkAsync(), + e => Assert.AreEqual("ParentNotFound", e.ErrorCode)); + } + + [RecordedTest] + [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/46907")] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task CreateGetSymbolicLinkAsync_OAuth() + { + // Arrange + ShareServiceClient oauthServiceClient = GetServiceClient_PremiumFileOAuth(); + await using DisposingDirectory test = await SharesClientBuilder.GetTestDirectoryAsync( + service: oauthServiceClient, + nfs: true); + ShareDirectoryClient directory = test.Directory; + + ShareFileClient source = InstrumentClient(await directory.CreateFileAsync(GetNewFileName(), maxSize: Constants.KB)); + ShareFileClient symlink = InstrumentClient(directory.GetFileClient(GetNewFileName())); + + // Act + await symlink.CreateSymbolicLinkAsync(linkText: source.Uri.ToString()); + await symlink.GetSymbolicLinkAsync(); + } + + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task CreateHardLinkAsync() + { + // Arrange + await using DisposingDirectory test = await SharesClientBuilder.GetTestDirectoryAsync(nfs: true); + ShareDirectoryClient directory = test.Directory; + + ShareFileClient source = InstrumentClient(await directory.CreateFileAsync(GetNewFileName(), maxSize: Constants.KB)); + ShareLeaseClient leaseClient = InstrumentClient(test.Share.GetShareLeaseClient(Recording.Random.NewGuid().ToString())); + ShareFileLease lease = await leaseClient.AcquireAsync(); + try + { + ShareFileClient hardLink = InstrumentClient(directory.GetFileClient(GetNewFileName())); + + // Act + Response response = await hardLink.CreateHardLinkAsync( + targetFile: $"{directory.Name}/{source.Name}", + conditions: new ShareFileRequestConditions() { LeaseId = lease.LeaseId }); + + // Assert + Assert.AreEqual(NfsFileType.Regular, response.Value.NfsProperties.FileType); + Assert.AreEqual("0", response.Value.NfsProperties.Owner); + Assert.AreEqual("0", response.Value.NfsProperties.Group); + Assert.AreEqual("0664", response.Value.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(2, response.Value.NfsProperties.LinkCount); + + Assert.IsNotNull(response.Value.SmbProperties.FileCreatedOn); + Assert.IsNotNull(response.Value.SmbProperties.FileLastWrittenOn); + Assert.IsNotNull(response.Value.SmbProperties.FileChangedOn); + Assert.IsNotNull(response.Value.SmbProperties.FileId); + Assert.IsNotNull(response.Value.SmbProperties.ParentId); + + Assert.IsNull(response.Value.SmbProperties.FileAttributes); + Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); + } + finally + { + await leaseClient.ReleaseAsync(); + } + } + + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task CreateHardLinkAsync_OAuth() + { + // Arrange + ShareServiceClient oauthServiceClient = GetServiceClient_PremiumFileOAuth(); + await using DisposingDirectory test = await SharesClientBuilder.GetTestDirectoryAsync( + service: oauthServiceClient, + nfs: true); + ShareDirectoryClient directory = test.Directory; + + ShareFileClient source = InstrumentClient(await directory.CreateFileAsync(GetNewFileName(), maxSize: Constants.KB)); + ShareFileClient hardLink = InstrumentClient(directory.GetFileClient(GetNewFileName())); + + // Act + Response response = await hardLink.CreateHardLinkAsync( + targetFile: $"{directory.Name}/{source.Name}"); + } + + [RecordedTest] + [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2025_05_05)] + public async Task CreateGetHardLinkAsync_Error() + { + // Arrange + await using DisposingShare test = await SharesClientBuilder.GetTestShareAsync(nfs: true); + // Note that the parent directory was not created in this test case. + ShareDirectoryClient directory = InstrumentClient(test.Share.GetDirectoryClient(GetNewDirectoryName())); + + ShareFileClient source = InstrumentClient(directory.GetFileClient(GetNewFileName())); + ShareFileClient hardLink = InstrumentClient(directory.GetFileClient(GetNewFileName())); + + // Act + await TestHelper.AssertExpectedExceptionAsync( + hardLink.CreateHardLinkAsync(targetFile: $"{directory.Name}/{source.Name}"), + e => Assert.AreEqual("ParentNotFound", e.ErrorCode)); + } + #region GenerateSasTests [RecordedTest] public void CanGenerateSas_ClientConstructors() diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/FileTestBase.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/FileTestBase.cs index a2b901bf2e383..8771d81258c8a 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/FileTestBase.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/FileTestBase.cs @@ -110,6 +110,9 @@ public ShareServiceClient GetServiceClient_FileServiceSasFile(string shareName, public ShareServiceClient GetServiceClient_OAuth() => SharesClientBuilder.GetServiceClient_OAuth(TestEnvironment.Credential); + public ShareServiceClient GetServiceClient_PremiumFileOAuth() + => SharesClientBuilder.GetServiceClient_PremiumFileOAuth(TestEnvironment.Credential); + public SasQueryParameters GetNewAccountSasCredentials(StorageSharedKeyCredential sharedKeyCredentials = default, AccountSasResourceTypes resourceTypes = AccountSasResourceTypes.Container, AccountSasPermissions permissions = AccountSasPermissions.Create | AccountSasPermissions.Delete) diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/NfsFileModeTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/NfsFileModeTests.cs new file mode 100644 index 0000000000000..aa9da871bd8e7 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/NfsFileModeTests.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Storage.Files.Shares.Models; +using NUnit.Framework; + +namespace Azure.Storage.Files.Shares.Tests +{ + public class NfsFileModeTests + { + [Test] + [TestCase("0000")] + [TestCase("1111")] + [TestCase("2222")] + [TestCase("3333")] + [TestCase("4444")] + [TestCase("5555")] + [TestCase("6666")] + [TestCase("7777")] + [TestCase("0124")] + [TestCase("4210")] + [TestCase("1357")] + [TestCase("7531")] + public void OctalPermissionsRoundTrip(string s) + { + // Act + NfsFileMode fileMode = NfsFileMode.ParseOctalFileMode(s); + string output = fileMode.ToOctalFileMode(); + + // Assert + Assert.AreEqual(s, output); + } + + [Test] + [TestCase("---------")] + [TestCase("rwxrwxrwx")] + [TestCase("r---w---x")] + [TestCase("rwsrwsrwt")] + [TestCase("rwSrwSrwT")] + public void SymbolicPermissionsRoundTrip(string s) + { + // Act + NfsFileMode fileMode = NfsFileMode.ParseSymbolicFileMode(s); + string output = fileMode.ToSymbolicFileMode(); + + // Assert + Assert.AreEqual(s, output); + } + } +} From b4400d537e012ef92782883aefa4c1b932790bd3 Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:01:49 -0800 Subject: [PATCH 05/10] STG 97 changelogs (#47186) --- sdk/storage/Azure.Storage.Blobs.Batch/CHANGELOG.md | 7 +------ .../Azure.Storage.Blobs.ChangeFeed/CHANGELOG.md | 7 +------ sdk/storage/Azure.Storage.Blobs/CHANGELOG.md | 7 +------ sdk/storage/Azure.Storage.Common/CHANGELOG.md | 7 +------ sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md | 7 +------ sdk/storage/Azure.Storage.Files.Shares/CHANGELOG.md | 11 +++++++---- sdk/storage/Azure.Storage.Queues/CHANGELOG.md | 5 +---- 7 files changed, 13 insertions(+), 38 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/CHANGELOG.md b/sdk/storage/Azure.Storage.Blobs.Batch/CHANGELOG.md index 440e95b4ea6e1..f6bcf9e2b44f6 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Blobs.Batch/CHANGELOG.md @@ -3,12 +3,7 @@ ## 12.21.0-beta.1 (Unreleased) ### Features Added - -### Breaking Changes - -### Bugs Fixed - -### Other Changes +- Added support for service version 2025-05-05. ## 12.20.0 (2024-11-12) diff --git a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/CHANGELOG.md b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/CHANGELOG.md index eaa0dca1ce579..af2ef51643b16 100644 --- a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/CHANGELOG.md @@ -3,12 +3,7 @@ ## 12.0.0-preview.53 (Unreleased) ### Features Added - -### Breaking Changes - -### Bugs Fixed - -### Other Changes +- Added support for service version 2025-05-05. ## 12.0.0-preview.52 (2024-11-12) diff --git a/sdk/storage/Azure.Storage.Blobs/CHANGELOG.md b/sdk/storage/Azure.Storage.Blobs/CHANGELOG.md index e59f18e7fbd38..3070cac9a9a6d 100644 --- a/sdk/storage/Azure.Storage.Blobs/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Blobs/CHANGELOG.md @@ -3,12 +3,7 @@ ## 12.24.0-beta.1 (Unreleased) ### Features Added - -### Breaking Changes - -### Bugs Fixed - -### Other Changes +- Added support for service version 2025-05-05. ## 12.23.0 (2024-11-12) diff --git a/sdk/storage/Azure.Storage.Common/CHANGELOG.md b/sdk/storage/Azure.Storage.Common/CHANGELOG.md index a2d9f13d92892..dc1764d16bc6e 100644 --- a/sdk/storage/Azure.Storage.Common/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Common/CHANGELOG.md @@ -3,12 +3,7 @@ ## 12.23.0-beta.1 (Unreleased) ### Features Added - -### Breaking Changes - -### Bugs Fixed - -### Other Changes +- This release contains bug fixes to improve quality. ## 12.22.0 (2024-11-12) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md b/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md index 27804061ada50..a5d4e56442265 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md @@ -3,12 +3,7 @@ ## 12.22.0-beta.1 (Unreleased) ### Features Added - -### Breaking Changes - -### Bugs Fixed - -### Other Changes +- Added support for service version 2025-05-05. ## 12.21.0 (2024-11-12) diff --git a/sdk/storage/Azure.Storage.Files.Shares/CHANGELOG.md b/sdk/storage/Azure.Storage.Files.Shares/CHANGELOG.md index b458e72774070..2d5e90999f73d 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Files.Shares/CHANGELOG.md @@ -3,12 +3,15 @@ ## 12.22.0-beta.1 (Unreleased) ### Features Added +- Added support for service version 2025-05-05. +- Added support for NFS over REST. ### Breaking Changes - -### Bugs Fixed - -### Other Changes +- The following APIs no longer send the x-ms-file-permission-key, x-ms-file-attributes, x-ms-file-creation-time, and x-ms-file-last-write-time request headers by default. These headers have been optional in the REST API since x-ms-version 2021-06-08: + - ShareFileClient.Create() and .CreateAsync() + - ShareFileClient.SetHttpHeaders() and .SetHttpHeadersAsync() + - ShareDirectoryClient.Create() and .CreateAsync() + - ShareDirectoryClient.SetHttpHeaders() and .SetHttpHeadersAsync() ## 12.21.0 (2024-11-12) diff --git a/sdk/storage/Azure.Storage.Queues/CHANGELOG.md b/sdk/storage/Azure.Storage.Queues/CHANGELOG.md index 33ad2c23f0700..38d8078961ca4 100644 --- a/sdk/storage/Azure.Storage.Queues/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Queues/CHANGELOG.md @@ -3,10 +3,7 @@ ## 12.22.0-beta.1 (Unreleased) ### Features Added - -### Breaking Changes - -### Bugs Fixed +- Added support for service version 2025-05-05. ### Other Changes From 67b32c03d18d0e4a49becb7bc212abd400557558 Mon Sep 17 00:00:00 2001 From: Sean McCullough Date: Tue, 19 Nov 2024 11:53:35 -0600 Subject: [PATCH 06/10] Export API --- .../api/Azure.Storage.Blobs.net8.0.cs | 1 + .../api/Azure.Storage.Common.net8.0.cs | 2 +- .../Azure.Storage.Files.DataLake.net8.0.cs | 1 + .../api/Azure.Storage.Files.Shares.net8.0.cs | 75 +++++++++++++++++++ .../api/Azure.Storage.Queues.net8.0.cs | 1 + 5 files changed, 79 insertions(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs index 167e4b1d926a1..c2c4e23aecd48 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs @@ -88,6 +88,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class BlobContainerClient diff --git a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.net8.0.cs b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.net8.0.cs index 121838723ee4f..aebd7cfb6a16d 100644 --- a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.net8.0.cs +++ b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.net8.0.cs @@ -183,7 +183,7 @@ public enum SasProtocol } public partial class SasQueryParameters { - public const string DefaultSasVersion = "2025-01-05"; + public const string DefaultSasVersion = "2025-05-05"; protected SasQueryParameters() { } protected SasQueryParameters(System.Collections.Generic.IDictionary values) { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs index 373451bb75efa..f743947be089d 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs @@ -36,6 +36,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class DataLakeDirectoryClient : Azure.Storage.Files.DataLake.DataLakePathClient diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs index 9391c7f96c977..8cf51e34818aa 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs @@ -149,6 +149,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class ShareDirectoryClient @@ -267,6 +268,8 @@ public ShareFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCredent public virtual System.Threading.Tasks.Task> CreateAsync(long maxSize, Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders httpHeaders, System.Collections.Generic.IDictionary metadata, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties, string filePermission, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions, System.Threading.CancellationToken cancellationToken) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> CreateAsync(long maxSize, Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders httpHeaders, System.Collections.Generic.IDictionary metadata, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties, string filePermission, System.Threading.CancellationToken cancellationToken) { throw null; } + public virtual Azure.Response CreateHardLink(string targetFile, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> CreateHardLinkAsync(string targetFile, Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response Delete(Azure.Storage.Files.Shares.Models.ShareFileRequestConditions conditions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Delete(System.Threading.CancellationToken cancellationToken) { throw null; } @@ -494,6 +497,15 @@ public enum FilePermissionFormat Sddl = 0, Binary = 1, } + public partial class FilePosixProperties + { + public FilePosixProperties() { } + public Azure.Storage.Files.Shares.Models.NfsFileMode FileMode { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.NfsFileType? FileType { get { throw null; } } + public string Group { get { throw null; } set { } } + public long? LinkCount { get { throw null; } } + public string Owner { get { throw null; } set { } } + } public partial class FileSmbProperties { public FileSmbProperties() { } @@ -511,16 +523,59 @@ public FileSmbProperties() { } } public static partial class FilesModelFactory { + public static Azure.Storage.Files.Shares.Models.FilePosixProperties FileNfsProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileItem ShareFileItem(bool isDirectory = false, string name = null, long? fileSize = default(long?), string id = null, Azure.Storage.Files.Shares.Models.ShareFileItemProperties properties = null, Azure.Storage.Files.Shares.Models.NtfsFileAttributes? fileAttributes = default(Azure.Storage.Files.Shares.Models.NtfsFileAttributes?), string permissionKey = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata, Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileDownloadInfo StorageFileDownloadInfo(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletionTime = default(System.DateTimeOffset), string copyStatusDescription = null, string contentDisposition = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServerEncrypted = false, string cacheControl = null, string fileAttributes = null, System.Collections.Generic.IEnumerable contentEncoding = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), byte[] contentHash = null, System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string contentRange = null, string filePermissionKey = null, string contentType = null, string fileId = null, long contentLength = (long)0, string fileParentId = null, System.Collections.Generic.IDictionary metadata = null, System.IO.Stream content = null, string copyId = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentType, string contentRange, Azure.ETag eTag, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, string acceptRanges, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, byte[] fileContentHash, bool isServiceEncrypted) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, string filePermissionKey = null, string fileAttributes = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string fileId = null, string fileParentId = null, Azure.Storage.Files.Shares.Models.NfsFileMode nfsFileMode = null, string owner = null, string group = null, Azure.Storage.Files.Shares.Models.NfsFileType nfsFileType = default(Azure.Storage.Files.Shares.Models.NfsFileType)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileItem StorageFileItem(bool isDirectory, string name, long? fileSize) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, Azure.Storage.Files.Shares.Models.NtfsFileAttributes fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } } + public partial class NfsFileMode + { + public NfsFileMode() { } + public bool EffectiveGroupIdentity { get { throw null; } set { } } + public bool EffectiveUserIdentity { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Group { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Other { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.RolePermissions Owner { get { throw null; } set { } } + public bool StickyBit { get { throw null; } set { } } + public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseOctalFileMode(string modeString) { throw null; } + public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseSymbolicFileMode(string modeString) { throw null; } + public string ToOctalFileMode() { throw null; } + public string ToSymbolicFileMode() { throw null; } + } + [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] + public readonly partial struct NfsFileType : System.IEquatable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public NfsFileType(string value) { throw null; } + public static Azure.Storage.Files.Shares.Models.NfsFileType Directory { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType Regular { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType Symlink { get { throw null; } } + public bool Equals(Azure.Storage.Files.Shares.Models.NfsFileType other) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override bool Equals(object obj) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override int GetHashCode() { throw null; } + public static bool operator ==(Azure.Storage.Files.Shares.Models.NfsFileType left, Azure.Storage.Files.Shares.Models.NfsFileType right) { throw null; } + public static implicit operator Azure.Storage.Files.Shares.Models.NfsFileType (string value) { throw null; } + public static bool operator !=(Azure.Storage.Files.Shares.Models.NfsFileType left, Azure.Storage.Files.Shares.Models.NfsFileType right) { throw null; } + public override string ToString() { throw null; } + } [System.FlagsAttribute] public enum NtfsFileAttributes { @@ -545,6 +600,14 @@ public partial class PermissionInfo internal PermissionInfo() { } public string FilePermissionKey { get { throw null; } } } + [System.FlagsAttribute] + public enum RolePermissions + { + None = 0, + Execute = 1, + Write = 2, + Read = 4, + } public partial class ShareAccessPolicy { public ShareAccessPolicy() { } @@ -629,6 +692,7 @@ public partial class ShareDirectoryCreateOptions public ShareDirectoryCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryGetFilesAndDirectoriesOptions @@ -643,6 +707,7 @@ public partial class ShareDirectoryInfo internal ShareDirectoryInfo() { } public Azure.ETag ETag { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryProperties @@ -652,12 +717,14 @@ internal ShareDirectoryProperties() { } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectorySetHttpHeadersOptions { public ShareDirectorySetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -767,6 +834,7 @@ public ShareFileCopyOptions() { } public Azure.Storage.Files.Shares.Models.PermissionCopyMode? FilePermissionCopyMode { get { throw null; } set { } } public bool? IgnoreReadOnly { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FilePermissionFormat? PermissionFormat { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.CopyableFileSmbProperties SmbPropertiesToCopy { get { throw null; } set { } } @@ -777,6 +845,7 @@ public ShareFileCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadDetails @@ -802,6 +871,7 @@ internal ShareFileDownloadDetails() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadInfo : System.IDisposable @@ -879,6 +949,7 @@ internal ShareFileInfo() { } public Azure.ETag ETag { get { throw null; } } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileItem @@ -967,6 +1038,7 @@ internal ShareFileProperties() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileRangeInfo @@ -1008,6 +1080,7 @@ public ShareFileSetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public long? NewSize { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.FlagsAttribute] @@ -1227,6 +1300,8 @@ public ShareSmbSettings() { } public static partial class SharesModelFactory { public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } } public partial class ShareSnapshotInfo diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net8.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net8.0.cs index 9f440eb3639d7..3bcbeb0dc9fb1 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net8.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net8.0.cs @@ -108,6 +108,7 @@ public enum ServiceVersion V2024_08_04 = 23, V2024_11_04 = 24, V2025_01_05 = 25, + V2025_05_05 = 26, } } public partial class QueueMessageDecodingFailedEventArgs : Azure.SyncAsyncEventArgs From ca8302c75015a59ecdc34fe494e89994acef728c Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Wed, 4 Dec 2024 11:04:47 -0600 Subject: [PATCH 07/10] Arch Board comments (#47409) --- .../api/Azure.Storage.Files.Shares.net6.0.cs | 41 ++++---- .../api/Azure.Storage.Files.Shares.net8.0.cs | 41 ++++---- ...ure.Storage.Files.Shares.netstandard2.0.cs | 41 ++++---- .../src/Models/FilePosixProperties.cs | 2 +- .../src/Models/FileSmbProperties.cs | 26 ++++- .../src/Models/NfsFileMode.cs | 9 +- ...Permissions.cs => PosixRolePermissions.cs} | 2 +- .../src/Models/RolePermissionExtensions.cs | 38 +++---- .../src/Models/ShareDirectoryCreateOptions.cs | 2 +- .../src/Models/ShareDirectoryInfo.cs | 14 ++- .../src/Models/ShareDirectoryProperties.cs | 6 +- .../ShareDirectorySetHttpHeadersOptions.cs | 2 +- .../src/Models/ShareFileCopyOptions.cs | 2 +- .../src/Models/ShareFileCreateOptions.cs | 2 +- .../src/Models/ShareFileDownloadDetails.cs | 6 +- .../src/Models/ShareFileInfo.cs | 4 +- .../src/Models/ShareFileProperties.cs | 6 +- .../Models/ShareFileSetHttpHeadersOptions.cs | 2 +- .../src/ShareClient.cs | 8 +- .../src/ShareDirectoryClient.cs | 62 ++++++------ .../src/ShareExtensions.cs | 18 ++-- .../src/ShareFileClient.cs | 98 ++++++++++--------- .../tests/DirectoryClientTests.cs | 30 +++--- .../tests/FileClientTests.cs | 74 +++++++------- 24 files changed, 288 insertions(+), 248 deletions(-) rename sdk/storage/Azure.Storage.Files.Shares/src/Models/{RolePermissions.cs => PosixRolePermissions.cs} (94%) diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs index 5fc271b35181b..b72c7b71f7373 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs @@ -523,13 +523,15 @@ public FileSmbProperties() { } } public static partial class FilesModelFactory { - public static Azure.Storage.Files.Shares.Models.FilePosixProperties FileNfsProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } + public static Azure.Storage.Files.Shares.Models.FilePosixProperties FilePosixProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } + public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileItem ShareFileItem(bool isDirectory = false, string name = null, long? fileSize = default(long?), string id = null, Azure.Storage.Files.Shares.Models.ShareFileItemProperties properties = null, Azure.Storage.Files.Shares.Models.NtfsFileAttributes? fileAttributes = default(Azure.Storage.Files.Shares.Models.NtfsFileAttributes?), string permissionKey = null) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata, Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileDownloadInfo StorageFileDownloadInfo(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletionTime = default(System.DateTimeOffset), string copyStatusDescription = null, string contentDisposition = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServerEncrypted = false, string cacheControl = null, string fileAttributes = null, System.Collections.Generic.IEnumerable contentEncoding = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), byte[] contentHash = null, System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string contentRange = null, string filePermissionKey = null, string contentType = null, string fileId = null, long contentLength = (long)0, string fileParentId = null, System.Collections.Generic.IDictionary metadata = null, System.IO.Stream content = null, string copyId = null) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentType, string contentRange, Azure.ETag eTag, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, string acceptRanges, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, byte[] fileContentHash, bool isServiceEncrypted) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] @@ -537,7 +539,7 @@ public static partial class FilesModelFactory public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, string filePermissionKey = null, string fileAttributes = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string fileId = null, string fileParentId = null, Azure.Storage.Files.Shares.Models.NfsFileMode nfsFileMode = null, string owner = null, string group = null, Azure.Storage.Files.Shares.Models.NfsFileType nfsFileType = default(Azure.Storage.Files.Shares.Models.NfsFileType)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileItem StorageFileItem(bool isDirectory, string name, long? fileSize) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, Azure.Storage.Files.Shares.Models.NtfsFileAttributes fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] @@ -548,13 +550,14 @@ public partial class NfsFileMode public NfsFileMode() { } public bool EffectiveGroupIdentity { get { throw null; } set { } } public bool EffectiveUserIdentity { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Group { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Other { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Owner { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Group { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Other { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Owner { get { throw null; } set { } } public bool StickyBit { get { throw null; } set { } } public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseOctalFileMode(string modeString) { throw null; } public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseSymbolicFileMode(string modeString) { throw null; } public string ToOctalFileMode() { throw null; } + public override string ToString() { throw null; } public string ToSymbolicFileMode() { throw null; } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -601,7 +604,7 @@ internal PermissionInfo() { } public string FilePermissionKey { get { throw null; } } } [System.FlagsAttribute] - public enum RolePermissions + public enum PosixRolePermissions { None = 0, Execute = 1, @@ -692,7 +695,7 @@ public partial class ShareDirectoryCreateOptions public ShareDirectoryCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryGetFilesAndDirectoriesOptions @@ -707,7 +710,7 @@ public partial class ShareDirectoryInfo internal ShareDirectoryInfo() { } public Azure.ETag ETag { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryProperties @@ -717,14 +720,14 @@ internal ShareDirectoryProperties() { } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectorySetHttpHeadersOptions { public ShareDirectorySetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -834,8 +837,8 @@ public ShareFileCopyOptions() { } public Azure.Storage.Files.Shares.Models.PermissionCopyMode? FilePermissionCopyMode { get { throw null; } set { } } public bool? IgnoreReadOnly { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FilePermissionFormat? PermissionFormat { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.CopyableFileSmbProperties SmbPropertiesToCopy { get { throw null; } set { } } } @@ -845,7 +848,7 @@ public ShareFileCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadDetails @@ -871,7 +874,7 @@ internal ShareFileDownloadDetails() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadInfo : System.IDisposable @@ -949,7 +952,7 @@ internal ShareFileInfo() { } public Azure.ETag ETag { get { throw null; } } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileItem @@ -1037,7 +1040,7 @@ internal ShareFileProperties() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileRangeInfo @@ -1079,7 +1082,7 @@ public ShareFileSetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public long? NewSize { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.FlagsAttribute] @@ -1298,8 +1301,8 @@ public ShareSmbSettings() { } } public static partial class SharesModelFactory { + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs index 8cf51e34818aa..238552cdc8567 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs @@ -523,13 +523,15 @@ public FileSmbProperties() { } } public static partial class FilesModelFactory { - public static Azure.Storage.Files.Shares.Models.FilePosixProperties FileNfsProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } + public static Azure.Storage.Files.Shares.Models.FilePosixProperties FilePosixProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } + public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileItem ShareFileItem(bool isDirectory = false, string name = null, long? fileSize = default(long?), string id = null, Azure.Storage.Files.Shares.Models.ShareFileItemProperties properties = null, Azure.Storage.Files.Shares.Models.NtfsFileAttributes? fileAttributes = default(Azure.Storage.Files.Shares.Models.NtfsFileAttributes?), string permissionKey = null) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata, Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileDownloadInfo StorageFileDownloadInfo(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletionTime = default(System.DateTimeOffset), string copyStatusDescription = null, string contentDisposition = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServerEncrypted = false, string cacheControl = null, string fileAttributes = null, System.Collections.Generic.IEnumerable contentEncoding = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), byte[] contentHash = null, System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string contentRange = null, string filePermissionKey = null, string contentType = null, string fileId = null, long contentLength = (long)0, string fileParentId = null, System.Collections.Generic.IDictionary metadata = null, System.IO.Stream content = null, string copyId = null) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentType, string contentRange, Azure.ETag eTag, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, string acceptRanges, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, byte[] fileContentHash, bool isServiceEncrypted) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] @@ -537,7 +539,7 @@ public static partial class FilesModelFactory public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, string filePermissionKey = null, string fileAttributes = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string fileId = null, string fileParentId = null, Azure.Storage.Files.Shares.Models.NfsFileMode nfsFileMode = null, string owner = null, string group = null, Azure.Storage.Files.Shares.Models.NfsFileType nfsFileType = default(Azure.Storage.Files.Shares.Models.NfsFileType)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileItem StorageFileItem(bool isDirectory, string name, long? fileSize) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, Azure.Storage.Files.Shares.Models.NtfsFileAttributes fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] @@ -548,13 +550,14 @@ public partial class NfsFileMode public NfsFileMode() { } public bool EffectiveGroupIdentity { get { throw null; } set { } } public bool EffectiveUserIdentity { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Group { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Other { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Owner { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Group { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Other { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Owner { get { throw null; } set { } } public bool StickyBit { get { throw null; } set { } } public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseOctalFileMode(string modeString) { throw null; } public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseSymbolicFileMode(string modeString) { throw null; } public string ToOctalFileMode() { throw null; } + public override string ToString() { throw null; } public string ToSymbolicFileMode() { throw null; } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -601,7 +604,7 @@ internal PermissionInfo() { } public string FilePermissionKey { get { throw null; } } } [System.FlagsAttribute] - public enum RolePermissions + public enum PosixRolePermissions { None = 0, Execute = 1, @@ -692,7 +695,7 @@ public partial class ShareDirectoryCreateOptions public ShareDirectoryCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryGetFilesAndDirectoriesOptions @@ -707,7 +710,7 @@ public partial class ShareDirectoryInfo internal ShareDirectoryInfo() { } public Azure.ETag ETag { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryProperties @@ -717,14 +720,14 @@ internal ShareDirectoryProperties() { } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectorySetHttpHeadersOptions { public ShareDirectorySetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -834,8 +837,8 @@ public ShareFileCopyOptions() { } public Azure.Storage.Files.Shares.Models.PermissionCopyMode? FilePermissionCopyMode { get { throw null; } set { } } public bool? IgnoreReadOnly { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FilePermissionFormat? PermissionFormat { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.CopyableFileSmbProperties SmbPropertiesToCopy { get { throw null; } set { } } } @@ -845,7 +848,7 @@ public ShareFileCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadDetails @@ -871,7 +874,7 @@ internal ShareFileDownloadDetails() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadInfo : System.IDisposable @@ -949,7 +952,7 @@ internal ShareFileInfo() { } public Azure.ETag ETag { get { throw null; } } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileItem @@ -1038,7 +1041,7 @@ internal ShareFileProperties() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileRangeInfo @@ -1080,7 +1083,7 @@ public ShareFileSetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public long? NewSize { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.FlagsAttribute] @@ -1299,8 +1302,8 @@ public ShareSmbSettings() { } } public static partial class SharesModelFactory { + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs index 5fc271b35181b..b72c7b71f7373 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs @@ -523,13 +523,15 @@ public FileSmbProperties() { } } public static partial class FilesModelFactory { - public static Azure.Storage.Files.Shares.Models.FilePosixProperties FileNfsProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } + public static Azure.Storage.Files.Shares.Models.FilePosixProperties FilePosixProperties(Azure.Storage.Files.Shares.Models.NfsFileMode fileMode, string owner, string group, Azure.Storage.Files.Shares.Models.NfsFileType fileType, long? linkCount) { throw null; } + public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileItem ShareFileItem(bool isDirectory = false, string name = null, long? fileSize = default(long?), string id = null, Azure.Storage.Files.Shares.Models.ShareFileItemProperties properties = null, Azure.Storage.Files.Shares.Models.NtfsFileAttributes? fileAttributes = default(Azure.Storage.Files.Shares.Models.NtfsFileAttributes?), string permissionKey = null) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata = null, Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryProperties StorageDirectoryProperties(System.Collections.Generic.IDictionary metadata, Azure.ETag eTag, System.DateTimeOffset lastModified, bool isServerEncrypted, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } public static Azure.Storage.Files.Shares.Models.ShareFileDownloadInfo StorageFileDownloadInfo(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletionTime = default(System.DateTimeOffset), string copyStatusDescription = null, string contentDisposition = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServerEncrypted = false, string cacheControl = null, string fileAttributes = null, System.Collections.Generic.IEnumerable contentEncoding = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), byte[] contentHash = null, System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string contentRange = null, string filePermissionKey = null, string contentType = null, string fileId = null, long contentLength = (long)0, string fileParentId = null, System.Collections.Generic.IDictionary metadata = null, System.IO.Stream content = null, string copyId = null) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, string contentRange = null, Azure.ETag eTag = default(Azure.ETag), System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, string acceptRanges = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, System.Uri copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, byte[] fileContentHash = null, bool isServiceEncrypted = false, Azure.Storage.Files.Shares.Models.ShareLeaseDuration leaseDuration = Azure.Storage.Files.Shares.Models.ShareLeaseDuration.Infinite, Azure.Storage.Files.Shares.Models.ShareLeaseState leaseState = Azure.Storage.Files.Shares.Models.ShareLeaseState.Available, Azure.Storage.Files.Shares.Models.ShareLeaseStatus leaseStatus = Azure.Storage.Files.Shares.Models.ShareLeaseStatus.Locked, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileDownloadDetails StorageFileDownloadProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentType, string contentRange, Azure.ETag eTag, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, string acceptRanges, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, byte[] fileContentHash, bool isServiceEncrypted) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] @@ -537,7 +539,7 @@ public static partial class FilesModelFactory public static Azure.Storage.Files.Shares.Models.ShareFileInfo StorageFileInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), bool isServerEncrypted = false, string filePermissionKey = null, string fileAttributes = null, System.DateTimeOffset fileCreationTime = default(System.DateTimeOffset), System.DateTimeOffset fileLastWriteTime = default(System.DateTimeOffset), System.DateTimeOffset fileChangeTime = default(System.DateTimeOffset), string fileId = null, string fileParentId = null, Azure.Storage.Files.Shares.Models.NfsFileMode nfsFileMode = null, string owner = null, string group = null, Azure.Storage.Files.Shares.Models.NfsFileType nfsFileType = default(Azure.Storage.Files.Shares.Models.NfsFileType)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileItem StorageFileItem(bool isDirectory, string name, long? fileSize) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } + public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified = default(System.DateTimeOffset), System.Collections.Generic.IDictionary metadata = null, long contentLength = (long)0, string contentType = null, Azure.ETag eTag = default(Azure.ETag), byte[] contentHash = null, System.Collections.Generic.IEnumerable contentEncoding = null, string cacheControl = null, string contentDisposition = null, System.Collections.Generic.IEnumerable contentLanguage = null, System.DateTimeOffset copyCompletedOn = default(System.DateTimeOffset), string copyStatusDescription = null, string copyId = null, string copyProgress = null, string copySource = null, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus = Azure.Storage.Files.Shares.Models.CopyStatus.Pending, bool isServerEncrypted = false, Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties posixProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareFileProperties StorageFileProperties(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, long contentLength, string contentType, Azure.ETag eTag, byte[] contentHash, System.Collections.Generic.IEnumerable contentEncoding, string cacheControl, string contentDisposition, System.Collections.Generic.IEnumerable contentLanguage, System.DateTimeOffset copyCompletedOn, string copyStatusDescription, string copyId, string copyProgress, string copySource, Azure.Storage.Files.Shares.Models.CopyStatus copyStatus, bool isServerEncrypted, Azure.Storage.Files.Shares.Models.NtfsFileAttributes fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string filePermissionKey, string fileId, string fileParentId) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] @@ -548,13 +550,14 @@ public partial class NfsFileMode public NfsFileMode() { } public bool EffectiveGroupIdentity { get { throw null; } set { } } public bool EffectiveUserIdentity { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Group { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Other { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.RolePermissions Owner { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Group { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Other { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.PosixRolePermissions Owner { get { throw null; } set { } } public bool StickyBit { get { throw null; } set { } } public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseOctalFileMode(string modeString) { throw null; } public static Azure.Storage.Files.Shares.Models.NfsFileMode ParseSymbolicFileMode(string modeString) { throw null; } public string ToOctalFileMode() { throw null; } + public override string ToString() { throw null; } public string ToSymbolicFileMode() { throw null; } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -601,7 +604,7 @@ internal PermissionInfo() { } public string FilePermissionKey { get { throw null; } } } [System.FlagsAttribute] - public enum RolePermissions + public enum PosixRolePermissions { None = 0, Execute = 1, @@ -692,7 +695,7 @@ public partial class ShareDirectoryCreateOptions public ShareDirectoryCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryGetFilesAndDirectoriesOptions @@ -707,7 +710,7 @@ public partial class ShareDirectoryInfo internal ShareDirectoryInfo() { } public Azure.ETag ETag { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectoryProperties @@ -717,14 +720,14 @@ internal ShareDirectoryProperties() { } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareDirectorySetHttpHeadersOptions { public ShareDirectorySetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] @@ -834,8 +837,8 @@ public ShareFileCopyOptions() { } public Azure.Storage.Files.Shares.Models.PermissionCopyMode? FilePermissionCopyMode { get { throw null; } set { } } public bool? IgnoreReadOnly { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FilePermissionFormat? PermissionFormat { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.CopyableFileSmbProperties SmbPropertiesToCopy { get { throw null; } set { } } } @@ -845,7 +848,7 @@ public ShareFileCreateOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadDetails @@ -871,7 +874,7 @@ internal ShareFileDownloadDetails() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileDownloadInfo : System.IDisposable @@ -949,7 +952,7 @@ internal ShareFileInfo() { } public Azure.ETag ETag { get { throw null; } } public bool IsServerEncrypted { get { throw null; } } public System.DateTimeOffset LastModified { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileItem @@ -1037,7 +1040,7 @@ internal ShareFileProperties() { } public Azure.Storage.Files.Shares.Models.ShareLeaseState LeaseState { get { throw null; } } public Azure.Storage.Files.Shares.Models.ShareLeaseStatus LeaseStatus { get { throw null; } } public System.Collections.Generic.IDictionary Metadata { get { throw null; } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } public partial class ShareFileRangeInfo @@ -1079,7 +1082,7 @@ public ShareFileSetHttpHeadersOptions() { } public Azure.Storage.Files.Shares.Models.ShareFilePermission FilePermission { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareFileHttpHeaders HttpHeaders { get { throw null; } set { } } public long? NewSize { get { throw null; } set { } } - public Azure.Storage.Files.Shares.Models.FilePosixProperties NfsProperties { get { throw null; } set { } } + public Azure.Storage.Files.Shares.Models.FilePosixProperties PosixProperties { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.FileSmbProperties SmbProperties { get { throw null; } set { } } } [System.FlagsAttribute] @@ -1298,8 +1301,8 @@ public ShareSmbSettings() { } } public static partial class SharesModelFactory { + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.FileSmbProperties FileSmbProperties(System.DateTimeOffset? fileChangedOn, string fileId, string parentId) { throw null; } - public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag = default(Azure.ETag), System.DateTimeOffset lastModified = default(System.DateTimeOffset), Azure.Storage.Files.Shares.Models.FileSmbProperties smbProperties = null, Azure.Storage.Files.Shares.Models.FilePosixProperties nfsProperties = null) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.Shares.Models.ShareDirectoryInfo StorageDirectoryInfo(Azure.ETag eTag, System.DateTimeOffset lastModified, string filePermissionKey, string fileAttributes, System.DateTimeOffset fileCreationTime, System.DateTimeOffset fileLastWriteTime, System.DateTimeOffset fileChangeTime, string fileId, string fileParentId) { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/FilePosixProperties.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/FilePosixProperties.cs index a71721c738ea5..84f1a02dd1cc4 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/FilePosixProperties.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/FilePosixProperties.cs @@ -49,7 +49,7 @@ public static partial class FilesModelFactory /// /// Creates a new StorageFileDownloadProperties instance for mocking. /// - public static FilePosixProperties FileNfsProperties( + public static FilePosixProperties FilePosixProperties( NfsFileMode fileMode, string owner, string group, diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/FileSmbProperties.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/FileSmbProperties.cs index 4c8569cc28e6a..4a6069bc9eaac 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/FileSmbProperties.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/FileSmbProperties.cs @@ -74,14 +74,34 @@ public override bool Equals(object other) public override int GetHashCode() => base.GetHashCode(); } - /// - /// FilesModelFactory provides utilities for mocking. - /// + /// + /// FilesModelFactory provides utilities for mocking. + /// + public static partial class FilesModelFactory + { + /// + /// Creates a new FileSmbProperties instance for mocking. + /// + public static FileSmbProperties FileSmbProperties( + DateTimeOffset? fileChangedOn, + string fileId, + string parentId) => new FileSmbProperties + { + FileChangedOn = fileChangedOn, + FileId = fileId, + ParentId = parentId + }; + } + + /// + /// FilesModelFactory provides utilities for mocking. + /// public static partial class SharesModelFactory { /// /// Creates a new FileSmbProperties instance for mocking. /// + [EditorBrowsable(EditorBrowsableState.Never)] public static FileSmbProperties FileSmbProperties( DateTimeOffset? fileChangedOn, string fileId, diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/NfsFileMode.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/NfsFileMode.cs index 3823708a0c17f..d1271a4e52acb 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/NfsFileMode.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/NfsFileMode.cs @@ -15,17 +15,17 @@ public class NfsFileMode /// /// Permissions the owner has over the file or directory. /// - public RolePermissions Owner { get; set; } + public PosixRolePermissions Owner { get; set; } /// /// Permissions the group has over the file or directory. /// - public RolePermissions Group { get; set; } + public PosixRolePermissions Group { get; set; } /// /// Permissions other have over the file or directory. /// - public RolePermissions Other { get; set; } + public PosixRolePermissions Other { get; set; } /// /// Set effective user ID (setuid) on the file or directory. @@ -201,5 +201,8 @@ public static NfsFileMode ParseSymbolicFileMode(string modeString) return nfsFileMode; } + + /// + public override string ToString() => ToSymbolicFileMode(); } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/PosixRolePermissions.cs similarity index 94% rename from sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissions.cs rename to sdk/storage/Azure.Storage.Files.Shares/src/Models/PosixRolePermissions.cs index 0d21ca0f1a5c1..6f7cf6d549678 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/PosixRolePermissions.cs @@ -9,7 +9,7 @@ namespace Azure.Storage.Files.Shares.Models /// Represents file permissions for a specific role. /// [Flags] - public enum RolePermissions + public enum PosixRolePermissions { /// /// No permissions. diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissionExtensions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissionExtensions.cs index 1f075014e8b29..dfb65f0af884d 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissionExtensions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/RolePermissionExtensions.cs @@ -8,16 +8,16 @@ namespace Azure.Storage.Files.Shares.Models { /// - /// Extension methods for . + /// Extension methods for . /// internal static class RolePermissionExtensions { /// /// Parses octal char to RolePermissions. /// - public static RolePermissions ParseOctalRolePermissions(char c) + public static PosixRolePermissions ParseOctalRolePermissions(char c) { - RolePermissions rolePermissions = RolePermissions.None; + PosixRolePermissions rolePermissions = PosixRolePermissions.None; int value = (int)char.GetNumericValue(c); @@ -28,17 +28,17 @@ public static RolePermissions ParseOctalRolePermissions(char c) if ((value & 4) > 0) { - rolePermissions |= RolePermissions.Read; + rolePermissions |= PosixRolePermissions.Read; } if ((value & 2) > 0) { - rolePermissions |= RolePermissions.Write; + rolePermissions |= PosixRolePermissions.Write; } if ((value & 1) > 0) { - rolePermissions |= RolePermissions.Execute; + rolePermissions |= PosixRolePermissions.Execute; } return rolePermissions; @@ -47,21 +47,21 @@ public static RolePermissions ParseOctalRolePermissions(char c) /// /// Returns the octal string representation of this RolePermissions. /// - public static string ToOctalRolePermissions(this RolePermissions rolePermissions) + public static string ToOctalRolePermissions(this PosixRolePermissions rolePermissions) { int result = 0; - if (rolePermissions.HasFlag(RolePermissions.Read)) + if (rolePermissions.HasFlag(PosixRolePermissions.Read)) { result |= 4; } - if (rolePermissions.HasFlag(RolePermissions.Write)) + if (rolePermissions.HasFlag(PosixRolePermissions.Write)) { result |= 2; } - if (rolePermissions.HasFlag(RolePermissions.Execute)) + if (rolePermissions.HasFlag(PosixRolePermissions.Execute)) { result |= 1; } @@ -72,11 +72,11 @@ public static string ToOctalRolePermissions(this RolePermissions rolePermissions /// /// Returns the symbolic string representation of this RolePermissions. /// - public static string ToSymbolicRolePermissions(this RolePermissions rolePermissions) + public static string ToSymbolicRolePermissions(this PosixRolePermissions rolePermissions) { StringBuilder stringBuilder = new StringBuilder(); - if (rolePermissions.HasFlag(RolePermissions.Read)) + if (rolePermissions.HasFlag(PosixRolePermissions.Read)) { stringBuilder.Append("r"); } @@ -85,7 +85,7 @@ public static string ToSymbolicRolePermissions(this RolePermissions rolePermissi stringBuilder.Append("-"); } - if (rolePermissions.HasFlag(RolePermissions.Write)) + if (rolePermissions.HasFlag(PosixRolePermissions.Write)) { stringBuilder.Append("w"); } @@ -94,7 +94,7 @@ public static string ToSymbolicRolePermissions(this RolePermissions rolePermissi stringBuilder.Append("-"); } - if (rolePermissions.HasFlag(RolePermissions.Execute)) + if (rolePermissions.HasFlag(PosixRolePermissions.Execute)) { stringBuilder.Append("x"); } @@ -106,7 +106,7 @@ public static string ToSymbolicRolePermissions(this RolePermissions rolePermissi return stringBuilder.ToString(); } - public static RolePermissions ParseSymbolicRolePermissions(string s, out bool setSticky) + public static PosixRolePermissions ParseSymbolicRolePermissions(string s, out bool setSticky) { if (s == null) { @@ -117,13 +117,13 @@ public static RolePermissions ParseSymbolicRolePermissions(string s, out bool se throw new FormatException($"s must be 3 characters long"); } - RolePermissions rolePermissions = new RolePermissions(); + PosixRolePermissions rolePermissions = new PosixRolePermissions(); setSticky = false; // Read character if (s[0] == 'r') { - rolePermissions |= RolePermissions.Read; + rolePermissions |= PosixRolePermissions.Read; } else if (s[0] != '-') { @@ -133,7 +133,7 @@ public static RolePermissions ParseSymbolicRolePermissions(string s, out bool se // Write character if (s[1] == 'w') { - rolePermissions |= RolePermissions.Write; + rolePermissions |= PosixRolePermissions.Write; } else if (s[1] != '-') { @@ -143,7 +143,7 @@ public static RolePermissions ParseSymbolicRolePermissions(string s, out bool se // Execute character if (s[2] == 'x' || s[2] == 's' || s[2] == 't') { - rolePermissions |= RolePermissions.Execute; + rolePermissions |= PosixRolePermissions.Execute; if (s[2] == 's' || s[2] == 't') { setSticky = true; diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryCreateOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryCreateOptions.cs index 93739869c18c8..018e2ab435208 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryCreateOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryCreateOptions.cs @@ -31,6 +31,6 @@ public class ShareDirectoryCreateOptions /// Note that this property is only applicable to directories created in NFS shares. /// /// - public FilePosixProperties NfsProperties { get; set; } + public FilePosixProperties PosixProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryInfo.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryInfo.cs index 5e7ab16df6cff..5f9527be6fa3c 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryInfo.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryInfo.cs @@ -35,7 +35,7 @@ public class ShareDirectoryInfo /// The directory's NFS properties. /// Only applicable to files in a NFS share. /// - public FilePosixProperties NfsProperties { get; internal set; } + public FilePosixProperties PosixProperties { get; internal set; } /// /// Constructor. @@ -46,7 +46,7 @@ internal ShareDirectoryInfo() { } /// /// FilesModelFactory provides utilities for mocking. /// - public static partial class SharesModelFactory + public static partial class FilesModelFactory { /// /// Creates a new StorageDirectoryInfo instance for mocking. @@ -55,15 +55,21 @@ public static ShareDirectoryInfo StorageDirectoryInfo( ETag eTag = default, DateTimeOffset lastModified = default, FileSmbProperties smbProperties = default, - FilePosixProperties nfsProperties = default) + FilePosixProperties posixProperties = default) => new ShareDirectoryInfo { ETag = eTag, LastModified = lastModified, SmbProperties = smbProperties, - NfsProperties = nfsProperties, + PosixProperties = posixProperties, }; + } + /// + /// FilesModelFactory provides utilities for mocking. + /// + public static partial class SharesModelFactory + { /// /// Creates a new StorageDirectoryInfo instance for mocking. /// diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryProperties.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryProperties.cs index 887f630ba2412..2300c581cd82d 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryProperties.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectoryProperties.cs @@ -46,7 +46,7 @@ public class ShareDirectoryProperties /// NFS properties. /// Note that this property is only applicable to directories created in NFS shares. /// - public FilePosixProperties NfsProperties { get; internal set; } + public FilePosixProperties PosixProperties { get; internal set; } /// /// Constructor. @@ -68,7 +68,7 @@ public static ShareDirectoryProperties StorageDirectoryProperties( DateTimeOffset lastModified = default, bool isServerEncrypted = default, FileSmbProperties smbProperties = default, - FilePosixProperties nfsProperties = default + FilePosixProperties posixProperties = default ) => new ShareDirectoryProperties { @@ -77,7 +77,7 @@ public static ShareDirectoryProperties StorageDirectoryProperties( LastModified = lastModified, IsServerEncrypted = isServerEncrypted, SmbProperties = smbProperties, - NfsProperties = nfsProperties + PosixProperties = posixProperties }; /// diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectorySetHttpHeadersOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectorySetHttpHeadersOptions.cs index d64c2c52f8147..d45a01d3ba305 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectorySetHttpHeadersOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareDirectorySetHttpHeadersOptions.cs @@ -23,6 +23,6 @@ public class ShareDirectorySetHttpHeadersOptions /// Optional properties to set on NFS files. /// Note that this property is only applicable to directories created in NFS shares. /// - public FilePosixProperties NfsProperties { get; set; } + public FilePosixProperties PosixProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCopyOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCopyOptions.cs index 358db59256ff0..c0e2e065ec3ad 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCopyOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCopyOptions.cs @@ -68,6 +68,6 @@ public class ShareFileCopyOptions /// /// Only applicable to NFS Files. NFS properties to set on the destination file. /// - public FilePosixProperties NfsProperties { get; set; } + public FilePosixProperties PosixProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateOptions.cs index 02e9ad752af54..aabd3b3cc3b46 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileCreateOptions.cs @@ -35,6 +35,6 @@ public class ShareFileCreateOptions /// Optional properties to set on NFS files. /// Note that this property is only applicable to files created in NFS shares. /// - public FilePosixProperties NfsProperties { get; set; } + public FilePosixProperties PosixProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadDetails.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadDetails.cs index 5d7958ead83a6..011af2e84cf8f 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadDetails.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadDetails.cs @@ -131,7 +131,7 @@ public partial class ShareFileDownloadDetails /// NFS properties. /// Note that this property is only applicable to files created in NFS shares. /// - public FilePosixProperties NfsProperties { get; internal set; } + public FilePosixProperties PosixProperties { get; internal set; } /// /// Constructor. @@ -169,7 +169,7 @@ public static ShareFileDownloadDetails StorageFileDownloadProperties( ShareLeaseState leaseState = default, ShareLeaseStatus leaseStatus = default, FileSmbProperties smbProperties = default, - FilePosixProperties nfsProperties = default) + FilePosixProperties posixProperties = default) { return new ShareFileDownloadDetails { @@ -193,7 +193,7 @@ public static ShareFileDownloadDetails StorageFileDownloadProperties( LeaseDuration = leaseDuration, LeaseState = leaseState, SmbProperties = smbProperties, - NfsProperties = nfsProperties, + PosixProperties = posixProperties, }; } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileInfo.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileInfo.cs index 37e9d5ab26877..29ce3b2f834b3 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileInfo.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileInfo.cs @@ -40,7 +40,7 @@ public class ShareFileInfo /// The file's NFS properties. /// Only applicable to files in a NFS share. /// - public FilePosixProperties NfsProperties { get; internal set; } + public FilePosixProperties PosixProperties { get; internal set; } /// /// Constructor. @@ -86,7 +86,7 @@ public static ShareFileInfo StorageFileInfo( FileId = fileId, ParentId = fileParentId }, - NfsProperties = new FilePosixProperties + PosixProperties = new FilePosixProperties { FileMode = nfsFileMode, Owner = owner, diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileProperties.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileProperties.cs index efd8c598a473e..722a99361a8ed 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileProperties.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileProperties.cs @@ -124,7 +124,7 @@ public class ShareFileProperties /// NFS properties. /// Note that this property is only applicable to files created in NFS shares. /// - public FilePosixProperties NfsProperties { get; internal set; } + public FilePosixProperties PosixProperties { get; internal set; } /// /// Constructor. @@ -159,7 +159,7 @@ public static ShareFileProperties StorageFileProperties( CopyStatus copyStatus = default, bool isServerEncrypted = default, FileSmbProperties smbProperties = default, - FilePosixProperties nfsProperties = default + FilePosixProperties posixProperties = default ) => new ShareFileProperties { LastModified = lastModified, @@ -180,7 +180,7 @@ public static ShareFileProperties StorageFileProperties( CopyStatus = copyStatus, IsServerEncrypted = isServerEncrypted, SmbProperties = smbProperties, - NfsProperties = nfsProperties, + PosixProperties = posixProperties, }; /// diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSetHttpHeadersOptions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSetHttpHeadersOptions.cs index af62257ceb0ba..6b3e95eae9c03 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSetHttpHeadersOptions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileSetHttpHeadersOptions.cs @@ -36,6 +36,6 @@ public class ShareFileSetHttpHeadersOptions /// Optional properties to set on NFS files. /// Note that this property is only applicable to files created in NFS shares. /// - public FilePosixProperties NfsProperties { get; set; } + public FilePosixProperties PosixProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareClient.cs index 3f23c2d40de16..b1289a14422ef 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareClient.cs @@ -3710,7 +3710,7 @@ public virtual Response CreateDirectory( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, async: false, cancellationToken, operationName: $"{nameof(ShareClient)}.{nameof(CreateDirectory)}") @@ -3766,7 +3766,7 @@ public virtual Response CreateDirectory( smbProperties, filePermission, filePermissionFormat: null, - nfsProperties: null, + posixProperties: null, async: false, cancellationToken, operationName: $"{nameof(ShareClient)}.{nameof(CreateDirectory)}") @@ -3811,7 +3811,7 @@ public virtual async Task> CreateDirectoryAsync( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, async: true, cancellationToken, operationName: $"{nameof(ShareClient)}.{nameof(CreateDirectory)}") @@ -3867,7 +3867,7 @@ public virtual async Task> CreateDirectoryAsync( smbProperties, filePermission, filePermissionFormat: null, - nfsProperties: null, + posixProperties: null, async: true, cancellationToken, operationName: $"{nameof(ShareClient)}.{nameof(CreateDirectory)}") diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareDirectoryClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareDirectoryClient.cs index 7b2af0156f107..4b1d45d8a65df 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareDirectoryClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareDirectoryClient.cs @@ -561,7 +561,7 @@ public virtual Response Create( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, async: false, // async cancellationToken: cancellationToken) .EnsureCompleted(); @@ -608,7 +608,7 @@ public virtual Response Create( smbProperties, filePermission, filePermissionFormat: null, - nfsProperties: null, + posixProperties: null, false, // async cancellationToken) .EnsureCompleted(); @@ -644,7 +644,7 @@ await CreateInternal( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, async: true, cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -691,7 +691,7 @@ await CreateInternal( smbProperties, filePermission, filePermissionFormat: null, - nfsProperties: null, + posixProperties: null, true, // async cancellationToken) .ConfigureAwait(false); @@ -716,7 +716,7 @@ await CreateInternal( /// /// Optional file permission format. /// - /// + /// /// Optional NFS properties. /// /// @@ -742,7 +742,7 @@ internal async Task> CreateInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, - FilePosixProperties nfsProperties, + FilePosixProperties posixProperties, bool async, CancellationToken cancellationToken, string operationName = default) @@ -776,9 +776,9 @@ internal async Task> CreateInternal( filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps?.FilePermissionKey, fileChangeTime: smbProps?.FileChangedOn.ToFileDateTimeString(), - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode.ToOctalFileMode(), + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode.ToOctalFileMode(), cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -793,9 +793,9 @@ internal async Task> CreateInternal( filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps?.FilePermissionKey, fileChangeTime: smbProps?.FileChangedOn.ToFileDateTimeString(), - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode.ToOctalFileMode(), + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode.ToOctalFileMode(), cancellationToken: cancellationToken); } @@ -851,7 +851,7 @@ public virtual Response CreateIfNotExists( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, async: false, cancellationToken).EnsureCompleted(); @@ -898,7 +898,7 @@ public virtual Response CreateIfNotExists( smbProperties, filePermission, filePermissionFormat: null, - nfsProperties: null, + posixProperties: null, async: false, cancellationToken).EnsureCompleted(); @@ -934,7 +934,7 @@ await CreateIfNotExistsInternal( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, async: true, cancellationToken: cancellationToken).ConfigureAwait(false); @@ -981,7 +981,7 @@ await CreateIfNotExistsInternal( smbProperties, filePermission, filePermissionFormat: null, - nfsProperties: null, + posixProperties: null, async: true, cancellationToken).ConfigureAwait(false); @@ -1006,7 +1006,7 @@ await CreateIfNotExistsInternal( /// /// Optional file permission format. /// - /// + /// /// Optional NFS properties. /// /// @@ -1032,7 +1032,7 @@ internal async Task> CreateIfNotExistsInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, - FilePosixProperties nfsProperties, + FilePosixProperties posixProperties, bool async, CancellationToken cancellationToken, string operationName = default) @@ -1049,7 +1049,7 @@ internal async Task> CreateIfNotExistsInternal( smbProperties, filePermission, filePermissionFormat, - nfsProperties, + posixProperties, async, cancellationToken, operationName: operationName ?? $"{nameof(ShareDirectoryClient)}.{nameof(CreateIfNotExists)}") @@ -1585,7 +1585,7 @@ public virtual Response SetHttpHeaders( options?.SmbProperties, options?.FilePermission?.Permission, options?.FilePermission?.PermissionFormat, - options?.NfsProperties, + options?.PosixProperties, false, // async cancellationToken) .EnsureCompleted(); @@ -1618,7 +1618,7 @@ await SetHttpHeadersInternal( options?.SmbProperties, options?.FilePermission?.Permission, options?.FilePermission?.PermissionFormat, - options?.NfsProperties, + options?.PosixProperties, true, // async cancellationToken) .ConfigureAwait(false); @@ -1660,7 +1660,7 @@ public virtual Response SetHttpHeaders( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, false, // async cancellationToken) .EnsureCompleted(); @@ -1700,7 +1700,7 @@ await SetHttpHeadersInternal( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, true, // async cancellationToken) .ConfigureAwait(false); @@ -1722,7 +1722,7 @@ await SetHttpHeadersInternal( /// /// Optional file permission format. /// - /// + /// /// Optional NFS properties. /// /// @@ -1744,7 +1744,7 @@ private async Task> SetHttpHeadersInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, - FilePosixProperties nfsProperties, + FilePosixProperties posixProperties, bool async, CancellationToken cancellationToken) { @@ -1776,9 +1776,9 @@ private async Task> SetHttpHeadersInternal( filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps.FilePermissionKey, fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode.ToOctalFileMode(), + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode.ToOctalFileMode(), cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -1792,9 +1792,9 @@ private async Task> SetHttpHeadersInternal( filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps.FilePermissionKey, fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode.ToOctalFileMode(), + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode.ToOctalFileMode(), cancellationToken: cancellationToken); } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareExtensions.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareExtensions.cs index 89fbdc06c7996..d616f072cf67f 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareExtensions.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareExtensions.cs @@ -116,7 +116,7 @@ internal static ShareDirectoryInfo ToShareDirectoryInfo(this ResponseWithHeaders FileId = response.Headers.FileId, ParentId = response.Headers.FileParentId }, - NfsProperties = new FilePosixProperties + PosixProperties = new FilePosixProperties { FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), Owner = response.Headers.Owner, @@ -148,7 +148,7 @@ internal static ShareDirectoryProperties ToShareDirectoryProperties(this Respons FileId = response.Headers.FileId, ParentId = response.Headers.FileParentId }, - NfsProperties = new FilePosixProperties() + PosixProperties = new FilePosixProperties() { FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), Owner = response.Headers.Owner, @@ -178,7 +178,7 @@ internal static ShareDirectoryInfo ToShareDirectoryInfo(this ResponseWithHeaders FileId = response.Headers.FileId, ParentId = response.Headers.FileParentId }, - NfsProperties = new FilePosixProperties + PosixProperties = new FilePosixProperties { FileMode = NfsFileMode.ParseOctalFileMode(response.Headers.FileMode), Owner = response.Headers.Owner, @@ -316,7 +316,7 @@ internal static ShareFileInfo ToShareFileInfo(this ResponseWithHeaders Create( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, conditions, async: false, cancellationToken) @@ -630,7 +630,7 @@ await CreateInternal( smbProperties: options?.SmbProperties, filePermission: options?.FilePermission?.Permission, filePermissionFormat: options?.FilePermission?.PermissionFormat, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, conditions, async: true, cancellationToken) @@ -696,7 +696,7 @@ public virtual Response Create( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions, async: false, cancellationToken) @@ -757,7 +757,7 @@ public virtual Response Create( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions: default, async: false, cancellationToken) @@ -823,7 +823,7 @@ await CreateInternal( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions, async: true, cancellationToken) @@ -884,7 +884,7 @@ await CreateInternal( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions: default, async: true, cancellationToken) @@ -919,7 +919,7 @@ await CreateInternal( /// /// Optional file permission format. /// - /// + /// /// Optional NFS properties. /// /// @@ -951,7 +951,7 @@ private async Task> CreateInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, - FilePosixProperties nfsProperties, + FilePosixProperties posixProperties, ShareFileRequestConditions conditions, bool async, CancellationToken cancellationToken, @@ -986,10 +986,10 @@ private async Task> CreateInternal( fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString(), fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString(), fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode.ToOctalFileMode(), - nfsFileType: nfsProperties?.FileType, + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode.ToOctalFileMode(), + nfsFileType: posixProperties?.FileType, metadata: metadata, filePermission: filePermission, filePermissionFormat: filePermissionFormat, @@ -1007,10 +1007,10 @@ private async Task> CreateInternal( fileCreationTime: smbProps.FileCreatedOn.ToFileDateTimeString(), fileLastWriteTime: smbProps.FileLastWrittenOn.ToFileDateTimeString(), fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode.ToOctalFileMode(), - nfsFileType: nfsProperties?.FileType, + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode.ToOctalFileMode(), + nfsFileType: posixProperties?.FileType, metadata: metadata, filePermission: filePermission, filePermissionFormat: filePermissionFormat, @@ -1322,7 +1322,7 @@ public virtual Response StartCopy( setArchiveAttribute: options?.Archive, conditions: options?.Conditions, copyableFileSmbProperties: options?.SmbPropertiesToCopy, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, async: false, cancellationToken: cancellationToken) .EnsureCompleted(); @@ -1400,7 +1400,7 @@ public virtual Response StartCopy( setArchiveAttribute, conditions, copyableFileSmbProperties: default, - nfsProperties: default, + posixProperties: default, async: false, cancellationToken) .EnsureCompleted(); @@ -1448,7 +1448,7 @@ public virtual Response StartCopy( setArchiveAttribute: default, conditions: default, copyableFileSmbProperties: default, - nfsProperties: default, + posixProperties: default, async: false, cancellationToken) .EnsureCompleted(); @@ -1493,7 +1493,7 @@ await StartCopyInternal( setArchiveAttribute: options?.Archive, conditions: options?.Conditions, copyableFileSmbProperties: options?.SmbPropertiesToCopy, - nfsProperties: options?.NfsProperties, + posixProperties: options?.PosixProperties, async: true, cancellationToken: cancellationToken). ConfigureAwait(false); @@ -1571,7 +1571,7 @@ await StartCopyInternal( setArchiveAttribute, conditions, copyableFileSmbProperties: default, - nfsProperties: default, + posixProperties: default, async: true, cancellationToken). ConfigureAwait(false); @@ -1619,7 +1619,7 @@ await StartCopyInternal( setArchiveAttribute: default, conditions: default, copyableFileSmbProperties: default, - nfsProperties: default, + posixProperties: default, async: true, cancellationToken). ConfigureAwait(false); @@ -1667,7 +1667,7 @@ await StartCopyInternal( /// /// SMB properties to copy from the source file. /// - /// + /// /// NFS files only. NFS properties to set on the destination file. /// /// @@ -1696,7 +1696,7 @@ private async Task> StartCopyInternal( bool? setArchiveAttribute, ShareFileRequestConditions conditions, CopyableFileSmbProperties? copyableFileSmbProperties, - FilePosixProperties nfsProperties, + FilePosixProperties posixProperties, bool async, CancellationToken cancellationToken) { @@ -1797,13 +1797,13 @@ private async Task> StartCopyInternal( ShareUriBuilder uriBuilder = new ShareUriBuilder(sourceUri); ModeCopyMode? modeCopyMode = null; - if (nfsProperties?.FileMode != null) + if (posixProperties?.FileMode != null) { modeCopyMode = ModeCopyMode.Override; } OwnerCopyMode? ownerCopyMode = null; - if (nfsProperties?.Owner != null || nfsProperties?.Group != null) + if (posixProperties?.Owner != null || posixProperties?.Group != null) { ownerCopyMode = OwnerCopyMode.Override; } @@ -1816,9 +1816,9 @@ private async Task> StartCopyInternal( filePermission: filePermission, filePermissionFormat: filePermissionFormat, filePermissionKey: smbProperties?.FilePermissionKey, - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode?.ToOctalFileMode(), + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode?.ToOctalFileMode(), fileModeCopyMode: modeCopyMode, fileOwnerCopyMode: ownerCopyMode, copyFileSmbInfo: copyFileSmbInfo, @@ -1834,9 +1834,9 @@ private async Task> StartCopyInternal( filePermission: filePermission, filePermissionFormat: filePermissionFormat, filePermissionKey: smbProperties?.FilePermissionKey, - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode?.ToOctalFileMode(), + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode?.ToOctalFileMode(), fileModeCopyMode: modeCopyMode, fileOwnerCopyMode: ownerCopyMode, copyFileSmbInfo: copyFileSmbInfo, @@ -3441,7 +3441,7 @@ public virtual Response SetHttpHeaders( options?.SmbProperties, options?.FilePermission?.Permission, options?.FilePermission?.PermissionFormat, - options?.NfsProperties, + options?.PosixProperties, conditions, async: false, cancellationToken) @@ -3484,7 +3484,7 @@ await SetHttpHeadersInternal( options?.SmbProperties, options?.FilePermission?.Permission, options?.FilePermission?.PermissionFormat, - options?.NfsProperties, + options?.PosixProperties, conditions, async: true, cancellationToken) @@ -3545,7 +3545,7 @@ public virtual Response SetHttpHeaders( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions, async: false, cancellationToken) @@ -3602,7 +3602,7 @@ public virtual Response SetHttpHeaders( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions: default, async: false, cancellationToken) @@ -3663,7 +3663,7 @@ await SetHttpHeadersInternal( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions, async: true, cancellationToken) @@ -3719,7 +3719,7 @@ await SetHttpHeadersInternal( smbProperties, filePermission, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions: default, async: true, cancellationToken) @@ -3751,7 +3751,7 @@ await SetHttpHeadersInternal( /// /// Optional file permission format. /// - /// + /// /// Optional NFS properties. /// /// @@ -3779,7 +3779,7 @@ private async Task> SetHttpHeadersInternal( FileSmbProperties smbProperties, string filePermission, FilePermissionFormat? filePermissionFormat, - FilePosixProperties nfsProperties, + FilePosixProperties posixProperties, ShareFileRequestConditions conditions, bool async, CancellationToken cancellationToken) @@ -3815,9 +3815,9 @@ private async Task> SetHttpHeadersInternal( filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps.FilePermissionKey, fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode.ToOctalFileMode(), + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode.ToOctalFileMode(), fileHttpHeaders: httpHeaders.ToFileHttpHeaders(), shareFileRequestConditions: conditions, cancellationToken: cancellationToken) @@ -3834,9 +3834,9 @@ private async Task> SetHttpHeadersInternal( filePermissionFormat: filePermissionFormat, filePermissionKey: smbProps.FilePermissionKey, fileChangeTime: smbProps.FileChangedOn.ToFileDateTimeString(), - owner: nfsProperties?.Owner, - group: nfsProperties?.Group, - fileMode: nfsProperties?.FileMode.ToOctalFileMode(), + owner: posixProperties?.Owner, + group: posixProperties?.Group, + fileMode: posixProperties?.FileMode.ToOctalFileMode(), fileHttpHeaders: httpHeaders.ToFileHttpHeaders(), shareFileRequestConditions: conditions, cancellationToken: cancellationToken); @@ -6993,6 +6993,7 @@ private async Task> CreateSymbolicLinkInternal( /// /// /// Path of the file to create the hard link to, not including the share. + /// For example: "targetDirectory/targetSubDirectory/.../targetFile" /// /// /// Optional to add conditions @@ -7026,6 +7027,7 @@ public virtual Response CreateHardLink( /// /// /// Path of the file to create the hard link to, not including the share. + /// For example: "targetDirectory/targetSubDirectory/.../targetFile" /// /// /// Optional to add conditions @@ -7266,7 +7268,7 @@ private async Task OpenWriteInternal( smbProperties: default, filePermission: default, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions: options?.OpenConditions, async: async, cancellationToken: cancellationToken) @@ -7297,7 +7299,7 @@ private async Task OpenWriteInternal( smbProperties: default, filePermission: default, filePermissionFormat: default, - nfsProperties: default, + posixProperties: default, conditions: options?.OpenConditions, async: async, cancellationToken: cancellationToken) diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/DirectoryClientTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/DirectoryClientTests.cs index bcf2ea683cc34..3fff06080fc71 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/DirectoryClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/DirectoryClientTests.cs @@ -602,7 +602,7 @@ public async Task CreateAsync_NFS() ShareDirectoryCreateOptions options = new ShareDirectoryCreateOptions { - NfsProperties = new FilePosixProperties + PosixProperties = new FilePosixProperties { Owner = owner, Group = group, @@ -614,10 +614,10 @@ public async Task CreateAsync_NFS() Response response = await directory.CreateAsync(options); // Assert - Assert.AreEqual(NfsFileType.Directory, response.Value.NfsProperties.FileType); - Assert.AreEqual(owner, response.Value.NfsProperties.Owner); - Assert.AreEqual(group, response.Value.NfsProperties.Group); - Assert.AreEqual(fileMode, response.Value.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(NfsFileType.Directory, response.Value.PosixProperties.FileType); + Assert.AreEqual(owner, response.Value.PosixProperties.Owner); + Assert.AreEqual(group, response.Value.PosixProperties.Group); + Assert.AreEqual(fileMode, response.Value.PosixProperties.FileMode.ToOctalFileMode()); Assert.IsNull(response.Value.SmbProperties.FileAttributes); Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); @@ -1031,12 +1031,12 @@ public async Task GetPropertiesAsync_NFS() Response response = await test.Directory.GetPropertiesAsync(); // Assert - Assert.AreEqual(NfsFileType.Directory, response.Value.NfsProperties.FileType); - Assert.AreEqual("0", response.Value.NfsProperties.Owner); - Assert.AreEqual("0", response.Value.NfsProperties.Group); - Assert.AreEqual("0755", response.Value.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(NfsFileType.Directory, response.Value.PosixProperties.FileType); + Assert.AreEqual("0", response.Value.PosixProperties.Owner); + Assert.AreEqual("0", response.Value.PosixProperties.Group); + Assert.AreEqual("0755", response.Value.PosixProperties.FileMode.ToOctalFileMode()); - Assert.IsNull(response.Value.NfsProperties.LinkCount); + Assert.IsNull(response.Value.PosixProperties.LinkCount); Assert.IsNull(response.Value.SmbProperties.FileAttributes); Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); } @@ -1293,7 +1293,7 @@ public async Task SetHttpHeadersAsync_NFS() ShareDirectorySetHttpHeadersOptions options = new ShareDirectorySetHttpHeadersOptions { - NfsProperties = new FilePosixProperties + PosixProperties = new FilePosixProperties { Owner = owner, Group = group, @@ -1305,11 +1305,11 @@ public async Task SetHttpHeadersAsync_NFS() Response response = await test.Directory.SetHttpHeadersAsync(options); // Assert - Assert.AreEqual(owner, response.Value.NfsProperties.Owner); - Assert.AreEqual(group, response.Value.NfsProperties.Group); - Assert.AreEqual(fileMode, response.Value.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(owner, response.Value.PosixProperties.Owner); + Assert.AreEqual(group, response.Value.PosixProperties.Group); + Assert.AreEqual(fileMode, response.Value.PosixProperties.FileMode.ToOctalFileMode()); - Assert.IsNull(response.Value.NfsProperties.LinkCount); + Assert.IsNull(response.Value.PosixProperties.LinkCount); Assert.IsNull(response.Value.SmbProperties.FileAttributes); Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); } diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs index b86ed902cfeaf..782fbc7f0c807 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs @@ -743,7 +743,7 @@ public async Task CreateAsync_NFS() ShareFileCreateOptions options = new ShareFileCreateOptions { - NfsProperties = new FilePosixProperties + PosixProperties = new FilePosixProperties { Owner = owner, Group = group, @@ -757,10 +757,10 @@ public async Task CreateAsync_NFS() options: options); // Assert - Assert.AreEqual(NfsFileType.Regular, response.Value.NfsProperties.FileType); - Assert.AreEqual(owner, response.Value.NfsProperties.Owner); - Assert.AreEqual(group, response.Value.NfsProperties.Group); - Assert.AreEqual(fileMode, response.Value.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(NfsFileType.Regular, response.Value.PosixProperties.FileType); + Assert.AreEqual(owner, response.Value.PosixProperties.Owner); + Assert.AreEqual(group, response.Value.PosixProperties.Group); + Assert.AreEqual(fileMode, response.Value.PosixProperties.FileMode.ToOctalFileMode()); Assert.IsNull(response.Value.SmbProperties.FileAttributes); Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); @@ -1377,11 +1377,11 @@ public async Task GetProperties_NFS() Response response = await test.File.GetPropertiesAsync(); // Assert - Assert.AreEqual(NfsFileType.Regular, response.Value.NfsProperties.FileType); - Assert.AreEqual("0", response.Value.NfsProperties.Owner); - Assert.AreEqual("0", response.Value.NfsProperties.Group); - Assert.AreEqual("0664", response.Value.NfsProperties.FileMode.ToOctalFileMode()); - Assert.AreEqual(1, response.Value.NfsProperties.LinkCount); + Assert.AreEqual(NfsFileType.Regular, response.Value.PosixProperties.FileType); + Assert.AreEqual("0", response.Value.PosixProperties.Owner); + Assert.AreEqual("0", response.Value.PosixProperties.Group); + Assert.AreEqual("0664", response.Value.PosixProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(1, response.Value.PosixProperties.LinkCount); Assert.IsNull(response.Value.SmbProperties.FileAttributes); Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); @@ -1786,7 +1786,7 @@ public async Task SetHttpHeadersAsync_NFS() ShareFileSetHttpHeadersOptions options = new ShareFileSetHttpHeadersOptions { - NfsProperties = new FilePosixProperties + PosixProperties = new FilePosixProperties { Owner = owner, Group = group, @@ -1798,10 +1798,10 @@ public async Task SetHttpHeadersAsync_NFS() Response response = await test.File.SetHttpHeadersAsync(options); // Assert - Assert.AreEqual(owner, response.Value.NfsProperties.Owner); - Assert.AreEqual(group, response.Value.NfsProperties.Group); - Assert.AreEqual(fileMode, response.Value.NfsProperties.FileMode.ToOctalFileMode()); - Assert.AreEqual(1, response.Value.NfsProperties.LinkCount); + Assert.AreEqual(owner, response.Value.PosixProperties.Owner); + Assert.AreEqual(group, response.Value.PosixProperties.Group); + Assert.AreEqual(fileMode, response.Value.PosixProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(1, response.Value.PosixProperties.LinkCount); Assert.IsNull(response.Value.SmbProperties.FileAttributes); Assert.IsNull(response.Value.SmbProperties.FilePermissionKey); @@ -2565,7 +2565,7 @@ await source.File.UploadRangeAsync( ShareFileCopyOptions options = new ShareFileCopyOptions { - NfsProperties = new FilePosixProperties() + PosixProperties = new FilePosixProperties() }; if (overwriteOwnerAndMode) @@ -2573,15 +2573,15 @@ await source.File.UploadRangeAsync( owner = "54321"; group = "12345"; fileMode = NfsFileMode.ParseOctalFileMode("7777"); - options.NfsProperties.Owner = owner; - options.NfsProperties.Group = group; - options.NfsProperties.FileMode = fileMode; + options.PosixProperties.Owner = owner; + options.PosixProperties.Group = group; + options.PosixProperties.FileMode = fileMode; } else { - owner = sourceProperties.Value.NfsProperties.Owner; - fileMode = sourceProperties.Value.NfsProperties.FileMode; - group = sourceProperties.Value.NfsProperties.Group; + owner = sourceProperties.Value.PosixProperties.Owner; + fileMode = sourceProperties.Value.PosixProperties.FileMode; + group = sourceProperties.Value.PosixProperties.Group; } // Act @@ -2589,9 +2589,9 @@ await source.File.UploadRangeAsync( Response destinationProperties = await destination.File.GetPropertiesAsync(); // Assert - Assert.AreEqual(owner, destinationProperties.Value.NfsProperties.Owner); - Assert.AreEqual(group, destinationProperties.Value.NfsProperties.Group); - Assert.AreEqual(fileMode.ToOctalFileMode(), destinationProperties.Value.NfsProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(owner, destinationProperties.Value.PosixProperties.Owner); + Assert.AreEqual(group, destinationProperties.Value.PosixProperties.Group); + Assert.AreEqual(fileMode.ToOctalFileMode(), destinationProperties.Value.PosixProperties.FileMode.ToOctalFileMode()); } [RecordedTest] @@ -3243,10 +3243,10 @@ await file.UploadRangeAsync( }); // Assert - Assert.AreEqual("0", response.Value.Details.NfsProperties.Owner); - Assert.AreEqual("0", response.Value.Details.NfsProperties.Group); - Assert.AreEqual("0664", response.Value.Details.NfsProperties.FileMode.ToOctalFileMode()); - Assert.AreEqual(1, response.Value.Details.NfsProperties.LinkCount); + Assert.AreEqual("0", response.Value.Details.PosixProperties.Owner); + Assert.AreEqual("0", response.Value.Details.PosixProperties.Group); + Assert.AreEqual("0664", response.Value.Details.PosixProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(1, response.Value.Details.PosixProperties.LinkCount); Assert.IsNull(response.Value.Details.SmbProperties.FileAttributes); Assert.IsNull(response.Value.Details.SmbProperties.FilePermissionKey); @@ -6834,9 +6834,9 @@ public async Task CreateGetSymbolicLinkAsync() options: options); // Assert - Assert.AreEqual(NfsFileType.Symlink, response.Value.NfsProperties.FileType); - Assert.AreEqual(owner, response.Value.NfsProperties.Owner); - Assert.AreEqual(group, response.Value.NfsProperties.Group); + Assert.AreEqual(NfsFileType.Symlink, response.Value.PosixProperties.FileType); + Assert.AreEqual(owner, response.Value.PosixProperties.Owner); + Assert.AreEqual(group, response.Value.PosixProperties.Group); Assert.AreEqual(fileCreatedOn, response.Value.SmbProperties.FileCreatedOn); Assert.AreEqual(fileLastWrittenOn, response.Value.SmbProperties.FileLastWrittenOn); @@ -6919,11 +6919,11 @@ public async Task CreateHardLinkAsync() conditions: new ShareFileRequestConditions() { LeaseId = lease.LeaseId }); // Assert - Assert.AreEqual(NfsFileType.Regular, response.Value.NfsProperties.FileType); - Assert.AreEqual("0", response.Value.NfsProperties.Owner); - Assert.AreEqual("0", response.Value.NfsProperties.Group); - Assert.AreEqual("0664", response.Value.NfsProperties.FileMode.ToOctalFileMode()); - Assert.AreEqual(2, response.Value.NfsProperties.LinkCount); + Assert.AreEqual(NfsFileType.Regular, response.Value.PosixProperties.FileType); + Assert.AreEqual("0", response.Value.PosixProperties.Owner); + Assert.AreEqual("0", response.Value.PosixProperties.Group); + Assert.AreEqual("0664", response.Value.PosixProperties.FileMode.ToOctalFileMode()); + Assert.AreEqual(2, response.Value.PosixProperties.LinkCount); Assert.IsNotNull(response.Value.SmbProperties.FileCreatedOn); Assert.IsNotNull(response.Value.SmbProperties.FileLastWrittenOn); From 61039397dfed4b84085c6e4dbcc389de4ba02543 Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:45:10 -0600 Subject: [PATCH 08/10] Renamed Symlink SymLink (#47413) --- .../api/Azure.Storage.Files.Shares.net6.0.cs | 2 +- .../api/Azure.Storage.Files.Shares.net8.0.cs | 2 +- .../api/Azure.Storage.Files.Shares.netstandard2.0.cs | 2 +- .../src/Generated/Models/NfsFileType.cs | 6 +++--- sdk/storage/Azure.Storage.Files.Shares/src/autorest.md | 2 +- .../Azure.Storage.Files.Shares/tests/FileClientTests.cs | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs index b72c7b71f7373..151a3be6555ff 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs @@ -568,7 +568,7 @@ public NfsFileMode() { } public NfsFileType(string value) { throw null; } public static Azure.Storage.Files.Shares.Models.NfsFileType Directory { get { throw null; } } public static Azure.Storage.Files.Shares.Models.NfsFileType Regular { get { throw null; } } - public static Azure.Storage.Files.Shares.Models.NfsFileType Symlink { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType SymLink { get { throw null; } } public bool Equals(Azure.Storage.Files.Shares.Models.NfsFileType other) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public override bool Equals(object obj) { throw null; } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs index 238552cdc8567..6429bb6e3e396 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs @@ -568,7 +568,7 @@ public NfsFileMode() { } public NfsFileType(string value) { throw null; } public static Azure.Storage.Files.Shares.Models.NfsFileType Directory { get { throw null; } } public static Azure.Storage.Files.Shares.Models.NfsFileType Regular { get { throw null; } } - public static Azure.Storage.Files.Shares.Models.NfsFileType Symlink { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType SymLink { get { throw null; } } public bool Equals(Azure.Storage.Files.Shares.Models.NfsFileType other) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public override bool Equals(object obj) { throw null; } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs index b72c7b71f7373..151a3be6555ff 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs @@ -568,7 +568,7 @@ public NfsFileMode() { } public NfsFileType(string value) { throw null; } public static Azure.Storage.Files.Shares.Models.NfsFileType Directory { get { throw null; } } public static Azure.Storage.Files.Shares.Models.NfsFileType Regular { get { throw null; } } - public static Azure.Storage.Files.Shares.Models.NfsFileType Symlink { get { throw null; } } + public static Azure.Storage.Files.Shares.Models.NfsFileType SymLink { get { throw null; } } public bool Equals(Azure.Storage.Files.Shares.Models.NfsFileType other) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public override bool Equals(object obj) { throw null; } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/NfsFileType.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/NfsFileType.cs index 2e40095b63aa6..efc9c74d412e6 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/NfsFileType.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/Models/NfsFileType.cs @@ -24,14 +24,14 @@ public NfsFileType(string value) private const string RegularValue = "Regular"; private const string DirectoryValue = "Directory"; - private const string SymlinkValue = "Symlink"; + private const string SymLinkValue = "SymLink"; /// Regular. public static NfsFileType Regular { get; } = new NfsFileType(RegularValue); /// Directory. public static NfsFileType Directory { get; } = new NfsFileType(DirectoryValue); - /// Symlink. - public static NfsFileType Symlink { get; } = new NfsFileType(SymlinkValue); + /// SymLink. + public static NfsFileType SymLink { get; } = new NfsFileType(SymLinkValue); /// Determines if two values are the same. public static bool operator ==(NfsFileType left, NfsFileType right) => left.Equals(right); /// Determines if two values are not the same. diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md index 9780abf2b3633..4a5ef0d967ae6 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/9ba64409c50ecbddc416ae30432ab0d17d0d97d6/specification/storage/data-plane/Microsoft.FileStorage/stable/2025-05-05/file.json + - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/fa7f2d0413dc34ae8d07482a24790ebda2b42194/specification/storage/data-plane/Microsoft.FileStorage/stable/2025-05-05/file.json generation1-convenience-client: true # https://github.com/Azure/autorest/issues/4075 skip-semantics-validation: true diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs index 782fbc7f0c807..f2f9e33686a49 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/FileClientTests.cs @@ -6834,7 +6834,7 @@ public async Task CreateGetSymbolicLinkAsync() options: options); // Assert - Assert.AreEqual(NfsFileType.Symlink, response.Value.PosixProperties.FileType); + Assert.AreEqual(NfsFileType.SymLink, response.Value.PosixProperties.FileType); Assert.AreEqual(owner, response.Value.PosixProperties.Owner); Assert.AreEqual(group, response.Value.PosixProperties.Group); Assert.AreEqual(fileCreatedOn, response.Value.SmbProperties.FileCreatedOn); From 3510407ce141d45021d79f5556335ce2760dab0a Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Wed, 4 Dec 2024 15:59:17 -0600 Subject: [PATCH 09/10] Updated autorest (#47418) --- sdk/storage/Azure.Storage.Files.Shares/src/autorest.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md index 4a5ef0d967ae6..b2678f62dd5fa 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/fa7f2d0413dc34ae8d07482a24790ebda2b42194/specification/storage/data-plane/Microsoft.FileStorage/stable/2025-05-05/file.json + - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/d18a495685ccec837b72891b4deea017f62e8190/specification/storage/data-plane/Microsoft.FileStorage/stable/2025-05-05/file.json generation1-convenience-client: true # https://github.com/Azure/autorest/issues/4075 skip-semantics-validation: true From 46a0fa44a28fa11eaf46df888d06c1cc39593986 Mon Sep 17 00:00:00 2001 From: Sean McCullough <44180881+seanmcc-msft@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:41:30 -0600 Subject: [PATCH 10/10] Enabled STG 97 live tests (#47185) --- .../api/Azure.Storage.Blobs.net6.0.cs | 4 +- .../api/Azure.Storage.Blobs.net8.0.cs | 4 +- .../api/Azure.Storage.Blobs.netstandard2.0.cs | 4 +- .../api/Azure.Storage.Blobs.netstandard2.1.cs | 4 +- sdk/storage/Azure.Storage.Blobs/assets.json | 2 +- .../tests/BlobBaseClientTests.cs | 2 +- .../tests/ContainerClientTests.cs | 2 +- .../tests/ServiceClientTests.cs | 2 +- .../src/Shared/StorageVersionExtensions.cs | 2 +- .../Azure.Storage.Files.DataLake.net6.0.cs | 2 +- .../Azure.Storage.Files.DataLake.net8.0.cs | 2 +- ...e.Storage.Files.DataLake.netstandard2.0.cs | 2 +- .../api/Azure.Storage.Files.Shares.net6.0.cs | 2 +- .../api/Azure.Storage.Files.Shares.net8.0.cs | 2 +- ...ure.Storage.Files.Shares.netstandard2.0.cs | 2 +- .../Azure.Storage.Files.Shares/assets.json | 2 +- .../tests/ServiceClientTests.cs | 39 +++++++++++++------ .../api/Azure.Storage.Queues.net6.0.cs | 4 +- .../api/Azure.Storage.Queues.net8.0.cs | 4 +- .../Azure.Storage.Queues.netstandard2.0.cs | 4 +- .../Azure.Storage.Queues.netstandard2.1.cs | 4 +- 21 files changed, 55 insertions(+), 40 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs index 18170fd30ce7a..51769f818526a 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs @@ -51,7 +51,7 @@ public BlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredential c } public partial class BlobClientOptions : Azure.Core.ClientOptions { - public BlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05) { } + public BlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05) { } public Azure.Storage.Blobs.Models.BlobAudience? Audience { get { throw null; } set { } } public Azure.Storage.Blobs.Models.CustomerProvidedKey? CustomerProvidedKey { get { throw null; } set { } } public bool EnableTenantDiscovery { get { throw null; } set { } } @@ -1851,7 +1851,7 @@ public PageBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredenti } public partial class SpecializedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions { - public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } + public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } } public static partial class SpecializedBlobExtensions diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs index c2c4e23aecd48..40b216f43dfad 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net8.0.cs @@ -51,7 +51,7 @@ public BlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredential c } public partial class BlobClientOptions : Azure.Core.ClientOptions { - public BlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05) { } + public BlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05) { } public Azure.Storage.Blobs.Models.BlobAudience? Audience { get { throw null; } set { } } public Azure.Storage.Blobs.Models.CustomerProvidedKey? CustomerProvidedKey { get { throw null; } set { } } public bool EnableTenantDiscovery { get { throw null; } set { } } @@ -1851,7 +1851,7 @@ public PageBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredenti } public partial class SpecializedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions { - public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } + public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } } public static partial class SpecializedBlobExtensions diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 8707ec2108684..41c77a5a3f5fd 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -51,7 +51,7 @@ public BlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredential c } public partial class BlobClientOptions : Azure.Core.ClientOptions { - public BlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05) { } + public BlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05) { } public Azure.Storage.Blobs.Models.BlobAudience? Audience { get { throw null; } set { } } public Azure.Storage.Blobs.Models.CustomerProvidedKey? CustomerProvidedKey { get { throw null; } set { } } public bool EnableTenantDiscovery { get { throw null; } set { } } @@ -1851,7 +1851,7 @@ public PageBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredenti } public partial class SpecializedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions { - public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } + public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } } public static partial class SpecializedBlobExtensions diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs index 8707ec2108684..41c77a5a3f5fd 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs @@ -51,7 +51,7 @@ public BlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredential c } public partial class BlobClientOptions : Azure.Core.ClientOptions { - public BlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05) { } + public BlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05) { } public Azure.Storage.Blobs.Models.BlobAudience? Audience { get { throw null; } set { } } public Azure.Storage.Blobs.Models.CustomerProvidedKey? CustomerProvidedKey { get { throw null; } set { } } public bool EnableTenantDiscovery { get { throw null; } set { } } @@ -1851,7 +1851,7 @@ public PageBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredenti } public partial class SpecializedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions { - public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_01_05) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } + public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2025_05_05) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } } public static partial class SpecializedBlobExtensions diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index 0facb33e2a026..73d4916e57607 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_5c382dfb14" + "Tag": "net/storage/Azure.Storage.Blobs_4d91c5bf9c" } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTests.cs index 8e51aa708c24b..d0ef9cabe003f 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTests.cs @@ -8597,7 +8597,7 @@ public async Task GetAccountInfoAsync_Error() // Act await TestHelper.AssertExpectedExceptionAsync( blobClient.GetAccountInfoAsync(), - e => Assert.AreEqual("ResourceNotFound", e.ErrorCode)); + e => Assert.AreEqual("NoAuthenticationInformation", e.ErrorCode)); } public IEnumerable AccessConditions_Data diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ContainerClientTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ContainerClientTests.cs index 451586836669f..e22726ff4fca4 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ContainerClientTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ContainerClientTests.cs @@ -4319,7 +4319,7 @@ public async Task GetAccountInfoAsync_Error() // Act await TestHelper.AssertExpectedExceptionAsync( containerClient.GetAccountInfoAsync(), - e => Assert.AreEqual("ResourceNotFound", e.ErrorCode)); + e => Assert.AreEqual("NoAuthenticationInformation", e.ErrorCode)); } [RecordedTest] diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ServiceClientTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ServiceClientTests.cs index c10fcfbd73ab2..34b15d3036694 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ServiceClientTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ServiceClientTests.cs @@ -522,7 +522,7 @@ public async Task GetAccountInfoAsync_Error() // Act await TestHelper.AssertExpectedExceptionAsync( service.GetAccountInfoAsync(), - e => Assert.AreEqual("ResourceNotFound", e.ErrorCode)); + e => Assert.AreEqual("NoAuthenticationInformation", e.ErrorCode)); } [RecordedTest] diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs index 3b4b0a4cafd92..b657980045ffe 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs @@ -46,7 +46,7 @@ internal static class StorageVersionExtensions /// public const ServiceVersion LatestVersion = #if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK || BlobDataMovementSDK || ShareDataMovementSDK - ServiceVersion.V2025_01_05; + ServiceVersion.V2025_05_05; #else ERROR_STORAGE_SERVICE_NOT_DEFINED; #endif diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs index a09b75e4ef259..7a1c3bd97cad0 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs @@ -2,7 +2,7 @@ namespace Azure.Storage.Files.DataLake { public partial class DataLakeClientOptions : Azure.Core.ClientOptions { - public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_01_05) { } + public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_05_05) { } public Azure.Storage.Files.DataLake.Models.DataLakeAudience? Audience { get { throw null; } set { } } public Azure.Storage.Files.DataLake.Models.DataLakeCustomerProvidedKey? CustomerProvidedKey { get { throw null; } set { } } public bool EnableTenantDiscovery { get { throw null; } set { } } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs index f743947be089d..e1cee30d23481 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net8.0.cs @@ -2,7 +2,7 @@ namespace Azure.Storage.Files.DataLake { public partial class DataLakeClientOptions : Azure.Core.ClientOptions { - public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_01_05) { } + public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_05_05) { } public Azure.Storage.Files.DataLake.Models.DataLakeAudience? Audience { get { throw null; } set { } } public Azure.Storage.Files.DataLake.Models.DataLakeCustomerProvidedKey? CustomerProvidedKey { get { throw null; } set { } } public bool EnableTenantDiscovery { get { throw null; } set { } } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index a09b75e4ef259..7a1c3bd97cad0 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -2,7 +2,7 @@ namespace Azure.Storage.Files.DataLake { public partial class DataLakeClientOptions : Azure.Core.ClientOptions { - public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_01_05) { } + public DataLakeClientOptions(Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion version = Azure.Storage.Files.DataLake.DataLakeClientOptions.ServiceVersion.V2025_05_05) { } public Azure.Storage.Files.DataLake.Models.DataLakeAudience? Audience { get { throw null; } set { } } public Azure.Storage.Files.DataLake.Models.DataLakeCustomerProvidedKey? CustomerProvidedKey { get { throw null; } set { } } public bool EnableTenantDiscovery { get { throw null; } set { } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs index 151a3be6555ff..68144bac3267c 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs @@ -115,7 +115,7 @@ public ShareClient(System.Uri shareUri, Azure.Storage.StorageSharedKeyCredential } public partial class ShareClientOptions : Azure.Core.ClientOptions { - public ShareClientOptions(Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion version = Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion.V2025_01_05) { } + public ShareClientOptions(Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion version = Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion.V2025_05_05) { } public bool? AllowSourceTrailingDot { get { throw null; } set { } } public bool? AllowTrailingDot { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareAudience? Audience { get { throw null; } set { } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs index 6429bb6e3e396..27e880fc4ae81 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net8.0.cs @@ -115,7 +115,7 @@ public ShareClient(System.Uri shareUri, Azure.Storage.StorageSharedKeyCredential } public partial class ShareClientOptions : Azure.Core.ClientOptions { - public ShareClientOptions(Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion version = Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion.V2025_01_05) { } + public ShareClientOptions(Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion version = Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion.V2025_05_05) { } public bool? AllowSourceTrailingDot { get { throw null; } set { } } public bool? AllowTrailingDot { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareAudience? Audience { get { throw null; } set { } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs index 151a3be6555ff..68144bac3267c 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs @@ -115,7 +115,7 @@ public ShareClient(System.Uri shareUri, Azure.Storage.StorageSharedKeyCredential } public partial class ShareClientOptions : Azure.Core.ClientOptions { - public ShareClientOptions(Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion version = Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion.V2025_01_05) { } + public ShareClientOptions(Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion version = Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion.V2025_05_05) { } public bool? AllowSourceTrailingDot { get { throw null; } set { } } public bool? AllowTrailingDot { get { throw null; } set { } } public Azure.Storage.Files.Shares.Models.ShareAudience? Audience { get { throw null; } set { } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index be24571a39a8d..0369c7abc0d76 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_0600fdfad4" + "Tag": "net/storage/Azure.Storage.Files.Shares_6cb8aed97e" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ServiceClientTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ServiceClientTests.cs index e52b1b2da1bc6..52916c13b0cd7 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ServiceClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ServiceClientTests.cs @@ -151,21 +151,36 @@ public async Task GetSetServicePropertiesAsync_SmbMultiChannel() Response propertiesResponse = await service.GetPropertiesAsync(); ShareServiceProperties properties = propertiesResponse.Value; - // Assert - Assert.IsFalse(properties.Protocol.Smb.Multichannel.Enabled); + if (properties.Protocol.Smb.Multichannel.Enabled == true) + { + // Act + properties.Protocol.Smb.Multichannel.Enabled = false; + await service.SetPropertiesAsync(properties); + propertiesResponse = await service.GetPropertiesAsync(); + properties = propertiesResponse.Value; - // Act - properties.Protocol.Smb.Multichannel.Enabled = true; - await service.SetPropertiesAsync(properties); - propertiesResponse = await service.GetPropertiesAsync(); - properties = propertiesResponse.Value; + // Assert + Assert.IsFalse(properties.Protocol.Smb.Multichannel.Enabled); - // Assert - Assert.IsTrue(properties.Protocol.Smb.Multichannel.Enabled); + // Cleanup + properties.Protocol.Smb.Multichannel.Enabled = true; + await service.SetPropertiesAsync(properties); + } + else + { + // Act + properties.Protocol.Smb.Multichannel.Enabled = true; + await service.SetPropertiesAsync(properties); + propertiesResponse = await service.GetPropertiesAsync(); + properties = propertiesResponse.Value; - // Cleanup - properties.Protocol.Smb.Multichannel.Enabled = false; - await service.SetPropertiesAsync(properties); + // Assert + Assert.IsTrue(properties.Protocol.Smb.Multichannel.Enabled); + + // Cleanup + properties.Protocol.Smb.Multichannel.Enabled = false; + await service.SetPropertiesAsync(properties); + } } [RecordedTest] diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net6.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net6.0.cs index 3bcbeb0dc9fb1..29bb96d69e4a9 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net6.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.net6.0.cs @@ -74,7 +74,7 @@ public QueueClient(System.Uri queueUri, Azure.Storage.StorageSharedKeyCredential } public partial class QueueClientOptions : Azure.Core.ClientOptions { - public QueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2025_01_05) { } + public QueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2025_05_05) { } public Azure.Storage.Queues.Models.QueueAudience? Audience { get { throw null; } set { } } public bool EnableTenantDiscovery { get { throw null; } set { } } public System.Uri GeoRedundantSecondaryUri { get { throw null; } set { } } @@ -427,7 +427,7 @@ public event System.EventHandler